perf bench futex: cache align the worer struct

By default the application uses malloc() and all available CPUs. This patch introduces NUMA support which means:
- memory is allocated node local via numa_alloc_local()
- all CPUs of the specified NUMA node are used. This is also true if the number of threads set is greater than the number of CPUs available on this node.

Signed-off-by Sebastian Andrzej Siewior <[email protected]>

---
 tools/perf/bench/Build        |  4 ++
 tools/perf/bench/futex-hash.c | 87 ++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 81 insertions(+), 10 deletions(-)

tools/perf/bench/Build +0 -0

index 60bf11943047..9e6e518d7d62 100644

tools/perf/bench/Build +0 -0

 perf-y += sched-messaging.o
 perf-y += sched-pipe.o
 perf-y += mem-functions.o

tools/perf/bench/futex-hash.c +77 -10

index d9e5e80bb4d0..8db4a5bd6a4e 100644
@@ -25,6 +25,9 @@
 
 #include <err.h>
 #include <sys/time.h>
+#ifdef CONFIG_NUMA
+#include <numa.h>
+#endif
 
 static unsigned int nthreads = 0;
 static unsigned int nsecs    = 10;

@@ -32,6 +35,7 @@ static unsigned int nsecs    = 10;
 static unsigned int nfutexes = 1024;
 static bool fshared = false, done = false, silent = false;
 static int futex_flag = 0;
+static int numa_node = -1;
 
 struct timeval start, end, runtime;
 static pthread_mutex_t thread_lock;

@@ -55,9 +59,28 @@ static const struct option options[] = {
 	OPT_UINTEGER('f', "futexes", &nfutexes, "Specify amount of futexes per threads"),
 	OPT_BOOLEAN( 's', "silent",  &silent,   "Silent mode: do not display data/details"),
 	OPT_BOOLEAN( 'S', "shared",  &fshared,  "Use shared futexes instead of private ones"),
+#ifdef CONFIG_NUMA
+	OPT_INTEGER( 'n', "numa",   &numa_node,  "Specify the NUMA node"),
+#endif
 	OPT_END()
 };
 
+#ifndef CONFIG_NUMA
+static int numa_run_on_node(int node __maybe_unused) { return 0; }
+static int numa_node_of_cpu(int node __maybe_unused) { return 0; }
+static void *numa_alloc_local(size_t size) { return malloc(size); }
+static void numa_free(void *p, size_t size __maybe_unused) { return free(p); }
+#endif
+
+static bool cpu_is_local(int cpu)
+{
+	if (numa_node < 0)
+		return true;
+	if (numa_node_of_cpu(cpu) == numa_node)
+		return true;
+	return false;
+}
+
 static const char * const bench_futex_hash_usage[] = {
 	"perf bench futex hash <options>",
 	NULL

@@ -123,6 +146,8 @@ int bench_futex_hash(int argc, const char **argv,
 	unsigned int i, ncpus;
 	pthread_attr_t thread_attr;
 	struct worker *worker = NULL;
+	char *node_str = NULL;
+	unsigned int cpunum;
 
 	argc = parse_options(argc, argv, options, bench_futex_hash_usage, 0);
 	if (argc) {

@@ -136,18 +161,50 @@ int bench_futex_hash(int argc, const char **argv,
 	act.sa_sigaction = toggle_done;
 	sigaction(SIGINT, &act, NULL);
 
-	if (!nthreads) /* default to the number of CPUs */
-		nthreads = ncpus;
+	if (!nthreads) {
+		/* default to the number of CPUs per NUMA node */
+		if (numa_node < 0) {
+			nthreads = ncpus;
+		} else {
+			for (i = 0; i < ncpus; i++) {
+				if (cpu_is_local(i))
+					nthreads++;
+			}
+			if (!nthreads)
+				err(EXIT_FAILURE, "No online CPUs for this node");
+		}
+	} else {
+		int cpu_available = 0;
 
-	worker = calloc(nthreads, sizeof(*worker));
+		for (i = 0; i < ncpus && !cpu_available; i++) {
+			if (cpu_is_local(i))
+				cpu_available = 1;
+		}
+		if (!cpu_available)
+			err(EXIT_FAILURE, "No online CPUs for this node");
+	}
+
+	if (numa_node >= 0) {
+		ret = numa_run_on_node(numa_node);
+		if (ret < 0)
+			err(EXIT_FAILURE, "numa_run_on_node");
+		ret = asprintf(&node_str, " on node %d", numa_node);
+		if (ret < 0)
+			err(EXIT_FAILURE, "numa_node, asprintf");
+	}
+
+	worker = numa_alloc_local(nthreads * sizeof(*worker));
 	if (!worker)
 		goto errmem;
 
 	if (!fshared)
 		futex_flag = FUTEX_PRIVATE_FLAG;
 
-	printf("Run summary [PID %d]: %d threads, each operating on %d [%s] futexes for %d secs.\n\n",
-	       getpid(), nthreads, nfutexes, fshared ? "shared":"private", nsecs);
+	printf("Run summary [PID %d]: %d threads%s, each operating on %d [%s] futexes for %d secs.\n\n",
+	       getpid(), nthreads,
+	       node_str ? : "",
+	       nfutexes, fshared ? "shared":"private",
+	       nsecs);
 
 	init_stats(&throughput_stats);
 	pthread_mutex_init(&thread_lock, NULL);

@@ -157,14 +214,24 @@ int bench_futex_hash(int argc, const char **argv,
 	threads_starting = nthreads;
 	pthread_attr_init(&thread_attr);
 	gettimeofday(&start, NULL);
-	for (i = 0; i < nthreads; i++) {
+	for (cpunum = 0, i = 0; i < nthreads; i++, cpunum++) {
+
+		do {
+			if (cpu_is_local(cpunum))
+				break;
+			cpunum++;
+			if (cpunum > ncpus)
+				cpunum = 0;
+		} while (1);
+
 		worker[i].tid = i;
-		worker[i].futex = calloc(nfutexes, sizeof(*worker[i].futex));
+		worker[i].futex = numa_alloc_local(nfutexes *
+						   sizeof(*worker[i].futex));
 		if (!worker[i].futex)
 			goto errmem;
 
 		CPU_ZERO(&cpu);
-		CPU_SET(i % ncpus, &cpu);
+		CPU_SET(cpunum % ncpus, &cpu);
 
 		ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(cpu_set_t), &cpu);
 		if (ret)

@@ -211,12 +278,12 @@ int bench_futex_hash(int argc, const char **argv,
 				       &worker[i].futex[nfutexes-1], t);
 		}
 
-		free(worker[i].futex);
+		numa_free(worker[i].futex, nfutexes * sizeof(*worker[i].futex));
 	}
 
 	print_summary();
 
-	free(worker);
+	numa_free(worker, nthreads * sizeof(*worker));
 	return ret;
 errmem:
 	err(EXIT_FAILURE, "calloc");

Signature · 1 line

2.9.3

Em Sun, Oct 16, 2016 at 09:08:03PM +0200, Sebastian Andrzej Siewior escreveu:

Quoted 13 lines

By default the application uses malloc() and all available CPUs. This patch introduces NUMA support which means:
- memory is allocated node local via numa_alloc_local()
- all CPUs of the specified NUMA node are used. This is also true if the number of threads set is greater than the number of CPUs available on this node.

Signed-off-by Sebastian Andrzej Siewior <[email protected]>

---
 tools/perf/bench/Build        |  4 ++
 tools/perf/bench/futex-hash.c | 87 ++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 81 insertions(+), 10 deletions(-)

tools/perf/bench/Build line 3

Quoted 8 lines

tools/perf/bench/Build +3 -0

index 60bf11943047..9e6e518d7d62 100644
@@ -1,3 +1,7 @@
+ifdef CONFIG_NUMA
+CFLAGS_futex-hash.o   += -DCONFIG_NUMA=1
+endif

Jiri, do we really need this? I.e. aren't the CONFIG_FOO defines available to tools?

Quoted 14 lines

Em Sun, Oct 16, 2016 at 09:08:03PM +0200, Sebastian Andrzej Siewior escreveu:

Quoted 13 lines

By default the application uses malloc() and all available CPUs. This patch introduces NUMA support which means:
- memory is allocated node local via numa_alloc_local()
- all CPUs of the specified NUMA node are used. This is also true if the number of threads set is greater than the number of CPUs available on this node.

Signed-off-by Sebastian Andrzej Siewior <[email protected]>

---
 tools/perf/bench/Build        |  4 ++
 tools/perf/bench/futex-hash.c | 87 ++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 81 insertions(+), 10 deletions(-)

Quoted 11 lines

tools/perf/bench/Build line 3

Quoted 8 lines

tools/perf/bench/Build +3 -0

index 60bf11943047..9e6e518d7d62 100644
@@ -1,3 +1,7 @@
+ifdef CONFIG_NUMA
+CFLAGS_futex-hash.o   += -DCONFIG_NUMA=1
+endif

Jiri, do we really need this? I.e. aren't the CONFIG_FOO defines available to tools?

not directly ATM.. it's prepared for the .config customary setting feature

meanwhile we set HAVE_* defines, like for CONFIG_NUMA we have:
-DHAVE_LIBNUMA_SUPPORT

you can check it in Makefile.config

By default the application uses malloc() and all available CPUs. This patch introduces NUMA support which means:
- memory is allocated node local via numa_alloc_local()
- all CPUs of the specified NUMA node are used. This is also true if the number of threads set is greater than the number of CPUs available on this node.

Quoted 6 lines · Sebastian Andrzej Siewior

By default the application uses malloc() and all available CPUs. This patch introduces NUMA support which means:
- memory is allocated node local via numa_alloc_local()
- all CPUs of the specified NUMA node are used. This is also true if the number of threads set is greater than the number of CPUs available on this node.

Can't we just use numactl to bind cpus and memory to be node-local?

Thanks,
Davidlohr

Quoted 9 lines · Davidlohr Bueso

Quoted 6 lines · Sebastian Andrzej Siewior

By default the application uses malloc() and all available CPUs. This patch introduces NUMA support which means:
- memory is allocated node local via numa_alloc_local()
- all CPUs of the specified NUMA node are used. This is also true if the number of threads set is greater than the number of CPUs available on this node.

Quoted 1 line

Can't we just use numactl to bind cpus and memory to be node-local?

something like
numactl --cpunodebind=$NODE --membind=$NODE perf ???
?

This should work for memory however since we use

	pthread_attr_setaffinity_np(&thread_attr, sizeof(cpu_set_t), &cpu);

we would need to query the affinity mask, and deploy threads based on that mask.

Using NUMA support within this bench-tool has also the side effect that the output gives all the option used.

+static int numa_node = -1;

tools/perf/bench/futex-hash.c +5 -0

+#ifndef HAVE_LIBNUMA_SUPPORT
+static int numa_run_on_node(int node __maybe_unused) { return 0; }
+static int numa_node_of_cpu(int node __maybe_unused) { return 0; }
+static void *numa_alloc_local(size_t size) { return malloc(size); }
+static void numa_free(void *p, size_t size __maybe_unused) { return free(p); }

tools/perf/bench/futex-hash.c +5 -0

+			for (i = 0; i < ncpus; i++) {
+				if (cpu_is_local(i))
+					nthreads++;
+			}
+			if (!nthreads)

It popped up in perf testing that the worker consumes some amount of CPU. It boils down to the increment of ops which causes cache line bouncing between the individual threads.

Quoted 3 lines · Sebastian Andrzej Siewior

It popped up in perf testing that the worker consumes some amount of CPU. It boils down to the increment of ops which causes cache line bouncing between the individual threads.

Are you referring to this?

      │         for (i = 0; i < nfutexes; i++, w->ops++) {
      │ be:   add    $0x1,%ebx

65.87 │ addq $0x1,0x18(%r12)

(which is like 65% of 13% on my box with a default futex-hash run).

Even better, could we get rid entirely of the ops increments and just use a local variable, then update the worker at the end of the function. The following makes 'perf' pretty much disappear in the profile.

Thanks,
Davidlohr

Sebastian noted that overhead for worker thread ops (throughput) accounting was producing 'perf' to appear in the profiles, consuming a non-trivial (ie 13%) amount of CPU. This is due to cacheline bouncing due to the increment of w->ops. We can easily fix this by just working on a local copy and updating the actual worker once done running, and ready to show the program summary. There is no danger of the worker being concurrent, so we can trust that no stale value is being seen by another thread.

Reported-by Sebastian Andrzej Siewior <[email protected]>

Quoted 9 lines · Davidlohr Bueso

Sebastian noted that overhead for worker thread ops (throughput) accounting was producing 'perf' to appear in the profiles, consuming a non-trivial (ie 13%) amount of CPU. This is due to cacheline bouncing due to the increment of w->ops. We can easily fix this by just working on a local copy and updating the actual worker once done running, and ready to show the program summary. There is no danger of the worker being concurrent, so we can trust that no stale value is being seen by another thread.

Quoted 1 line

Reported-by Sebastian Andrzej Siewior <[email protected]>

Acked-by Sebastian Andrzej Siewior <[email protected]>

tools/perf/bench/futex-hash.c static const char * const bench_futex_hash_usage[] = { · line 68

Quoted 11 lines

tools/perf/bench/futex-hash.c +2 -1

@@ -63,8 +63,9 @@ static const char * const bench_futex_hash_usage[] = {
 static void *workerfn(void *arg)
 {
 	int ret;
-	unsigned int i;
 	struct worker *w = (struct worker *) arg;
+	unsigned int i;
+	unsigned long ops = w->ops; /* avoid cacheline bouncing */

we start at 0 so there is probably no need to init it with w->ops.