Skip to contentSkip to conversation

perf/record: add num-synthesize-threads option

PATCH Superseded Ian Rogers 2 participants 5 messages Apr 16, 2020 – Apr 22, 2020 perf events

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Ian Rogers

author reviewer
#0 lore raw

PATCH 1/1 perf/record: add num-synthesize-threads option

From: Stephane Eranian <[email protected]>

To control degree of parallelism of the synthesize_mmap() code which is scanning /proc/PID/task/PID/maps and can be time consuming. Mimic perf top way of handling the option.
If not specified will default to 1 thread, i.e. default behavior before this option.

---
 tools/perf/Documentation/perf-record.txt |  4 +++
 tools/perf/builtin-record.c              | 35 ++++++++++++++++++++++--
 tools/perf/util/record.h                 |  1 +
 3 files changed, 38 insertions(+), 2 deletions(-)
tools/perf/Documentation/perf-record.txt +4 -0
index b3f3b3f1c161..6e8b4649307c 100644
@@ -596,6 +596,10 @@ Make a copy of /proc/kcore and place it into a directory with the perf data file
 Limit the sample data max size, <size> is expected to be a number with
 appended unit character - B/K/M/G
 
+--num-thread-synthesize::
+	The number of threads to run when synthesizing events for existing processes.
+	By default, the number of threads equals 1.
+
 SEE ALSO
 --------
 linkperf:perf-stat[1], linkperf:perf-list[1], linkperf:perf-intel-pt[1]
tools/perf/builtin-record.c +33 -2
index 1ab349abe904..2f97d0c32a75 100644
@@ -43,6 +43,7 @@
 #include "util/time-utils.h"
 #include "util/units.h"
 #include "util/bpf-event.h"
+#include "util/util.h"
 #include "asm/bug.h"
 #include "perf.h"
 
@@ -50,6 +51,7 @@
 #include <inttypes.h>
 #include <locale.h>
 #include <poll.h>
+#include <pthread.h>
 #include <unistd.h>
 #include <sched.h>
 #include <signal.h>
@@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,
 	return record__write(rec, NULL, event, event->header.size);
 }
 
+static int process_locked_synthesized_event(struct perf_tool *tool,
+				     union perf_event *event,
+				     struct perf_sample *sample __maybe_unused,
+				     struct machine *machine __maybe_unused)
+{
+	static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
+	int ret;
+
+	pthread_mutex_lock(&synth_lock);
+	ret = process_synthesized_event(tool, event, sample, machine);
+	pthread_mutex_unlock(&synth_lock);
+	return ret;
+}
+
 static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
 {
 	struct record *rec = to;
@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)
 	struct perf_tool *tool = &rec->tool;
 	int fd = perf_data__fd(data);
 	int err = 0;
+	int (*f)(struct perf_tool *, union perf_event *, struct perf_sample *,
+		struct machine *) = process_synthesized_event;
 
 	if (rec->opts.tail_synthesize != tail)
 		return 0;
@@ -1402,9 +1420,18 @@ static int record__synthesize(struct record *rec, bool tail)
 	if (err < 0)
 		pr_warning("Couldn't synthesize cgroup events.\n");
 
+	if (rec->opts.nr_threads_synthesize > 1) {
+		perf_set_multithreaded();
+		f = process_locked_synthesized_event;
+	}
+
 	err = __machine__synthesize_threads(machine, tool, &opts->target, rec->evlist->core.threads,
-					    process_synthesized_event, opts->sample_address,
-					    1);
+					    f, opts->sample_address,
+					    rec->opts.nr_threads_synthesize);
+
+	if (rec->opts.nr_threads_synthesize > 1)
+		perf_set_singlethreaded();
+
 out:
 	return err;
 }
@@ -2232,6 +2259,7 @@ static struct record record = {
 			.default_per_cpu = true,
 		},
 		.mmap_flush          = MMAP_FLUSH_DEFAULT,
+		.nr_threads_synthesize = 1,
 	},
 	.tool = {
 		.sample		= process_sample_event,
@@ -2421,6 +2449,9 @@ static struct option __record_options[] = {
 #endif
 	OPT_CALLBACK(0, "max-size", &record.output_max_size,
 		     "size", "Limit the maximum size of the output file", parse_output_max_size),
+	OPT_UINTEGER(0, "num-thread-synthesize",
+		     &record.opts.nr_threads_synthesize,
+		     "number of threads to run for event synthesis"),
 	OPT_END()
 };
 
tools/perf/util/record.h +1 -0
index 24316458be20..923565c3b155 100644
@@ -68,6 +68,7 @@ struct record_opts {
 	int	      affinity;
 	int	      mmap_flush;
 	unsigned int  comp_level;
+	unsigned int  nr_threads_synthesize;
 };
 
 extern const char * const *record_usage;
Signature · 1 line
2.26.0.110.g2183baf09c-goog
#1 lore raw

On Wed, Apr 15, 2020 at 05:13:03PM -0700, Ian Rogers wrote:

SNIP

Quoted 25 lines
tools/perf/builtin-record.c +3 -0
index 1ab349abe904..2f97d0c32a75 100644
@@ -43,6 +43,7 @@
 #include "util/time-utils.h"
 #include "util/units.h"
 #include "util/bpf-event.h"
+#include "util/util.h"
 #include "asm/bug.h"
 #include "perf.h"
 
@@ -50,6 +51,7 @@
 #include <inttypes.h>
 #include <locale.h>
 #include <poll.h>
+#include <pthread.h>
 #include <unistd.h>
 #include <sched.h>
 #include <signal.h>
@@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,
 	return record__write(rec, NULL, event, event->header.size);
 }
 
+static int process_locked_synthesized_event(struct perf_tool *tool,

tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518

Quoted 10 lines
tools/perf/builtin-record.c +10 -0
+				     union perf_event *event,
+				     struct perf_sample *sample __maybe_unused,
+				     struct machine *machine __maybe_unused)
+{
+	static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
+	int ret;
+
+	pthread_mutex_lock(&synth_lock);
+	ret = process_synthesized_event(tool, event, sample, machine);
+	pthread_mutex_unlock(&synth_lock);

hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?

tools/perf/builtin-record.c static int record__synthesize(struct record *rec, bool tail) · line 1308

Quoted 12 lines
tools/perf/builtin-record.c +5 -0
+	return ret;
+}
+
 static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
 {
 	struct record *rec = to;
@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)
 	struct perf_tool *tool = &rec->tool;
 	int fd = perf_data__fd(data);
 	int err = 0;
+	int (*f)(struct perf_tool *, union perf_event *, struct perf_sample *,
+		struct machine *) = process_synthesized_event;

there's event_op typedef in util/tools.h

jirka

author reviewer
#2 lore raw
Quoted 29 lines · Jiri Olsa

On Wed, Apr 15, 2020 at 05:13:03PM -0700, Ian Rogers wrote:

SNIP

Quoted 24 lines
tools/perf/builtin-record.c +2 -0
index 1ab349abe904..2f97d0c32a75 100644
@@ -43,6 +43,7 @@
 #include "util/time-utils.h"
 #include "util/units.h"
 #include "util/bpf-event.h"
+#include "util/util.h"
 #include "asm/bug.h"
 #include "perf.h"
 
@@ -50,6 +51,7 @@
 #include <inttypes.h>
 #include <locale.h>
 #include <poll.h>
+#include <pthread.h>
 #include <unistd.h>
 #include <sched.h>
 #include <signal.h>
@@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,
      return record__write(rec, NULL, event, event->header.size);
 }
 
Quoted 14 lines

tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518

Quoted 11 lines
tools/perf/builtin-record.c +11 -0
+static int process_locked_synthesized_event(struct perf_tool *tool,
+                                  union perf_event *event,
+                                  struct perf_sample *sample __maybe_unused,
+                                  struct machine *machine __maybe_unused)
+{
+     static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
+     int ret;
+
+     pthread_mutex_lock(&synth_lock);
+     ret = process_synthesized_event(tool, event, sample, machine);
+     pthread_mutex_unlock(&synth_lock);

hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?

We see long synthesis times of the order seconds on loaded >100 core servers. I've not been able to create a reproduction on my desktop. You are right that making synthesis multithreaded will suffer from Amdahl's law if the write is a synchronization point. Measuring with the following patch in place:
https://lore.kernel.org/lkml/[email protected]/
without threads the portion that needs a lock is less than 1.5% of execution time and so there's plenty to still run in parallel:
...
- 32.59% __perf_event__synthesize_threads
- 32.54% __event__synthesize_thread
+ 22.13% perf_event__synthesize_mmap_events
+ 6.68% perf_event__get_comm_ids.constprop.0
+ 1.49% process_synthesized_event
+ 1.29% __GI___readdir64
+ 0.60% __opendir
...

The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/[email protected]/
shows:

Computing performance of multi threaded perf event synthesis by synthesizing events on CPU 0:
Number of synthesis threads: 1
Average synthesis took: 127729.000 usec (+- 3372.880 usec) Average num. events: 21548.600 (+- 0.306)
Average time per event 5.927 usec
Number of synthesis threads: 2
Average synthesis took: 88863.500 usec (+- 385.168 usec) Average num. events: 21552.800 (+- 0.327)
Average time per event 4.123 usec
Number of synthesis threads: 3
Average synthesis took: 83257.400 usec (+- 348.617 usec) Average num. events: 21553.200 (+- 0.327)
Average time per event 3.863 usec
Number of synthesis threads: 4
Average synthesis took: 75093.000 usec (+- 422.978 usec) Average num. events: 21554.200 (+- 0.200)
Average time per event 3.484 usec
Number of synthesis threads: 5
Average synthesis took: 64896.600 usec (+- 353.348 usec) Average num. events: 21558.000 (+- 0.000)
Average time per event 3.010 usec
Number of synthesis threads: 6
Average synthesis took: 59210.200 usec (+- 342.890 usec) Average num. events: 21560.000 (+- 0.000)
Average time per event 2.746 usec
Number of synthesis threads: 7
Average synthesis took: 54093.900 usec (+- 306.247 usec) Average num. events: 21562.000 (+- 0.000)
Average time per event 2.509 usec
Number of synthesis threads: 8
Average synthesis took: 48938.700 usec (+- 341.732 usec) Average num. events: 21564.000 (+- 0.000)
Average time per event 2.269 usec

The event logic there is using an atomic rather than a lock and the scaling isn't linear as not all the logic is threaded. Still with 8 threads we see things going about 2.6 times faster. On a large loaded machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the --num-thread-synthesize is defaulted to 1.

Quoted 6 lines
Quoted 6 lines
tools/perf/builtin-record.c +3 -0
+     return ret;
+}
+
 static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
 {
      struct record *rec = to;
Quoted 8 lines

tools/perf/builtin-record.c static int record__synthesize(struct record *rec, bool tail) · line 1308

Quoted 6 lines
tools/perf/builtin-record.c +2 -0
@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)
      struct perf_tool *tool = &rec->tool;
      int fd = perf_data__fd(data);
      int err = 0;
+     int (*f)(struct perf_tool *, union perf_event *, struct perf_sample *,
+             struct machine *) = process_synthesized_event;

there's event_op typedef in util/tools.h

Thanks! Updated here:
https://lore.kernel.org/lkml/[email protected]/

Ian

Quoted 2 lines

jirka

#3 lore raw

On Mon, Apr 20, 2020 at 05:31:41PM -0700, Ian Rogers wrote:

SNIP

Quoted 68 lines
Quoted 10 lines

tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518

Quoted 7 lines
tools/perf/builtin-record.c +7 -0
+{
+     static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
+     int ret;
+
+     pthread_mutex_lock(&synth_lock);
+     ret = process_synthesized_event(tool, event, sample, machine);
+     pthread_mutex_unlock(&synth_lock);

hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?

We see long synthesis times of the order seconds on loaded >100 core servers. I've not been able to create a reproduction on my desktop. You are right that making synthesis multithreaded will suffer from Amdahl's law if the write is a synchronization point. Measuring with the following patch in place:
https://lore.kernel.org/lkml/[email protected]/
without threads the portion that needs a lock is less than 1.5% of execution time and so there's plenty to still run in parallel:
...
- 32.59% __perf_event__synthesize_threads
- 32.54% __event__synthesize_thread
+ 22.13% perf_event__synthesize_mmap_events
+ 6.68% perf_event__get_comm_ids.constprop.0
+ 1.49% process_synthesized_event
+ 1.29% __GI___readdir64
+ 0.60% __opendir
...

The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/[email protected]/
shows:

Computing performance of multi threaded perf event synthesis by synthesizing events on CPU 0:
Number of synthesis threads: 1
Average synthesis took: 127729.000 usec (+- 3372.880 usec) Average num. events: 21548.600 (+- 0.306)
Average time per event 5.927 usec
Number of synthesis threads: 2
Average synthesis took: 88863.500 usec (+- 385.168 usec) Average num. events: 21552.800 (+- 0.327)
Average time per event 4.123 usec
Number of synthesis threads: 3
Average synthesis took: 83257.400 usec (+- 348.617 usec) Average num. events: 21553.200 (+- 0.327)
Average time per event 3.863 usec
Number of synthesis threads: 4
Average synthesis took: 75093.000 usec (+- 422.978 usec) Average num. events: 21554.200 (+- 0.200)
Average time per event 3.484 usec
Number of synthesis threads: 5
Average synthesis took: 64896.600 usec (+- 353.348 usec) Average num. events: 21558.000 (+- 0.000)
Average time per event 3.010 usec
Number of synthesis threads: 6
Average synthesis took: 59210.200 usec (+- 342.890 usec) Average num. events: 21560.000 (+- 0.000)
Average time per event 2.746 usec
Number of synthesis threads: 7
Average synthesis took: 54093.900 usec (+- 306.247 usec) Average num. events: 21562.000 (+- 0.000)
Average time per event 2.509 usec
Number of synthesis threads: 8
Average synthesis took: 48938.700 usec (+- 341.732 usec) Average num. events: 21564.000 (+- 0.000)
Average time per event 2.269 usec

Quoted 6 lines

The event logic there is using an atomic rather than a lock and the scaling isn't linear as not all the logic is threaded. Still with 8 threads we see things going about 2.6 times faster. On a large loaded machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the --num-thread-synthesize is defaulted to 1.

ok, nice ;) sorry I did not get to this before you posted v2, but could you plz send v3 with above in the changelog?

thanks,
jirka

author reviewer
#4 lore raw
Quoted 80 lines · Jiri Olsa

On Mon, Apr 20, 2020 at 05:31:41PM -0700, Ian Rogers wrote:

SNIP

Quoted 68 lines
Quoted 10 lines

tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518

Quoted 7 lines
tools/perf/builtin-record.c +7 -0
+{
+     static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
+     int ret;
+
+     pthread_mutex_lock(&synth_lock);
+     ret = process_synthesized_event(tool, event, sample, machine);
+     pthread_mutex_unlock(&synth_lock);

hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?

We see long synthesis times of the order seconds on loaded >100 core servers. I've not been able to create a reproduction on my desktop. You are right that making synthesis multithreaded will suffer from Amdahl's law if the write is a synchronization point. Measuring with the following patch in place:
https://lore.kernel.org/lkml/[email protected]/
without threads the portion that needs a lock is less than 1.5% of execution time and so there's plenty to still run in parallel:
...
- 32.59% __perf_event__synthesize_threads
- 32.54% __event__synthesize_thread
+ 22.13% perf_event__synthesize_mmap_events
+ 6.68% perf_event__get_comm_ids.constprop.0
+ 1.49% process_synthesized_event
+ 1.29% __GI___readdir64
+ 0.60% __opendir
...

The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/[email protected]/
shows:

Computing performance of multi threaded perf event synthesis by synthesizing events on CPU 0:
Number of synthesis threads: 1
Average synthesis took: 127729.000 usec (+- 3372.880 usec) Average num. events: 21548.600 (+- 0.306)
Average time per event 5.927 usec
Number of synthesis threads: 2
Average synthesis took: 88863.500 usec (+- 385.168 usec) Average num. events: 21552.800 (+- 0.327)
Average time per event 4.123 usec
Number of synthesis threads: 3
Average synthesis took: 83257.400 usec (+- 348.617 usec) Average num. events: 21553.200 (+- 0.327)
Average time per event 3.863 usec
Number of synthesis threads: 4
Average synthesis took: 75093.000 usec (+- 422.978 usec) Average num. events: 21554.200 (+- 0.200)
Average time per event 3.484 usec
Number of synthesis threads: 5
Average synthesis took: 64896.600 usec (+- 353.348 usec) Average num. events: 21558.000 (+- 0.000)
Average time per event 3.010 usec
Number of synthesis threads: 6
Average synthesis took: 59210.200 usec (+- 342.890 usec) Average num. events: 21560.000 (+- 0.000)
Average time per event 2.746 usec
Number of synthesis threads: 7
Average synthesis took: 54093.900 usec (+- 306.247 usec) Average num. events: 21562.000 (+- 0.000)
Average time per event 2.509 usec
Number of synthesis threads: 8
Average synthesis took: 48938.700 usec (+- 341.732 usec) Average num. events: 21564.000 (+- 0.000)
Average time per event 2.269 usec

Quoted 6 lines

The event logic there is using an atomic rather than a lock and the scaling isn't linear as not all the logic is threaded. Still with 8 threads we see things going about 2.6 times faster. On a large loaded machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the --num-thread-synthesize is defaulted to 1.

Quoted 2 lines

ok, nice ;) sorry I did not get to this before you posted v2, but could you plz send v3 with above in the changelog?

Done.
https://lore.kernel.org/lkml/[email protected]/T/#u
Not done initially as I wasn't sure the argument was coherent and also the other two patches haven't landed - it would be nice to avoid linking to the patches on the mailing list.

Thanks,
Ian

Quoted 3 lines

thanks,
jirka