perf/record: add num-synthesize-threads option
Ian Rogers
reviewerPATCH 1/1 perf/record: add num-synthesize-threads option
From: Stephane Eranian <[email protected]>
To control degree of parallelism of the synthesize_mmap() code which is scanning /proc/PID/task/PID/maps and can be time consuming. Mimic perf top way of handling the option.
If not specified will default to 1 thread, i.e. default behavior before this option.
--- tools/perf/Documentation/perf-record.txt | 4 +++ tools/perf/builtin-record.c | 35 ++++++++++++++++++++++-- tools/perf/util/record.h | 1 + 3 files changed, 38 insertions(+), 2 deletions(-)
tools/perf/Documentation/perf-record.txt +4 -0
@@ -596,6 +596,10 @@ Make a copy of /proc/kcore and place it into a directory with the perf data file Limit the sample data max size, <size> is expected to be a number with appended unit character - B/K/M/G +--num-thread-synthesize:: + The number of threads to run when synthesizing events for existing processes. + By default, the number of threads equals 1. + SEE ALSO -------- linkperf:perf-stat[1], linkperf:perf-list[1], linkperf:perf-intel-pt[1]
tools/perf/builtin-record.c +33 -2
@@ -43,6 +43,7 @@#include "util/time-utils.h" #include "util/units.h" #include "util/bpf-event.h" +#include "util/util.h" #include "asm/bug.h" #include "perf.h"
@@ -50,6 +51,7 @@#include <inttypes.h> #include <locale.h> #include <poll.h> +#include <pthread.h> #include <unistd.h> #include <sched.h> #include <signal.h>
@@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,return record__write (rec,NULL , event, event->header.size); } +static int process_locked_synthesized_event (struct perf_tool *tool, +union perf_event *event, +struct perf_sample *sample__maybe_unused , +struct machine *machine__maybe_unused ) +{ +static pthread_mutex_t synth_lock =PTHREAD_MUTEX_INITIALIZER ; +int ret; + +pthread_mutex_lock (&synth_lock); + ret =process_synthesized_event (tool, event, sample, machine); +pthread_mutex_unlock (&synth_lock); +return ret; +} +static int record__pushfn (struct mmap *map,void *to,void *bf,size_t size) {struct record *rec = to;
@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)struct perf_tool *tool = &rec->tool;int fd =perf_data__fd (data);int err =0 ; +int (*f)(struct perf_tool *,union perf_event *,struct perf_sample *, +struct machine *) = process_synthesized_event;if (rec->opts.tail_synthesize != tail)return 0 ;
@@ -1402,9 +1420,18 @@ static int record__synthesize(struct record *rec, bool tail)if (err <0 )pr_warning ("Couldn't synthesize cgroup events.\n" ); +if (rec->opts.nr_threads_synthesize >1 ) { +perf_set_multithreaded (); + f = process_locked_synthesized_event; + } + err =__machine__synthesize_threads (machine, tool, &opts->target, rec->evlist->core.threads, - process_synthesized_event, opts->sample_address, -1 ); + f, opts->sample_address, + rec->opts.nr_threads_synthesize); + +if (rec->opts.nr_threads_synthesize >1 ) +perf_set_singlethreaded (); + out:return err; }
@@ -2232,6 +2259,7 @@ static struct record record = { .default_per_cpu = true, }, .mmap_flush = MMAP_FLUSH_DEFAULT, + .nr_threads_synthesize = 1, }, .tool = { .sample = process_sample_event,
@@ -2421,6 +2449,9 @@ static struct option __record_options[] = {#endif OPT_CALLBACK (0 ,"max-size" , &record.output_max_size,"size" ,"Limit the maximum size of the output file" , parse_output_max_size), +OPT_UINTEGER (0 ,"num-thread-synthesize" , + &record.opts.nr_threads_synthesize, +"number of threads to run for event synthesis" ), OPT_END() };
tools/perf/util/record.h +1 -0
@@ -68,6 +68,7 @@ struct record_opts {int affinity;int mmap_flush;unsigned int comp_level; +unsigned int nr_threads_synthesize; };extern const char *const *record_usage;
Signature · 1 line
2.26.0.110.g2183baf09c-goog
On Wed, Apr 15, 2020 at 05:13:03PM -0700, Ian Rogers wrote:
SNIP
Quoted 25 lines
tools/perf/builtin-record.c +3
-0@@ -43,6 +43,7 @@#include "util/time-utils.h" #include "util/units.h" #include "util/bpf-event.h" +#include "util/util.h" #include "asm/bug.h" #include "perf.h" @@ -50,6 +51,7 @@#include <inttypes.h> #include <locale.h> #include <poll.h> +#include <pthread.h> #include <unistd.h> #include <sched.h> #include <signal.h> @@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,return record__write (rec,NULL , event, event->header.size); } +static int process_locked_synthesized_event(struct perf_tool *tool,
tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518
Quoted 10 lines
tools/perf/builtin-record.c +10
-0+union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER; +int ret; + +pthread_mutex_lock (&synth_lock); + ret =process_synthesized_event (tool, event, sample, machine); +pthread_mutex_unlock (&synth_lock);
hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?
tools/perf/builtin-record.c static int record__synthesize(struct record *rec, bool tail) · line 1308
Quoted 12 lines
tools/perf/builtin-record.c +5
-0+return ret; +} +static int record__pushfn (struct mmap *map,void *to,void *bf,size_t size) {struct record *rec = to;@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)struct perf_tool *tool = &rec->tool;int fd =perf_data__fd (data);int err =0 ; +int (*f)(struct perf_tool *,union perf_event *,struct perf_sample *, +struct machine *) = process_synthesized_event;
there's event_op typedef in util/tools.h
jirka
Ian Rogers
reviewerQuoted 29 lines · Jiri Olsa
On Wed, Apr 15, 2020 at 05:13:03PM -0700, Ian Rogers wrote:
SNIP
Quoted 24 lines
tools/perf/builtin-record.c +2
-0@@ -43,6 +43,7 @@#include "util/time-utils.h" #include "util/units.h" #include "util/bpf-event.h" +#include "util/util.h" #include "asm/bug.h" #include "perf.h" @@ -50,6 +51,7 @@#include <inttypes.h> #include <locale.h> #include <poll.h> +#include <pthread.h> #include <unistd.h> #include <sched.h> #include <signal.h> @@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,return record__write (rec,NULL , event, event->header.size); }
Quoted 14 lines
tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518
Quoted 11 lines
tools/perf/builtin-record.c +11
-0+static int process_locked_synthesized_event (struct perf_tool *tool, +union perf_event *event, +struct perf_sample *sample__maybe_unused , +struct machine *machine__maybe_unused ) +{ +static pthread_mutex_t synth_lock =PTHREAD_MUTEX_INITIALIZER ; +int ret; + +pthread_mutex_lock (&synth_lock); + ret =process_synthesized_event (tool, event, sample, machine); +pthread_mutex_unlock (&synth_lock);hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?
We see long synthesis times of the order seconds on loaded >100 core servers. I've not been able to create a reproduction on my desktop. You are right that making synthesis multithreaded will suffer from Amdahl's law if the write is a synchronization point. Measuring with the following patch in place:
https://lore.kernel.org/lkml/[email protected]/
without threads the portion that needs a lock is less than 1.5% of execution time and so there's plenty to still run in parallel:
...
- 32.59% __perf_event__synthesize_threads
- 32.54% __event__synthesize_thread
+ 22.13% perf_event__synthesize_mmap_events
+ 6.68% perf_event__get_comm_ids.constprop.0
+ 1.49% process_synthesized_event
+ 1.29% __GI___readdir64
+ 0.60% __opendir
...
The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/[email protected]/
shows:
Computing performance of multi threaded perf event synthesis by synthesizing events on CPU 0:
Number of synthesis threads: 1
Average synthesis took: 127729.000 usec (+- 3372.880 usec) Average num. events: 21548.600 (+- 0.306)
Average time per event 5.927 usec
Number of synthesis threads: 2
Average synthesis took: 88863.500 usec (+- 385.168 usec) Average num. events: 21552.800 (+- 0.327)
Average time per event 4.123 usec
Number of synthesis threads: 3
Average synthesis took: 83257.400 usec (+- 348.617 usec) Average num. events: 21553.200 (+- 0.327)
Average time per event 3.863 usec
Number of synthesis threads: 4
Average synthesis took: 75093.000 usec (+- 422.978 usec) Average num. events: 21554.200 (+- 0.200)
Average time per event 3.484 usec
Number of synthesis threads: 5
Average synthesis took: 64896.600 usec (+- 353.348 usec) Average num. events: 21558.000 (+- 0.000)
Average time per event 3.010 usec
Number of synthesis threads: 6
Average synthesis took: 59210.200 usec (+- 342.890 usec) Average num. events: 21560.000 (+- 0.000)
Average time per event 2.746 usec
Number of synthesis threads: 7
Average synthesis took: 54093.900 usec (+- 306.247 usec) Average num. events: 21562.000 (+- 0.000)
Average time per event 2.509 usec
Number of synthesis threads: 8
Average synthesis took: 48938.700 usec (+- 341.732 usec) Average num. events: 21564.000 (+- 0.000)
Average time per event 2.269 usec
The event logic there is using an atomic rather than a lock and the scaling isn't linear as not all the logic is threaded. Still with 8 threads we see things going about 2.6 times faster. On a large loaded machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the --num-thread-synthesize is defaulted to 1.
Quoted 6 lines
Quoted 6 lines
tools/perf/builtin-record.c +3
-0+return ret; +} +static int record__pushfn (struct mmap *map,void *to,void *bf,size_t size) {struct record *rec = to;
Quoted 8 lines
tools/perf/builtin-record.c static int record__synthesize(struct record *rec, bool tail) · line 1308
Quoted 6 lines
tools/perf/builtin-record.c +2
-0@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)struct perf_tool *tool = &rec->tool;int fd =perf_data__fd (data);int err =0 ; +int (*f)(struct perf_tool *,union perf_event *,struct perf_sample *, +struct machine *) = process_synthesized_event;there's
event_optypedef inutil/tools.h
Thanks! Updated here:
https://lore.kernel.org/lkml/[email protected]/
Ian
Quoted 2 lines
jirka
On Mon, Apr 20, 2020 at 05:31:41PM -0700, Ian Rogers wrote:
SNIP
Quoted 68 lines
Quoted 10 lines
tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518
Quoted 7 lines
tools/perf/builtin-record.c +7
-0+{ +static pthread_mutex_t synth_lock =PTHREAD_MUTEX_INITIALIZER ; +int ret; + +pthread_mutex_lock (&synth_lock); + ret =process_synthesized_event (tool, event, sample, machine); +pthread_mutex_unlock (&synth_lock);hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?
We see long synthesis times of the order seconds on loaded >100 core servers. I've not been able to create a reproduction on my desktop. You are right that making synthesis multithreaded will suffer from Amdahl's law if the write is a synchronization point. Measuring with the following patch in place:
https://lore.kernel.org/lkml/[email protected]/
without threads the portion that needs a lock is less than 1.5% of execution time and so there's plenty to still run in parallel:
...
- 32.59%__perf_event__synthesize_threads
- 32.54%__event__synthesize_thread
+ 22.13%perf_event__synthesize_mmap_events
+ 6.68% perf_event__get_comm_ids.constprop.0
+ 1.49%process_synthesized_event
+ 1.29%__GI___readdir64
+ 0.60%__opendir
...The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/[email protected]/
shows:Computing performance of multi threaded perf event synthesis by synthesizing events on CPU 0:
Number of synthesis threads: 1
Average synthesis took: 127729.000 usec (+- 3372.880 usec) Average num. events: 21548.600 (+- 0.306)
Average time per event 5.927 usec
Number of synthesis threads: 2
Average synthesis took: 88863.500 usec (+- 385.168 usec) Average num. events: 21552.800 (+- 0.327)
Average time per event 4.123 usec
Number of synthesis threads: 3
Average synthesis took: 83257.400 usec (+- 348.617 usec) Average num. events: 21553.200 (+- 0.327)
Average time per event 3.863 usec
Number of synthesis threads: 4
Average synthesis took: 75093.000 usec (+- 422.978 usec) Average num. events: 21554.200 (+- 0.200)
Average time per event 3.484 usec
Number of synthesis threads: 5
Average synthesis took: 64896.600 usec (+- 353.348 usec) Average num. events: 21558.000 (+- 0.000)
Average time per event 3.010 usec
Number of synthesis threads: 6
Average synthesis took: 59210.200 usec (+- 342.890 usec) Average num. events: 21560.000 (+- 0.000)
Average time per event 2.746 usec
Number of synthesis threads: 7
Average synthesis took: 54093.900 usec (+- 306.247 usec) Average num. events: 21562.000 (+- 0.000)
Average time per event 2.509 usec
Number of synthesis threads: 8
Average synthesis took: 48938.700 usec (+- 341.732 usec) Average num. events: 21564.000 (+- 0.000)
Average time per event 2.269 usec
Quoted 6 lines
The event logic there is using an atomic rather than a lock and the scaling isn't linear as not all the logic is threaded. Still with 8 threads we see things going about 2.6 times faster. On a large loaded machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the --num-thread-synthesize is defaulted to 1.
ok, nice ;) sorry I did not get to this before you posted v2, but could you plz send v3 with above in the changelog?
thanks,
jirka
Ian Rogers
reviewerQuoted 80 lines · Jiri Olsa
On Mon, Apr 20, 2020 at 05:31:41PM -0700, Ian Rogers wrote:
SNIP
Quoted 68 lines
Quoted 10 lines
tools/perf/builtin-record.c static int process_synthesized_event(struct perf_tool *tool, · line 518
Quoted 7 lines
tools/perf/builtin-record.c +7
-0+{ +static pthread_mutex_t synth_lock =PTHREAD_MUTEX_INITIALIZER ; +int ret; + +pthread_mutex_lock (&synth_lock); + ret =process_synthesized_event (tool, event, sample, machine); +pthread_mutex_unlock (&synth_lock);hum, so how much faster is the synthesizing with threads in record, given that we serialize it on every event that goes to the file?
We see long synthesis times of the order seconds on loaded >100 core servers. I've not been able to create a reproduction on my desktop. You are right that making synthesis multithreaded will suffer from Amdahl's law if the write is a synchronization point. Measuring with the following patch in place:
https://lore.kernel.org/lkml/[email protected]/
without threads the portion that needs a lock is less than 1.5% of execution time and so there's plenty to still run in parallel:
...
- 32.59%__perf_event__synthesize_threads
- 32.54%__event__synthesize_thread
+ 22.13%perf_event__synthesize_mmap_events
+ 6.68% perf_event__get_comm_ids.constprop.0
+ 1.49%process_synthesized_event
+ 1.29%__GI___readdir64
+ 0.60%__opendir
...The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/[email protected]/
shows:Computing performance of multi threaded perf event synthesis by synthesizing events on CPU 0:
Number of synthesis threads: 1
Average synthesis took: 127729.000 usec (+- 3372.880 usec) Average num. events: 21548.600 (+- 0.306)
Average time per event 5.927 usec
Number of synthesis threads: 2
Average synthesis took: 88863.500 usec (+- 385.168 usec) Average num. events: 21552.800 (+- 0.327)
Average time per event 4.123 usec
Number of synthesis threads: 3
Average synthesis took: 83257.400 usec (+- 348.617 usec) Average num. events: 21553.200 (+- 0.327)
Average time per event 3.863 usec
Number of synthesis threads: 4
Average synthesis took: 75093.000 usec (+- 422.978 usec) Average num. events: 21554.200 (+- 0.200)
Average time per event 3.484 usec
Number of synthesis threads: 5
Average synthesis took: 64896.600 usec (+- 353.348 usec) Average num. events: 21558.000 (+- 0.000)
Average time per event 3.010 usec
Number of synthesis threads: 6
Average synthesis took: 59210.200 usec (+- 342.890 usec) Average num. events: 21560.000 (+- 0.000)
Average time per event 2.746 usec
Number of synthesis threads: 7
Average synthesis took: 54093.900 usec (+- 306.247 usec) Average num. events: 21562.000 (+- 0.000)
Average time per event 2.509 usec
Number of synthesis threads: 8
Average synthesis took: 48938.700 usec (+- 341.732 usec) Average num. events: 21564.000 (+- 0.000)
Average time per event 2.269 usecQuoted 6 lines
The event logic there is using an atomic rather than a lock and the scaling isn't linear as not all the logic is threaded. Still with 8 threads we see things going about 2.6 times faster. On a large loaded machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the --num-thread-synthesize is defaulted to 1.
Quoted 2 lines
ok, nice ;) sorry I did not get to this before you posted v2, but could you plz send v3 with above in the changelog?
Done.
https://lore.kernel.org/lkml/[email protected]/T/#u
Not done initially as I wasn't sure the argument was coherent and also the other two patches haven't landed - it would be nice to avoid linking to the patches on the mailing list.
Thanks,
Ian
Quoted 3 lines
thanks,
jirka