perf lock: Allow concurrent record and report
Namhyung Kim
maintainerPATCH 1/1 perf lock: Allow concurrent record and report
To support live monitoring of kernel lock contention without BPF, it should support something like below:
# perf lock record -a -o- sleep 1 | perf lock contention -i- contended total wait max wait avg wait type caller
2 10.27 us 6.17 us 5.13 us spinlock load_balance+0xc03
1 5.29 us 5.29 us 5.29 us rwlock:W ep_scan_ready_list+0x54
1 4.12 us 4.12 us 4.12 us spinlock smpboot_thread_fn+0x116
1 3.28 us 3.28 us 3.28 us mutex pipe_read+0x50To do that, it needs to handle HEAD_ATTR, HEADER_EVENT_UPDATE and HEADER_TRACING_DATA which are generated only for the pipe mode. And setting event handler also should be delayed until it gets the event information.
--- tools/perf/builtin-lock.c | 68 +++++++++++++++-------- tools/perf/tests/shell/lock_contention.sh | 15 ++++- 2 files changed, 60 insertions(+), 23 deletions(-)
tools/perf/builtin-lock.c +46 -22
@@ -1390,6 +1390,34 @@ static int dump_info(void)return rc; } +static const struct evsel_str_handler lock_tracepoints[] = { + {"lock:lock_acquire" , evsel__process_lock_acquire, },/* CONFIG_LOCKDEP */ + {"lock:lock_acquired" , evsel__process_lock_acquired, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ + {"lock:lock_contended" , evsel__process_lock_contended, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ + {"lock:lock_release" , evsel__process_lock_release, },/* CONFIG_LOCKDEP */ +}; + +static const struct evsel_str_handler contention_tracepoints[] = { + {"lock:contention_begin" , evsel__process_contention_begin, }, + {"lock:contention_end" , evsel__process_contention_end, }, +}; + +static int process_event_update (struct perf_tool *tool, +union perf_event *event, +struct evlist **pevlist) +{ +int ret; + + ret =perf_event__process_event_update (tool, event, pevlist); +if (ret <0 ) +return ret; + +/* this can return -EEXIST since we call it for each evsel */ +perf_session__set_tracepoints_handlers (session, lock_tracepoints); +perf_session__set_tracepoints_handlers (session, contention_tracepoints); +return 0 ; +} +typedef int (*tracepoint_handler )(struct evsel *evsel,struct perf_sample *sample);
@@ -1545,28 +1573,19 @@ static void print_contention_result(struct lock_contention *con)print_bad_events (bad, total); } -static const struct evsel_str_handler lock_tracepoints[] = { - {"lock:lock_acquire" , evsel__process_lock_acquire, },/* CONFIG_LOCKDEP */ - {"lock:lock_acquired" , evsel__process_lock_acquired, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ - {"lock:lock_contended" , evsel__process_lock_contended, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ - {"lock:lock_release" , evsel__process_lock_release, },/* CONFIG_LOCKDEP */ -}; - -static const struct evsel_str_handler contention_tracepoints[] = { - {"lock:contention_begin" , evsel__process_contention_begin, }, - {"lock:contention_end" , evsel__process_contention_end, }, -}; -static bool force;static int __cmd_report (bool display_info) {int err = -EINVAL ;struct perf_tool eops = { + .attr = perf_event__process_attr, + .event_update = process_event_update, .sample = process_sample_event, .comm = perf_event__process_comm, .mmap = perf_event__process_mmap, .namespaces = perf_event__process_namespaces, + .tracing_data = perf_event__process_tracing_data, .ordered_events = true, };struct perf_data data = {
@@ -1585,17 +1604,19 @@ static int __cmd_report(bool display_info) symbol_conf.sort_by_name = true;symbol__init (&session->header.env); -if (!perf_session__has_traces (session,"lock record" )) - goto out_delete; +if (!data.is_pipe) { +if (!perf_session__has_traces (session,"lock record" )) + goto out_delete; -if (perf_session__set_tracepoints_handlers (session, lock_tracepoints)) { -pr_err ("Initializing perf session tracepoint handlers failed\n" ); - goto out_delete; - } +if (perf_session__set_tracepoints_handlers (session, lock_tracepoints)) { +pr_err ("Initializing perf session tracepoint handlers failed\n" ); + goto out_delete; + } -if (perf_session__set_tracepoints_handlers (session, contention_tracepoints)) { -pr_err ("Initializing perf session tracepoint handlers failed\n" ); - goto out_delete; +if (perf_session__set_tracepoints_handlers (session, contention_tracepoints)) { +pr_err ("Initializing perf session tracepoint handlers failed\n" ); + goto out_delete; + } }if (setup_output_field (false, output_fields))
@@ -1633,9 +1654,12 @@ static int __cmd_contention(int argc, const char **argv) {int err = -EINVAL ;struct perf_tool eops = { + .attr = perf_event__process_attr, + .event_update = process_event_update, .sample = process_sample_event, .comm = perf_event__process_comm, .mmap = perf_event__process_mmap, + .tracing_data = perf_event__process_tracing_data, .ordered_events = true, };struct perf_data data = {
@@ -1698,7 +1722,7 @@ static int __cmd_contention(int argc, const char **argv)pr_err ("lock contention BPF setup failed\n" ); goto out_delete; } - }else { + }else if (!data.is_pipe) {if (!perf_session__has_traces (session,"lock record" )) goto out_delete;
tools/perf/tests/shell/lock_contention.sh +14 -1
@@ -53,7 +53,7 @@ test_bpf() if ! perf lock con -b true > /dev/null 2>&1 ; then echo "[Skip] No BPF support" - exit + return fi # the perf lock contention output goes to the stderr
@@ -65,9 +65,22 @@ test_bpf() fi } +test_record_concurrent() +{ + echo "Testing perf lock record and perf lock contention at the same time" + perf lock record -o- -- perf bench sched messaging 2> /dev/null | \ + perf lock contention -i- -E 1 -q 2> ${result} + if [ $(cat "${result}" | wc -l) != "1" ]; then + echo "[Fail] Recorded result count is not 1:" $(cat "${result}" | wc -l) + err=1 + exit + fi +} + check test_record test_bpf +test_record_concurrent exit ${err}
Signature · 1 line
2.38.1.431.g37b22c650d-goog
Quoted 12 lines · Namhyung Kim
To support live monitoring of kernel lock contention without BPF, it should support something like below:
# perf lock record -a -o- sleep 1 | perf lock contention -i- contended total wait max wait avg wait type caller2 10.27 us 6.17 us 5.13 us spinlock load_balance+0xc03 1 5.29 us 5.29 us 5.29 us rwlock:W ep_scan_ready_list+0x54 1 4.12 us 4.12 us 4.12 us spinlock smpboot_thread_fn+0x116 1 3.28 us 3.28 us 3.28 us mutex pipe_read+0x50
Quoted 6 lines
To do that, it needs to handle
HEAD_ATTR,HEADER_EVENT_UPDATEandHEADER_TRACING_DATAwhich are generated only for the pipe mode. And setting event handler also should be delayed until it gets the event information.Signed-off-by Namhyung Kim <[email protected]>
Thanks,
Ian
Quoted 166 lines
--- tools/perf/builtin-lock.c | 68 +++++++++++++++-------- tools/perf/tests/shell/lock_contention.sh | 15 ++++- 2 files changed, 60 insertions(+), 23 deletions(-)tools/perf/builtin-lock.c +46
-22@@ -1390,6 +1390,34 @@ static int dump_info(void)return rc; } +static const struct evsel_str_handler lock_tracepoints[] = { + {"lock:lock_acquire" , evsel__process_lock_acquire, },/* CONFIG_LOCKDEP */ + {"lock:lock_acquired" , evsel__process_lock_acquired, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ + {"lock:lock_contended" , evsel__process_lock_contended, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ + {"lock:lock_release" , evsel__process_lock_release, },/* CONFIG_LOCKDEP */ +}; + +static const struct evsel_str_handler contention_tracepoints[] = { + {"lock:contention_begin" , evsel__process_contention_begin, }, + {"lock:contention_end" , evsel__process_contention_end, }, +}; + +static int process_event_update (struct perf_tool *tool, +union perf_event *event, +struct evlist **pevlist) +{ +int ret; + + ret =perf_event__process_event_update (tool, event, pevlist); +if (ret <0 ) +return ret; + +/* this can return -EEXIST since we call it for each evsel */ +perf_session__set_tracepoints_handlers (session, lock_tracepoints); +perf_session__set_tracepoints_handlers (session, contention_tracepoints); +return 0 ; +} +typedef int (*tracepoint_handler )(struct evsel *evsel,struct perf_sample *sample);@@ -1545,28 +1573,19 @@ static void print_contention_result(struct lock_contention *con)print_bad_events (bad, total); } -static const struct evsel_str_handler lock_tracepoints[] = { - {"lock:lock_acquire" , evsel__process_lock_acquire, },/* CONFIG_LOCKDEP */ - {"lock:lock_acquired" , evsel__process_lock_acquired, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ - {"lock:lock_contended" , evsel__process_lock_contended, },/* CONFIG_LOCKDEP, CONFIG_LOCK_STAT */ - {"lock:lock_release" , evsel__process_lock_release, },/* CONFIG_LOCKDEP */ -}; - -static const struct evsel_str_handler contention_tracepoints[] = { - {"lock:contention_begin" , evsel__process_contention_begin, }, - {"lock:contention_end" , evsel__process_contention_end, }, -}; -static bool force;static int __cmd_report (bool display_info) {int err = -EINVAL ;struct perf_tool eops = { + .attr = perf_event__process_attr, + .event_update = process_event_update, .sample = process_sample_event, .comm = perf_event__process_comm, .mmap = perf_event__process_mmap, .namespaces = perf_event__process_namespaces, + .tracing_data = perf_event__process_tracing_data, .ordered_events = true, };struct perf_data data = {@@ -1585,17 +1604,19 @@ static int __cmd_report(bool display_info) symbol_conf.sort_by_name = true;symbol__init (&session->header.env); -if (!perf_session__has_traces (session,"lock record" )) - goto out_delete; +if (!data.is_pipe) { +if (!perf_session__has_traces (session,"lock record" )) + goto out_delete; -if (perf_session__set_tracepoints_handlers (session, lock_tracepoints)) { -pr_err ("Initializing perf session tracepoint handlers failed\n" ); - goto out_delete; - } +if (perf_session__set_tracepoints_handlers (session, lock_tracepoints)) { +pr_err ("Initializing perf session tracepoint handlers failed\n" ); + goto out_delete; + } -if (perf_session__set_tracepoints_handlers (session, contention_tracepoints)) { -pr_err ("Initializing perf session tracepoint handlers failed\n" ); - goto out_delete; +if (perf_session__set_tracepoints_handlers (session, contention_tracepoints)) { +pr_err ("Initializing perf session tracepoint handlers failed\n" ); + goto out_delete; + } }if (setup_output_field (false, output_fields))@@ -1633,9 +1654,12 @@ static int __cmd_contention(int argc, const char **argv) {int err = -EINVAL ;struct perf_tool eops = { + .attr = perf_event__process_attr, + .event_update = process_event_update, .sample = process_sample_event, .comm = perf_event__process_comm, .mmap = perf_event__process_mmap, + .tracing_data = perf_event__process_tracing_data, .ordered_events = true, };struct perf_data data = {@@ -1698,7 +1722,7 @@ static int __cmd_contention(int argc, const char **argv)pr_err ("lock contention BPF setup failed\n" ); goto out_delete; } - }else { + }else if (!data.is_pipe) {if (!perf_session__has_traces (session,"lock record" )) goto out_delete;tools/perf/tests/shell/lock_contention.sh +14
-2@@ -53,7 +53,7 @@ test_bpf() if ! perf lock con -b true > /dev/null 2>&1 ; then echo "[Skip] No BPF support" - exit + return fi # the perf lock contention output goes to the stderr@@ -65,9 +65,22 @@ test_bpf() fi } +test_record_concurrent() +{ + echo "Testing perf lock record and perf lock contention at the same time" + perf lock record -o- -- perf bench sched messaging 2> /dev/null | \ + perf lock contention -i- -E 1 -q 2> ${result} + if [ $(cat "${result}" | wc -l) != "1" ]; then + echo "[Fail] Recorded result count is not 1:" $(cat "${result}" | wc -l) + err=1 + exit + fi +} + check test_record test_bpf +test_record_concurrent exit ${err} --2.38.1.431.g37b22c650d-goog
Em Tue, Nov 08, 2022 at 03:26:01PM -0800, Ian Rogers escreveu:
Quoted 13 lines · Namhyung Kim
Quoted 12 lines
To support live monitoring of kernel lock contention without BPF, it should support something like below:
# perf lock record -a -o- sleep 1 | perf lock contention -i- contended total wait max wait avg wait type caller2 10.27 us 6.17 us 5.13 us spinlock load_balance+0xc03 1 5.29 us 5.29 us 5.29 us rwlock:W ep_scan_ready_list+0x54 1 4.12 us 4.12 us 4.12 us spinlock smpboot_thread_fn+0x116 1 3.28 us 3.28 us 3.28 us mutex pipe_read+0x50
Quoted 8 lines
Quoted 6 lines
To do that, it needs to handle
HEAD_ATTR,HEADER_EVENT_UPDATEandHEADER_TRACING_DATAwhich are generated only for the pipe mode. And setting event handler also should be delayed until it gets the event information.Signed-off-by Namhyung Kim <[email protected]>Acked-by Ian Rogers <[email protected]>
Thanks, applied.
- Arnaldo