perf lock: Add contention subcommand (v2)
Namhyung Kim
maintainercover perf lock: Add contention subcommand (v2)
Hello,
It's to add a new subcommand 'contention' (shortly 'con') to perf lock.
Changes in v2)
* bugfix is merged already
* fix build error in patch 2
The new subcommand is to handle the new lock:contention_{begin,end} tracepoints and shows lock type and caller address like below:
$ perf lock contention contended total wait max wait avg wait type caller
238 1.41 ms 29.20 us 5.94 us spinlock update_blocked_averages+0x4c
1 902.08 us 902.08 us 902.08 us rwsem:R do_user_addr_fault+0x1dd
81 330.30 us 17.24 us 4.08 us spinlock _nohz_idle_balance+0x172
2 89.54 us 61.26 us 44.77 us spinlock do_anonymous_page+0x16d
24 78.36 us 12.27 us 3.27 us mutex pipe_read+0x56
2 71.58 us 59.56 us 35.79 us spinlock __handle_mm_fault+0x6aa
6 25.68 us 6.89 us 4.28 us spinlock do_idle+0x28d
1 18.46 us 18.46 us 18.46 us rtmutex exec_fw_cmd+0x21b
3 15.25 us 6.26 us 5.08 us spinlock tick_do_update_jiffies64+0x2c...
where rwsem:R stands for read access (down_read) for a rw-semaphore. Other types of lock access is obvious and it doesn't detect optimistic spinning on mutex yet. This is just a base work for lock contention analysis and more will come later.
You can get this from 'perf/lock-subcmd-v2' branch on
git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.git
Thanks,
Namhyung
Namhyung Kim (5):
perf lock: Add flags field in the lock_stat
perf lock: Add lock aggregation enum
perf lock: Add 'contention' subcommand
perf lock: Add -k and -F options to 'contention' subcommand perf lock: Support -t option for 'contention' subcommand
tools/perf/Documentation/perf-lock.txt | 23 +- tools/perf/builtin-lock.c | 401 +++++++++++++++++++++++-- 2 files changed, 390 insertions(+), 34 deletions(-)
base-commit: 9fe9b252c7c022df8e503435e778f15c04dfa3bf
Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Namhyung Kim
maintainerPATCH 1/5 perf lock: Add flags field in the lock_stat
For lock contention tracepoint analysis, it needs to keep the flags. As nr_readlock and nr_trylock fields are not used for it, let's make it a union.
--- tools/perf/builtin-lock.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-)
tools/perf/builtin-lock.c +13 -9
@@ -59,7 +59,10 @@ struct lock_stat {unsigned int nr_contended;unsigned int nr_release; -unsigned int nr_readlock; +union { +unsigned int nr_readlock; +unsigned int flags; + };unsigned int nr_trylock;/* these times are in nano sec. */
@@ -516,7 +519,7 @@ static struct lock_stat *lock_stat_find(u64 addr)return NULL ; } -static struct lock_stat *lock_stat_findnew (u64 addr,const char *name) +static struct lock_stat *lock_stat_findnew (u64 addr,const char *name,int flags) {struct hlist_head *entry =lockhashentry (addr);struct lock_stat *ret, *new;
@@ -531,13 +534,13 @@ static struct lock_stat *lock_stat_findnew(u64 addr, const char *name) goto alloc_failed; new->addr = addr; - new->name =zalloc (sizeof (char ) *strlen (name) +1 ); + new->name =strdup (name);if (!new->name) {free (new); goto alloc_failed; } -strcpy (new->name, name); + new->flags = flags; new->wait_time_min =ULLONG_MAX ;hlist_add_head (&new->hash_entry, entry);
@@ -624,7 +627,7 @@ static int report_lock_acquire_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;
@@ -696,7 +699,7 @@ static int report_lock_acquired_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;
@@ -758,7 +761,7 @@ static int report_lock_contended_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;
@@ -813,7 +816,7 @@ static int report_lock_release_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;
@@ -985,11 +988,12 @@ static int report_lock_contention_begin_event(struct evsel *evsel,if (!ls) {char buf[128 ];const char *caller = buf; +unsigned int flags =evsel__intval (evsel, sample,"flags" );if (lock_contention_caller (evsel, sample, buf,sizeof (buf)) <0 ) caller ="Unknown" ; - ls =lock_stat_findnew (addr, caller); + ls =lock_stat_findnew (addr, caller, flags);if (!ls)return -ENOMEM ; }
Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Arnaldo Carvalho de Melo
maintainerRe: [PATCH 1/5] perf lock: Add flags field in the lock_stat
Em Mon, Jul 25, 2022 at 11:31:20AM -0700, Namhyung Kim escreveu:
Quoted 36 lines
For lock contention tracepoint analysis, it needs to keep the flags. As
nr_readlockandnr_trylockfields are not used for it, let's make it a union.Signed-off-by Namhyung Kim <[email protected]>--- tools/perf/builtin-lock.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-)tools/perf/builtin-lock.c +5
-2@@ -59,7 +59,10 @@ struct lock_stat {unsigned int nr_contended;unsigned int nr_release; -unsigned int nr_readlock; +union { +unsigned int nr_readlock; +unsigned int flags; + };unsigned int nr_trylock;/* these times are in nano sec. */ @@ -516,7 +519,7 @@ static struct lock_stat *lock_stat_find(u64 addr)return NULL ; } -static struct lock_stat *lock_stat_findnew (u64 addr,const char *name) +static struct lock_stat *lock_stat_findnew (u64 addr,const char *name,int flags) {struct hlist_head *entry =lockhashentry (addr);struct lock_stat *ret, *new;@@ -531,13 +534,13 @@ static struct lock_stat *lock_stat_findnew(u64 addr, const char *name) goto alloc_failed;
tools/perf/builtin-lock.c static struct lock_stat *lock_stat_findnew(u64 addr, const char *name) · line 540
Quoted 10 lines
tools/perf/builtin-lock.c +1
-2new->addr = addr; - new->name =zalloc (sizeof (char ) *strlen (name) +1 ); + new->name =strdup (name);if (!new->name) {free (new); goto alloc_failed; } -strcpy (new->name, name);
I'm applying to speed things up, but please separate such unrelated bits in another patch next time.
- Arnaldo
Quoted 56 lines
tools/perf/builtin-lock.c +7
-5+ new->flags = flags; new->wait_time_min =ULLONG_MAX ;hlist_add_head (&new->hash_entry, entry);@@ -624,7 +627,7 @@ static int report_lock_acquire_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;@@ -696,7 +699,7 @@ static int report_lock_acquired_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;@@ -758,7 +761,7 @@ static int report_lock_contended_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;@@ -813,7 +816,7 @@ static int report_lock_release_event(struct evsel *evsel,if (show_thread_stats) addr = sample->tid; - ls =lock_stat_findnew (addr, name); + ls =lock_stat_findnew (addr, name,0 );if (!ls)return -ENOMEM ;@@ -985,11 +988,12 @@ static int report_lock_contention_begin_event(struct evsel *evsel,if (!ls) {char buf[128 ];const char *caller = buf; +unsigned int flags =evsel__intval (evsel, sample,"flags" );if (lock_contention_caller (evsel, sample, buf,sizeof (buf)) <0 ) caller ="Unknown" ; - ls =lock_stat_findnew (addr, caller); + ls =lock_stat_findnew (addr, caller, flags);if (!ls)return -ENOMEM ; }Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Signature · 1 line
- Arnaldo
Namhyung Kim
maintainerPATCH 2/5 perf lock: Add lock aggregation enum
In reply to Namhyung KimIntroduce the aggr_mode variable to prepare the later code change. The default is LOCK_AGGR_ADDR which aggregate the result for the lock instances. When -t/--threads option is given, it'd be set to LOCK_AGGR_TASK. The LOCK_AGGR_CALLER is for the contention analysis and it'd aggregate the stat by comparing the callstacks.
--- tools/perf/builtin-lock.c | 114 +++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 20 deletions(-)
tools/perf/builtin-lock.c +94 -20
@@ -126,6 +126,12 @@ static struct rb_root thread_stats;static bool combine_locks;static bool show_thread_stats; +static enum { +LOCK_AGGR_ADDR , +LOCK_AGGR_TASK , +LOCK_AGGR_CALLER , +} aggr_mode =LOCK_AGGR_ADDR ; +/* * CONTENTION_STACK_DEPTH * Number of stack trace entries to find callers
@@ -622,12 +628,22 @@ static int report_lock_acquire_event(struct evsel *evsel,const char *name =evsel__strval (evsel, sample,"name" );u64 addr =evsel__intval (evsel, sample,"lockdep_addr" );int flag =evsel__intval (evsel, sample,"flags" ); +u64 key; -/* abuse ls->addr for tid */ -if (show_thread_stats) - addr = sample->tid; +switch (aggr_mode) { +case LOCK_AGGR_ADDR : + key = addr; +break ; +case LOCK_AGGR_TASK : + key = sample->tid; +break ; +case LOCK_AGGR_CALLER : +default : +pr_err ("Invalid aggregation mode: %d\n" , aggr_mode); +return -EINVAL ; + } - ls =lock_stat_findnew (addr, name,0 ); + ls =lock_stat_findnew (key, name,0 );if (!ls)return -ENOMEM ;
@@ -695,11 +711,22 @@ static int report_lock_acquired_event(struct evsel *evsel,u64 contended_term;const char *name =evsel__strval (evsel, sample,"name" );u64 addr =evsel__intval (evsel, sample,"lockdep_addr" ); +u64 key; -if (show_thread_stats) - addr = sample->tid; +switch (aggr_mode) { +case LOCK_AGGR_ADDR : + key = addr; +break ; +case LOCK_AGGR_TASK : + key = sample->tid; +break ; +case LOCK_AGGR_CALLER : +default : +pr_err ("Invalid aggregation mode: %d\n" , aggr_mode); +return -EINVAL ; + } - ls =lock_stat_findnew (addr, name,0 ); + ls =lock_stat_findnew (key, name,0 );if (!ls)return -ENOMEM ;
@@ -757,11 +784,22 @@ static int report_lock_contended_event(struct evsel *evsel,struct lock_seq_stat *seq;const char *name =evsel__strval (evsel, sample,"name" );u64 addr =evsel__intval (evsel, sample,"lockdep_addr" ); +u64 key; -if (show_thread_stats) - addr = sample->tid; +switch (aggr_mode) { +case LOCK_AGGR_ADDR : + key = addr; +break ; +case LOCK_AGGR_TASK : + key = sample->tid; +break ; +case LOCK_AGGR_CALLER : +default : +pr_err ("Invalid aggregation mode: %d\n" , aggr_mode); +return -EINVAL ; + } - ls =lock_stat_findnew (addr, name,0 ); + ls =lock_stat_findnew (key, name,0 );if (!ls)return -ENOMEM ;
@@ -812,11 +850,22 @@ static int report_lock_release_event(struct evsel *evsel,struct lock_seq_stat *seq;const char *name =evsel__strval (evsel, sample,"name" );u64 addr =evsel__intval (evsel, sample,"lockdep_addr" ); +u64 key; -if (show_thread_stats) - addr = sample->tid; +switch (aggr_mode) { +case LOCK_AGGR_ADDR : + key = addr; +break ; +case LOCK_AGGR_TASK : + key = sample->tid; +break ; +case LOCK_AGGR_CALLER : +default : +pr_err ("Invalid aggregation mode: %d\n" , aggr_mode); +return -EINVAL ; + } - ls =lock_stat_findnew (addr, name,0 ); + ls =lock_stat_findnew (key, name,0 );if (!ls)return -ENOMEM ;
@@ -980,11 +1029,22 @@ static int report_lock_contention_begin_event(struct evsel *evsel,struct thread_stat *ts;struct lock_seq_stat *seq;u64 addr =evsel__intval (evsel, sample,"lock_addr" ); +u64 key; -if (show_thread_stats) - addr = sample->tid; +switch (aggr_mode) { +case LOCK_AGGR_ADDR : + key = addr; +break ; +case LOCK_AGGR_TASK : + key = sample->tid; +break ; +case LOCK_AGGR_CALLER : +default : +pr_err ("Invalid aggregation mode: %d\n" , aggr_mode); +return -EINVAL ; + } - ls =lock_stat_find (addr); + ls =lock_stat_find (key);if (!ls) {char buf[128 ];const char *caller = buf;
@@ -993,7 +1053,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel,if (lock_contention_caller (evsel, sample, buf,sizeof (buf)) <0 ) caller ="Unknown" ; - ls =lock_stat_findnew (addr, caller, flags); + ls =lock_stat_findnew (key, caller, flags);if (!ls)return -ENOMEM ; }
@@ -1050,11 +1110,22 @@ static int report_lock_contention_end_event(struct evsel *evsel,struct lock_seq_stat *seq;u64 contended_term;u64 addr =evsel__intval (evsel, sample,"lock_addr" ); +u64 key; -if (show_thread_stats) - addr = sample->tid; +switch (aggr_mode) { +case LOCK_AGGR_ADDR : + key = addr; +break ; +case LOCK_AGGR_TASK : + key = sample->tid; +break ; +case LOCK_AGGR_CALLER : +default : +pr_err ("Invalid aggregation mode: %d\n" , aggr_mode); +return -EINVAL ; + } - ls =lock_stat_find (addr); + ls =lock_stat_find (key);if (!ls)return 0 ;
@@ -1416,6 +1487,9 @@ static int __cmd_report(bool display_info)if (select_key ()) goto out_delete; +if (show_thread_stats) + aggr_mode =LOCK_AGGR_TASK ; + err =perf_session__process_events (session);if (err) goto out_delete;
Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Namhyung Kim
maintainerPATCH 3/5 perf lock: Add 'contention' subcommand
In reply to Namhyung KimThe 'perf lock contention' processes the lock contention events and displays the result like perf lock report. Right now, there's not much difference between the two but the lock contention specific features will come soon.
$ perf lock contention contended total wait max wait avg wait type caller
238 1.41 ms 29.20 us 5.94 us spinlock update_blocked_averages+0x4c
1 902.08 us 902.08 us 902.08 us rwsem:R do_user_addr_fault+0x1dd
81 330.30 us 17.24 us 4.08 us spinlock _nohz_idle_balance+0x172
2 89.54 us 61.26 us 44.77 us spinlock do_anonymous_page+0x16d
24 78.36 us 12.27 us 3.27 us mutex pipe_read+0x56
2 71.58 us 59.56 us 35.79 us spinlock __handle_mm_fault+0x6aa
6 25.68 us 6.89 us 4.28 us spinlock do_idle+0x28d
1 18.46 us 18.46 us 18.46 us rtmutex exec_fw_cmd+0x21b
3 15.25 us 6.26 us 5.08 us spinlock tick_do_update_jiffies64+0x2c--- tools/perf/Documentation/perf-lock.txt | 4 +- tools/perf/builtin-lock.c | 212 ++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 2 deletions(-)
tools/perf/Documentation/perf-lock.txt +3 -1
@@ -8,7 +8,7 @@ perf-lock - Analyze lock events SYNOPSIS -------- [verse] -'perf lock' {record|report|script|info} +'perf lock' {record|report|script|info|contention} DESCRIPTION -----------
@@ -27,6 +27,8 @@ and statistics with this 'perf lock' command. 'perf lock info' shows metadata like threads or addresses of lock instances. + 'perf lock contention' shows contention statistics. + COMMON OPTIONS --------------
tools/perf/builtin-lock.c +211 -1
277 lines not on this page · raw message
Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Namhyung Kim
maintainerPATCH 4/5 perf lock: Add -k and -F options to 'contention' subcommand
In reply to Namhyung KimLike perf lock report, add -k/--key and -F/--field options to control output formatting and sorting. Note that it has slightly different default options as some fields are not available and to optimize the screen space.
--- tools/perf/Documentation/perf-lock.txt | 15 ++++++++ tools/perf/builtin-lock.c | 51 ++++++++++++++++++++------ 2 files changed, 55 insertions(+), 11 deletions(-)
tools/perf/Documentation/perf-lock.txt +15 -0
@@ -105,6 +105,21 @@ INFO OPTIONS --map:: dump map of lock instances (address:name table) +CONTENTION OPTIONS +-------------- + +-k:: +--key=<value>:: + Sorting key. Possible values: contended, wait_total (default), + wait_max, wait_min, avg_wait. + +-F:: +--field=<value>:: + Output fields. By default it shows all but the wait_min fields + and users can customize that using this. Possible values: + contended, wait_total, wait_max, wait_min, avg_wait. + + SEE ALSO -------- linkperf:perf[1]
tools/perf/builtin-lock.c +40 -11
122 lines not on this page · raw message
Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Namhyung Kim
maintainerPATCH 5/5 perf lock: Support -t option for 'contention' subcommand
In reply to Namhyung KimLike perf lock report, it can report lock contention stat of each task.
$ perf lock contention -t contended total wait max wait avg wait pid comm
5 945.20 us 902.08 us 189.04 us 316167 EventManager_De
33 98.17 us 6.78 us 2.97 us 766063 kworker/0:1-get
7 92.47 us 61.26 us 13.21 us 316170 EventManager_De
14 76.31 us 12.87 us 5.45 us 12949 timedcall
24 76.15 us 12.27 us 3.17 us 767992 sched-pipe
15 75.62 us 11.93 us 5.04 us 15127 switchto-defaul
24 71.84 us 5.59 us 2.99 us 629168 kworker/u513:2-
17 67.41 us 7.94 us 3.96 us 13504 coroner-
1 59.56 us 59.56 us 59.56 us 316165 EventManager_De
14 56.21 us 6.89 us 4.01 us 0 swapper--- tools/perf/Documentation/perf-lock.txt | 4 ++++ tools/perf/builtin-lock.c | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-)
tools/perf/Documentation/perf-lock.txt +4 -0
@@ -119,6 +119,10 @@ CONTENTION OPTIONS and users can customize that using this. Possible values: contended, wait_total, wait_max, wait_min, avg_wait. +-t:: +--threads:: + Show per-thread lock contention stat + SEE ALSO --------
tools/perf/builtin-lock.c +20 -2
@@ -1560,7 +1560,10 @@ static void print_contention_result(void)list_for_each_entry (key, &lock_keys, list)pr_info ("%*s " , key->len, key->header); -pr_info (" %10s %s\n\n" ,"type" ,"caller" ); +if (show_thread_stats) +pr_info (" %10s %s\n\n" ,"pid" ,"comm" ); +else +pr_info (" %10s %s\n\n" ,"type" ,"caller" ); bad = total =0 ;while ((st =pop_from_result ())) {
@@ -1573,6 +1576,16 @@ static void print_contention_result(void)pr_info (" " ); } +if (show_thread_stats) { +struct thread *t; +int pid = st->addr; + +/* st->addr contains tid of thread */ + t =perf_session__findnew (session, pid); +pr_info (" %10d %s\n" , pid,thread__comm_str (t)); +continue ; + } +pr_info (" %10s %s\n" ,get_type_str (st), st->name); }
@@ -1703,7 +1716,10 @@ static int __cmd_contention(void)if (select_key (true)) goto out_delete; - aggr_mode =LOCK_AGGR_CALLER ; +if (show_thread_stats) + aggr_mode =LOCK_AGGR_TASK ; +else + aggr_mode =LOCK_AGGR_CALLER ; err =perf_session__process_events (session);if (err)
@@ -1843,6 +1859,8 @@ int cmd_lock(int argc, const char **argv) "key for sorting (contended / wait_total / wait_max / wait_min / avg_wait)"),OPT_STRING ('F' ,"field" , &output_fields,"contended,wait_total,wait_max,avg_wait" ,"output fields (contended / wait_total / wait_max / wait_min / avg_wait)" ), +OPT_BOOLEAN ('t' ,"threads" , &show_thread_stats, +"show per-thread lock stats" ), OPT_PARENT(lock_options) };
Signature · 1 line
2.37.1.359.gd136c6c3e2-goog
Arnaldo Carvalho de Melo
maintainerEm Mon, Jul 25, 2022 at 11:31:19AM -0700, Namhyung Kim escreveu:
Quoted 25 lines
Hello,
It's to add a new subcommand 'contention' (shortly 'con') to perf lock.
Changes in v2)
* bugfix is merged already
* fix build error in patch 2The new subcommand is to handle the new lock:contention_{begin,end} tracepoints and shows lock type and caller address like below:
$ perf lock contention contended total wait max wait avg wait type caller238 1.41 ms 29.20 us 5.94 us spinlock update_blocked_averages+0x4c 1 902.08 us 902.08 us 902.08 us rwsem:R do_user_addr_fault+0x1dd 81 330.30 us 17.24 us 4.08 us spinlock _nohz_idle_balance+0x172 2 89.54 us 61.26 us 44.77 us spinlock do_anonymous_page+0x16d 24 78.36 us 12.27 us 3.27 us mutex pipe_read+0x56 2 71.58 us 59.56 us 35.79 us spinlock __handle_mm_fault+0x6aa 6 25.68 us 6.89 us 4.28 us spinlock do_idle+0x28d 1 18.46 us 18.46 us 18.46 us rtmutex exec_fw_cmd+0x21b 3 15.25 us 6.26 us 5.08 us spinlock tick_do_update_jiffies64+0x2c...
Quoted 8 lines
where rwsem:R stands for read access (
down_read) for a rw-semaphore. Other types of lock access is obvious and it doesn't detect optimistic spinning on mutex yet. This is just a base work for lock contention analysis and more will come later.You can get this from 'perf/lock-subcmd-v2' branch on
git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.git
Thanks, applied locally, pushed to tmp.perf/core, will go to perf/core after further automated tests.
- Arnaldo
Quoted 19 lines
Thanks,
NamhyungNamhyung Kim (5):
perf lock: Add flags field in thelock_stat
perf lock: Add lock aggregation enum
perf lock: Add 'contention' subcommand
perf lock: Add -k and -F options to 'contention' subcommand perf lock: Support -t option for 'contention' subcommandtools/perf/Documentation/perf-lock.txt | 23 +- tools/perf/builtin-lock.c | 401 +++++++++++++++++++++++-- 2 files changed, 390 insertions(+), 34 deletions(-)base-commit:
9fe9b252c7c022df8e503435e778f15c04dfa3bfSignature · 1 line
2.37.1.359.gd136c6c3e2-goog
Signature · 1 line
- Arnaldo