perf lock contention: Add cgroup support (v1)
Subsystems (2)
Namhyung Kim
maintainercover perf lock contention: Add cgroup support (v1)
Hello,
The cgroup support comes with two flavors. One is to aggregate the result by cgroups and the other is to filter result for the given cgroups. For now, it only works in BPF mode.
The first one is -g/--lock-cgroup option to show lock stats by cgroups like below. The cgroup names were shortened for brevity:
$ sudo perf lock con -abg perf bench sched messaging contended total wait max wait avg wait cgroup
1052 3.34 ms 84.71 us 3.17 us /app-org.gnome.Terminal.slice/vte-spawn-52221fb8-b33f-4a52-b5c3-e35d1e6fc0e0.scope
13 106.60 us 11.48 us 8.20 us /session.slice/[email protected]
12 21.20 us 4.93 us 1.77 us /
3 12.10 us 8.80 us 4.03 us /session-4.scope
2 10.98 us 7.50 us 5.49 us /app-gnome-firefox\x2desr-34054.scope
2 6.04 us 4.88 us 3.02 us /app-gnome-google\x2dchrome-6442.scope
1 5.63 us 5.63 us 5.63 us /app-org.gnome.Terminal.slice/gnome-terminal-server.service
1 3.51 us 3.51 us 3.51 us /pipewire.service
1 2.15 us 2.15 us 2.15 us /pipewire-pulse.service
1 742 ns 742 ns 742 ns /dbus.serviceThe other is -G/--cgroup-filter option to show lock stats only from the given cgroups. It doesn't support cgroup hierarchy and regex matching.
$ sudo perf lock con -abt -G / perf bench sched messaging contended total wait max wait avg wait pid comm
2 10.58 us 8.39 us 5.29 us 257552 kworker/4:1
2 9.76 us 7.96 us 4.88 us 0 swapper
4 5.36 us 2.09 us 1.34 us 255462 kworker/0:2
3 3.33 us 1.48 us 1.11 us 257680 kworker/3:1
2 2.59 us 1.46 us 1.29 us 257478 kworker/2:2
1 1.50 us 1.50 us 1.50 us 15 rcu_preemptYou can also use these two options together. :)
The two more test cases were added to the existing lock contention test.
The code is available at 'perf/lock-cgroup-v1' branch in the tree below.
git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.git
Thanks,
Namhyung
Namhyung Kim (5):
perf tools: Add read_all_cgroups() and __cgroup_find() perf lock contention: Prepare to handle cgroups perf lock contention: Add -g/--lock-cgroup option perf lock contention: Add -G/--cgroup-filter option perf test: Improve perf lock contention test
tools/perf/Documentation/perf-lock.txt | 8 ++ tools/perf/builtin-lock.c | 99 ++++++++++++++++++- tools/perf/tests/shell/lock_contention.sh | 45 +++++++++ tools/perf/util/bpf_lock_contention.c | 51 +++++++++- .../perf/util/bpf_skel/lock_contention.bpf.c | 48 ++++++++- tools/perf/util/bpf_skel/lock_data.h | 3 +- tools/perf/util/cgroup.c | 63 ++++++++++-- tools/perf/util/cgroup.h | 5 + tools/perf/util/lock-contention.h | 10 +- 9 files changed, 312 insertions(+), 20 deletions(-)
base-commit: d2045f87154bf67a50ebefe28d2ca0e1e3f8eef1
Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Namhyung Kim
maintainerPATCH 1/5 perf tools: Add read_all_cgroups() and __cgroup_find()
The read_all_cgroups() is to build a tree of cgroups in the system and users can look up a cgroup using __cgroup_find().
--- tools/perf/util/cgroup.c | 61 ++++++++++++++++++++++++++++++++++------ tools/perf/util/cgroup.h | 4 +++ 2 files changed, 57 insertions(+), 8 deletions(-)
tools/perf/util/cgroup.c +53 -8
@@ -48,28 +48,36 @@ static int open_cgroup(const char *name) }#ifdef HAVE_FILE_HANDLE -int read_cgroup_id (struct cgroup *cgrp) +static u64 __read_cgroup_id (const char *path) { -char path[PATH_MAX +1 ]; -char mnt[PATH_MAX +1 ];struct {struct file_handle fh;uint64_t cgroup_id; } handle;int mount_id; + handle.fh.handle_bytes =sizeof (handle.cgroup_id); +if (name_to_handle_at (AT_FDCWD , path, &handle.fh, &mount_id,0 ) <0 ) +return -1ULL ; + +return handle.cgroup_id; +} + +int read_cgroup_id (struct cgroup *cgrp) +{ +char path[PATH_MAX +1 ]; +char mnt[PATH_MAX +1 ]; +if (cgroupfs_find_mountpoint (mnt,PATH_MAX +1 ,"perf_event" ))return -1 ;scnprintf (path,PATH_MAX ,"%s/%s" , mnt, cgrp->name); - handle.fh.handle_bytes =sizeof (handle.cgroup_id); -if (name_to_handle_at (AT_FDCWD , path, &handle.fh, &mount_id,0 ) <0 ) -return -1 ; - - cgrp->id = handle.cgroup_id; + cgrp->id =__read_cgroup_id (path);return 0 ; } +#else +static inline u64 __read_cgroup_id (const char *path) {return -1ULL ; }#endif /* HAVE_FILE_HANDLE */ #ifndef CGROUP2_SUPER_MAGIC
@@ -562,6 +570,11 @@ struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,return cgrp; } +struct cgroup *__cgroup__find (struct rb_root *root,uint64_t id) +{ +return __cgroup__findnew (root, id,/*create=*/ false,/*path=*/ NULL ); +} +struct cgroup *cgroup__find (struct perf_env *env,uint64_t id) {struct cgroup *cgrp;
@@ -587,3 +600,35 @@ void perf_env__purge_cgroups(struct perf_env *env) }up_write (&env->cgroups.lock); } + +void read_all_cgroups (struct rb_root *root) +{ +char mnt[PATH_MAX ]; +struct cgroup_name *cn; +int prefix_len; + +if (cgroupfs_find_mountpoint (mnt,sizeof (mnt),"perf_event" )) +return ; + +/* cgroup_name will have a full path, skip the root directory */ + prefix_len =strlen (mnt); + +/* collect all cgroups in the cgroup_list */ +if (nftw (mnt, add_cgroup_name,20 ,0 ) <0 ) +return ; + +list_for_each_entry (cn, &cgroup_list, list) { +const char *name; +u64 cgrp_id; + +/* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; +if (name[0 ] =='\0' ) + name ="/" ; + + cgrp_id =__read_cgroup_id (cn->name); +__cgroup__findnew (root, cgrp_id,/*create=*/ true, name); + } + +release_cgroup_list (); +}
tools/perf/util/cgroup.h +4 -0
@@ -37,6 +37,7 @@ int parse_cgroups(const struct option *opt, const char *str, int unset);struct cgroup *cgroup__findnew (struct perf_env *env,uint64_t id,const char *path);struct cgroup *cgroup__find (struct perf_env *env,uint64_t id); +struct cgroup *__cgroup__find (struct rb_root *root,uint64_t id);void perf_env__purge_cgroups (struct perf_env *env);
@@ -49,6 +50,9 @@ static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) }#endif /* HAVE_FILE_HANDLE */ +/* read all cgroups in the system and save them in the rbtree */ +void read_all_cgroups (struct rb_root *root); +int cgroup_is_v2 (const char *subsys);#endif /* __CGROUP_H__ */
Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Arnaldo Carvalho de Melo
maintainerRe: [PATCH 1/5] perf tools: Add read_all_cgroups() and __cgroup_find()
Em Wed, Aug 30, 2023 at 04:01:22PM -0700, Namhyung Kim escreveu:
Quoted 2 lines
The
read_all_cgroups()is to build a tree of cgroups in the system and users can look up a cgroup using__cgroup_find().
⬢[acme@toolbox perf-tools-next]$ alias m='make -k BUILD_BPF_SKEL=1 CORESIGHT=1 O=/tmp/build/perf-tools-next -C tools/perf install-bin && git status && perf test python' ⬢[acme@toolbox perf-tools-next]$ m
make: Entering directory '/var/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -u tools/include/uapi/linux/perf_event.h include/uapi/linux/perf_event.h
diff -u tools/arch/x86/include/asm/cpufeatures.h arch/x86/include/asm/cpufeatures.h
diff -u tools/arch/x86/include/asm/msr-index.h arch/x86/include/asm/msr-index.h
diff -u tools/arch/arm64/include/uapi/asm/perf_regs.h arch/arm64/include/uapi/asm/perf_regs.h
INSTALL libsubcmd_headers
INSTALL libperf_headers
INSTALL libapi_headers
INSTALL libsymbol_headers
INSTALL libbpf_headers
CC /tmp/build/perf-tools-next/builtin-lock.o
CC /tmp/build/perf-tools-next/util/bpf_lock_contention.obuiltin-lock.c: In function ‘__cmd_contention’:
builtin-lock.c:2162:9: error: too few arguments to function ‘lock_contention_finish’
2162 | lock_contention_finish();
| ^~~~~~~~~~~~~~~~~~~~~~
In file included from builtin-lock.c:14:
util/lock-contention.h:156:5: note: declared here
156 | int lock_contention_finish(struct lock_contention *con);
| ^~~~~~~~~~~~~~~~~~~~~~make[3]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:97: /tmp/build/perf-tools-next/builtin-lock.o] Error 1 make[3]: *** Waiting for unfinished jobs....util/bpf_lock_contention.c: In function ‘lock_contention_get_name’:util/bpf_lock_contention.c:231:34: error: ‘struct contention_key’ has no member named ‘lock_addr_or_cgroup’
231 | u64 cgrp_id = key->lock_addr_or_cgroup;
| ^~
make[4]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:97: /tmp/build/perf-tools-next/util/bpf_lock_contention.o] Error 1 make[3]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:150: util] Error 2
make[2]: *** [Makefile.perf:662: /tmp/build/perf-tools-next/perf-in.o] Error 2
make[1]: *** [Makefile.perf:238: sub-make] Error 2
make: *** [Makefile:113: install-bin] Error 2
make: Leaving directory '/var/home/acme/git/perf-tools-next/tools/perf'
⬢[acme@toolbox perf-tools-next]$
Trying to figure this out.
- Arnaldo
Quoted 128 lines
Signed-off-by Namhyung Kim <[email protected]>--- tools/perf/util/cgroup.c | 61 ++++++++++++++++++++++++++++++++++------ tools/perf/util/cgroup.h | 4 +++ 2 files changed, 57 insertions(+), 8 deletions(-)tools/perf/util/cgroup.c +53
-8@@ -48,28 +48,36 @@ static int open_cgroup(const char *name) }#ifdef HAVE_FILE_HANDLE -int read_cgroup_id (struct cgroup *cgrp) +static u64 __read_cgroup_id (const char *path) { -char path[PATH_MAX +1 ]; -char mnt[PATH_MAX +1 ];struct {struct file_handle fh;uint64_t cgroup_id; } handle;int mount_id; + handle.fh.handle_bytes =sizeof (handle.cgroup_id); +if (name_to_handle_at (AT_FDCWD , path, &handle.fh, &mount_id,0 ) <0 ) +return -1ULL ; + +return handle.cgroup_id; +} + +int read_cgroup_id (struct cgroup *cgrp) +{ +char path[PATH_MAX +1 ]; +char mnt[PATH_MAX +1 ]; +if (cgroupfs_find_mountpoint (mnt,PATH_MAX +1 ,"perf_event" ))return -1 ;scnprintf (path,PATH_MAX ,"%s/%s" , mnt, cgrp->name); - handle.fh.handle_bytes =sizeof (handle.cgroup_id); -if (name_to_handle_at (AT_FDCWD , path, &handle.fh, &mount_id,0 ) <0 ) -return -1 ; - - cgrp->id = handle.cgroup_id; + cgrp->id =__read_cgroup_id (path);return 0 ; } +#else +static inline u64 __read_cgroup_id (const char *path) {return -1ULL ; }#endif /* HAVE_FILE_HANDLE */ #ifndef CGROUP2_SUPER_MAGIC @@ -562,6 +570,11 @@ struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,return cgrp; } +struct cgroup *__cgroup__find (struct rb_root *root,uint64_t id) +{ +return __cgroup__findnew (root, id,/*create=*/ false,/*path=*/ NULL ); +} +struct cgroup *cgroup__find (struct perf_env *env,uint64_t id) {struct cgroup *cgrp;@@ -587,3 +600,35 @@ void perf_env__purge_cgroups(struct perf_env *env) }up_write (&env->cgroups.lock); } + +void read_all_cgroups (struct rb_root *root) +{ +char mnt[PATH_MAX ]; +struct cgroup_name *cn; +int prefix_len; + +if (cgroupfs_find_mountpoint (mnt,sizeof (mnt),"perf_event" )) +return ; + +/* cgroup_name will have a full path, skip the root directory */ + prefix_len =strlen (mnt); + +/* collect all cgroups in the cgroup_list */ +if (nftw (mnt, add_cgroup_name,20 ,0 ) <0 ) +return ; + +list_for_each_entry (cn, &cgroup_list, list) { +const char *name; +u64 cgrp_id; + +/* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; +if (name[0 ] =='\0' ) + name ="/" ; + + cgrp_id =__read_cgroup_id (cn->name); +__cgroup__findnew (root, cgrp_id,/*create=*/ true, name); + } + +release_cgroup_list (); +}tools/perf/util/cgroup.h +4
-0@@ -37,6 +37,7 @@ int parse_cgroups(const struct option *opt, const char *str, int unset);struct cgroup *cgroup__findnew (struct perf_env *env,uint64_t id,const char *path);struct cgroup *cgroup__find (struct perf_env *env,uint64_t id); +struct cgroup *__cgroup__find (struct rb_root *root,uint64_t id);void perf_env__purge_cgroups (struct perf_env *env);@@ -49,6 +50,9 @@ static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) }#endif /* HAVE_FILE_HANDLE */ +/* read all cgroups in the system and save them in the rbtree */ +void read_all_cgroups (struct rb_root *root); +int cgroup_is_v2 (const char *subsys);#endif /* __CGROUP_H__ */ Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Signature · 1 line
- Arnaldo
Arnaldo Carvalho de Melo
maintainerRe: [PATCH 1/5] perf tools: Add read_all_cgroups() and __cgroup_find()
Em Wed, Sep 06, 2023 at 12:38:17PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 43 lines
Em Wed, Aug 30, 2023 at 04:01:22PM -0700, Namhyung Kim escreveu:
Quoted 2 lines
The
read_all_cgroups()is to build a tree of cgroups in the system and users can look up a cgroup using__cgroup_find().⬢[acme@toolbox perf-tools-next]$ alias m='make -k BUILD_BPF_SKEL=1 CORESIGHT=1 O=/tmp/build/perf-tools-next -C
tools/perfinstall-bin && git status && perf test python' ⬢[acme@toolbox perf-tools-next]$ m
make: Entering directory '/var/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -utools/include/uapi/linux/perf_event.hinclude/uapi/linux/perf_event.h
diff -utools/arch/x86/include/asm/cpufeatures.harch/x86/include/asm/cpufeatures.h
diff -utools/arch/x86/include/asm/msr-index.harch/x86/include/asm/msr-index.h
diff -utools/arch/arm64/include/uapi/asm/perf_regs.harch/arm64/include/uapi/asm/perf_regs.hINSTALL
libsubcmd_headers
INSTALLlibperf_headers
INSTALLlibapi_headers
INSTALLlibsymbol_headers
INSTALLlibbpf_headers
CC/tmp/build/perf-tools-next/builtin-lock.o
CC/tmp/build/perf-tools-next/util/bpf_lock_contention.obuiltin-lock.c: In function ‘__cmd_contention’:
builtin-lock.c:2162:9: error: too few arguments to function ‘lock_contention_finish’2162 | lock_contention_finish(); | ^~~~~~~~~~~~~~~~~~~~~~ In file included from builtin-lock.c:14: util/lock-contention.h:156:5: note: declared here 156 | int lock_contention_finish(struct lock_contention *con); | ^~~~~~~~~~~~~~~~~~~~~~make[3]: *** [
/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:97:/tmp/build/perf-tools-next/builtin-lock.o] Error 1 make[3]: *** Waiting for unfinished jobs....util/bpf_lock_contention.c: In function ‘lock_contention_get_name’:util/bpf_lock_contention.c:231:34: error: ‘struct contention_key’ has no member named ‘lock_addr_or_cgroup’
231 | u64cgrp_id=key->lock_addr_or_cgroup;
| ^~
make[4]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:97:/tmp/build/perf-tools-next/util/bpf_lock_contention.o] Error 1 make[3]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:150: util] Error 2
make[2]: *** [Makefile.perf:662:/tmp/build/perf-tools-next/perf-in.o] Error 2
make[1]: *** [Makefile.perf:238: sub-make] Error 2
make: *** [Makefile:113: install-bin] Error 2
make: Leaving directory '/var/home/acme/git/perf-tools-next/tools/perf'
⬢[acme@toolbox perf-tools-next]$
Quoted 1 line
Trying to figure this out.
So it works on the following patch:
⬢[acme@toolbox perf-tools-next]$ git log --oneline -5 94a54d498ae35c66 (HEAD) perf lock contention: Add -g/--lock-cgroup option defe88978441a00d perf lock contention: Prepare to handle cgroups cc0717270d2f0daa perf cgroup: Add read_all_cgroups() and __cgroup_find() 752d73a1dd62cd4a perf shell completion: Support completion of metrics/metricgroups 72aa5816258bf9fe perf completion: Support completion of libpfm4 events ⬢[acme@toolbox perf-tools-next]$
Please check and submit a v2.
- Arnaldo
Quoted 134 lines
- Arnaldo
Quoted 128 lines
Signed-off-by Namhyung Kim <[email protected]>--- tools/perf/util/cgroup.c | 61 ++++++++++++++++++++++++++++++++++------ tools/perf/util/cgroup.h | 4 +++ 2 files changed, 57 insertions(+), 8 deletions(-)tools/perf/util/cgroup.c +53
-8@@ -48,28 +48,36 @@ static int open_cgroup(const char *name) }#ifdef HAVE_FILE_HANDLE -int read_cgroup_id (struct cgroup *cgrp) +static u64 __read_cgroup_id (const char *path) { -char path[PATH_MAX +1 ]; -char mnt[PATH_MAX +1 ];struct {struct file_handle fh;uint64_t cgroup_id; } handle;int mount_id; + handle.fh.handle_bytes =sizeof (handle.cgroup_id); +if (name_to_handle_at (AT_FDCWD , path, &handle.fh, &mount_id,0 ) <0 ) +return -1ULL ; + +return handle.cgroup_id; +} + +int read_cgroup_id (struct cgroup *cgrp) +{ +char path[PATH_MAX +1 ]; +char mnt[PATH_MAX +1 ]; +if (cgroupfs_find_mountpoint (mnt,PATH_MAX +1 ,"perf_event" ))return -1 ;scnprintf (path,PATH_MAX ,"%s/%s" , mnt, cgrp->name); - handle.fh.handle_bytes =sizeof (handle.cgroup_id); -if (name_to_handle_at (AT_FDCWD , path, &handle.fh, &mount_id,0 ) <0 ) -return -1 ; - - cgrp->id = handle.cgroup_id; + cgrp->id =__read_cgroup_id (path);return 0 ; } +#else +static inline u64 __read_cgroup_id (const char *path) {return -1ULL ; }#endif /* HAVE_FILE_HANDLE */ #ifndef CGROUP2_SUPER_MAGIC @@ -562,6 +570,11 @@ struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,return cgrp; } +struct cgroup *__cgroup__find (struct rb_root *root,uint64_t id) +{ +return __cgroup__findnew (root, id,/*create=*/ false,/*path=*/ NULL ); +} +struct cgroup *cgroup__find (struct perf_env *env,uint64_t id) {struct cgroup *cgrp;@@ -587,3 +600,35 @@ void perf_env__purge_cgroups(struct perf_env *env) }up_write (&env->cgroups.lock); } + +void read_all_cgroups (struct rb_root *root) +{ +char mnt[PATH_MAX ]; +struct cgroup_name *cn; +int prefix_len; + +if (cgroupfs_find_mountpoint (mnt,sizeof (mnt),"perf_event" )) +return ; + +/* cgroup_name will have a full path, skip the root directory */ + prefix_len =strlen (mnt); + +/* collect all cgroups in the cgroup_list */ +if (nftw (mnt, add_cgroup_name,20 ,0 ) <0 ) +return ; + +list_for_each_entry (cn, &cgroup_list, list) { +const char *name; +u64 cgrp_id; + +/* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; +if (name[0 ] =='\0' ) + name ="/" ; + + cgrp_id =__read_cgroup_id (cn->name); +__cgroup__findnew (root, cgrp_id,/*create=*/ true, name); + } + +release_cgroup_list (); +}tools/perf/util/cgroup.h +4
-0@@ -37,6 +37,7 @@ int parse_cgroups(const struct option *opt, const char *str, int unset);struct cgroup *cgroup__findnew (struct perf_env *env,uint64_t id,const char *path);struct cgroup *cgroup__find (struct perf_env *env,uint64_t id); +struct cgroup *__cgroup__find (struct rb_root *root,uint64_t id);void perf_env__purge_cgroups (struct perf_env *env);@@ -49,6 +50,9 @@ static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) }#endif /* HAVE_FILE_HANDLE */ +/* read all cgroups in the system and save them in the rbtree */ +void read_all_cgroups (struct rb_root *root); +int cgroup_is_v2 (const char *subsys);#endif /* __CGROUP_H__ */ Signature · 1 line
2.42.0.283.g2d96d420d3-googSignature · 1 line
- Arnaldo
Signature · 1 line
- Arnaldo
Namhyung Kim
maintainerRe: [PATCH 1/5] perf tools: Add read_all_cgroups() and __cgroup_find()
Hi Arnaldo,
Quoted 57 lines · Arnaldo Carvalho de Melo
Em Wed, Sep 06, 2023 at 12:38:17PM -0300, Arnaldo Carvalho de Melo escreveu:
Quoted 43 lines
Em Wed, Aug 30, 2023 at 04:01:22PM -0700, Namhyung Kim escreveu:
Quoted 2 lines
The
read_all_cgroups()is to build a tree of cgroups in the system and users can look up a cgroup using__cgroup_find().⬢[acme@toolbox perf-tools-next]$ alias m='make -k BUILD_BPF_SKEL=1 CORESIGHT=1 O=/tmp/build/perf-tools-next -C
tools/perfinstall-bin && git status && perf test python' ⬢[acme@toolbox perf-tools-next]$ m
make: Entering directory '/var/home/acme/git/perf-tools-next/tools/perf'
BUILD: Doing 'make -j32' parallel build
Warning: Kernel ABI header differences:
diff -utools/include/uapi/linux/perf_event.hinclude/uapi/linux/perf_event.h
diff -utools/arch/x86/include/asm/cpufeatures.harch/x86/include/asm/cpufeatures.h
diff -utools/arch/x86/include/asm/msr-index.harch/x86/include/asm/msr-index.h
diff -utools/arch/arm64/include/uapi/asm/perf_regs.harch/arm64/include/uapi/asm/perf_regs.hINSTALL
libsubcmd_headers
INSTALLlibperf_headers
INSTALLlibapi_headers
INSTALLlibsymbol_headers
INSTALLlibbpf_headers
CC/tmp/build/perf-tools-next/builtin-lock.o
CC/tmp/build/perf-tools-next/util/bpf_lock_contention.obuiltin-lock.c: In function ‘__cmd_contention’:
builtin-lock.c:2162:9: error: too few arguments to function ‘lock_contention_finish’2162 | lock_contention_finish(); | ^~~~~~~~~~~~~~~~~~~~~~ In file included from builtin-lock.c:14: util/lock-contention.h:156:5: note: declared here 156 | int lock_contention_finish(struct lock_contention *con); | ^~~~~~~~~~~~~~~~~~~~~~make[3]: *** [
/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:97:/tmp/build/perf-tools-next/builtin-lock.o] Error 1 make[3]: *** Waiting for unfinished jobs....util/bpf_lock_contention.c: In function ‘lock_contention_get_name’:util/bpf_lock_contention.c:231:34: error: ‘struct contention_key’ has no member named ‘lock_addr_or_cgroup’
231 | u64cgrp_id=key->lock_addr_or_cgroup;
| ^~
make[4]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:97:/tmp/build/perf-tools-next/util/bpf_lock_contention.o] Error 1 make[3]: *** [/var/home/acme/git/perf-tools-next/tools/build/Makefile.build:150: util] Error 2
make[2]: *** [Makefile.perf:662:/tmp/build/perf-tools-next/perf-in.o] Error 2
make[1]: *** [Makefile.perf:238: sub-make] Error 2
make: *** [Makefile:113: install-bin] Error 2
make: Leaving directory '/var/home/acme/git/perf-tools-next/tools/perf'
⬢[acme@toolbox perf-tools-next]$Quoted 1 line
Trying to figure this out.
So it works on the following patch:
⬢[acme@toolbox perf-tools-next]$ git log --oneline -5
94a54d498ae35c66(HEAD) perf lock contention: Add -g/--lock-cgroup optiondefe88978441a00dperf lock contention: Prepare to handle cgroupscc0717270d2f0daaperf cgroup: Addread_all_cgroups()and__cgroup_find()752d73a1dd62cd4aperf shell completion: Support completion of metrics/metricgroups72aa5816258bf9feperf completion: Support completion of libpfm4 events ⬢[acme@toolbox perf-tools-next]$
Quoted 1 line
Please check and submit a v2.
Thanks for the report, I'll fix it.
Thanks,
Namhyung
Namhyung Kim
maintainerPATCH 2/5 perf lock contention: Prepare to handle cgroups
In reply to Namhyung KimSave cgroup info and display cgroup names if requested. This is a preparation for the next patch.
--- tools/perf/builtin-lock.c | 1 + tools/perf/util/bpf_lock_contention.c | 26 +++++++++++++++++++++++++- tools/perf/util/lock-contention.h | 9 +++++++-- 3 files changed, 33 insertions(+), 3 deletions(-)
tools/perf/builtin-lock.c +1 -0
@@ -2040,6 +2040,7 @@ static int __cmd_contention(int argc, const char **argv) .filters = &filters, .save_callstack = needs_callstack(), .owner = show_lock_owner, + .cgroups = RB_ROOT, }; lockhash_table =calloc (LOCKHASH_SIZE ,sizeof (*lockhash_table));
tools/perf/util/bpf_lock_contention.c +25 -1
@@ -1,4 +1,5 @@// SPDX-License-Identifier: GPL-2.0 +#include "util/cgroup.h" #include "util/debug.h" #include "util/evlist.h" #include "util/machine.h"
@@ -151,6 +152,10 @@ int lock_contention_prepare(struct lock_contention *con) skel->bss->needs_callstack = con->save_callstack; skel->bss->lock_owner = con->owner; +if (con->use_cgroup) { +read_all_cgroups (&con->cgroups); + } +bpf_program__set_autoload (skel->progs.collect_lock_syms, false);lock_contention_bpf__attach (skel);
@@ -222,6 +227,17 @@ static const char *lock_contention_get_name(struct lock_contention *con,return "" ; } +if (con->use_cgroup) { +u64 cgrp_id = key->lock_addr_or_cgroup; +struct cgroup *cgrp =__cgroup__find (&con->cgroups, cgrp_id); + +if (cgrp) +return cgrp->name; + +snprintf (name_buf,sizeof (name_buf),"cgroup:%lu" , cgrp_id); +return name_buf; + } +/* LOCK_AGGR_CALLER: skip lock internal functions */ while (machine__is_lock_function (machine, stack_trace[idx]) && idx < con->max_stack -1 )
@@ -364,12 +380,20 @@ int lock_contention_read(struct lock_contention *con)return err; } -int lock_contention_finish (void ) +int lock_contention_finish (struct lock_contention *con) {if (skel) { skel->bss->enabled =0 ;lock_contention_bpf__destroy (skel); } +while (!RB_EMPTY_ROOT (&con->cgroups)) { +struct rb_node *node =rb_first (&con->cgroups); + struct cgroup *cgrp = rb_entry(node, struct cgroup, node); + +rb_erase (node, &con->cgroups); +cgroup__put (cgrp); + } +return 0 ; }
tools/perf/util/lock-contention.h +7 -2
@@ -136,6 +136,7 @@ struct lock_contention {struct hlist_head *result;struct lock_filter *filters;struct lock_contention_fails fails; +struct rb_root cgroups;unsigned long map_nr_entries;int max_stack;int stack_skip;
@@ -143,6 +144,7 @@ struct lock_contention {int owner;int nr_filtered;bool save_callstack; +bool use_cgroup; };#ifdef HAVE_BPF_SKEL
@@ -151,7 +153,7 @@ int lock_contention_prepare(struct lock_contention *con);int lock_contention_start (void );int lock_contention_stop (void );int lock_contention_read (struct lock_contention *con); -int lock_contention_finish (void ); +int lock_contention_finish (struct lock_contention *con);#else /* !HAVE_BPF_SKEL */
@@ -162,7 +164,10 @@ static inline int lock_contention_prepare(struct lock_contention *con __maybe_unstatic inline int lock_contention_start (void ) {return 0 ; }static inline int lock_contention_stop (void ) {return 0 ; } -static inline int lock_contention_finish (void ) {return 0 ; } +static inline int lock_contention_finish (struct lock_contention *con__maybe_unused ) +{ +return 0 ; +}static inline int lock_contention_read (struct lock_contention *con__maybe_unused ) {
Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Namhyung Kim
maintainerPATCH 3/5 perf lock contention: Add -g/--lock-cgroup option
In reply to Namhyung KimThe -g option shows lock contention stats break down by cgroups. Add LOCK_AGGR_CGROUP mode and use it instead of use_cgroup field.
$ sudo ./perf lock con -abg sleep 1 contended total wait max wait avg wait cgroup
8 15.70 us 6.34 us 1.96 us /
2 1.48 us 747 ns 738 ns /user.slice/.../app.slice/app-gnome-google\x2dchrome-6442.scope
1 848 ns 848 ns 848 ns /user.slice/.../session.slice/[email protected]
1 220 ns 220 ns 220 ns /user.slice/.../session.slice/pipewire-pulse.serviceFor now, the cgroup mode only works with BPF (-b).
--- tools/perf/Documentation/perf-lock.txt | 4 ++ tools/perf/builtin-lock.c | 42 ++++++++++++++++++- tools/perf/util/bpf_lock_contention.c | 14 ++++--- .../perf/util/bpf_skel/lock_contention.bpf.c | 31 +++++++++++++- tools/perf/util/bpf_skel/lock_data.h | 3 +- tools/perf/util/lock-contention.h | 1 - 6 files changed, 85 insertions(+), 10 deletions(-)
tools/perf/Documentation/perf-lock.txt +4 -0
@@ -208,6 +208,10 @@ CONTENTION OPTIONS Show results using a CSV-style output to make it easy to import directly into spreadsheets. Columns are separated by the string specified in SEP. +-g:: +--lock-cgroup:: + Show lock contention stat by cgroup. Requires --use-bpf. + SEE ALSO --------
tools/perf/builtin-lock.c +40 -2
108 lines not on this page · raw message
tools/perf/util/bpf_lock_contention.c +9 -5
@@ -152,7 +152,10 @@ int lock_contention_prepare(struct lock_contention *con) skel->bss->needs_callstack = con->save_callstack; skel->bss->lock_owner = con->owner; -if (con->use_cgroup) { +if (con->aggr_mode ==LOCK_AGGR_CGROUP ) { +if (cgroup_is_v2 ("perf_event" )) + skel->bss->use_cgroup_v2 =1 ; +read_all_cgroups (&con->cgroups); }
@@ -214,12 +217,12 @@ static const char *lock_contention_get_name(struct lock_contention *con,return "siglock" ;/* global locks with symbols */ - sym =machine__find_kernel_symbol (machine, key->lock_addr, &kmap); + sym =machine__find_kernel_symbol (machine, key->lock_addr_or_cgroup, &kmap);if (sym)return sym->name;/* try semi-global locks collected separately */ -if (!bpf_map_lookup_elem (lock_fd, &key->lock_addr, &flags)) { +if (!bpf_map_lookup_elem (lock_fd, &key->lock_addr_or_cgroup, &flags)) {if (flags ==LOCK_CLASS_RQLOCK )return "rq_lock" ; }
@@ -227,7 +230,7 @@ static const char *lock_contention_get_name(struct lock_contention *con,return "" ; } -if (con->use_cgroup) { +if (con->aggr_mode ==LOCK_AGGR_CGROUP ) {u64 cgrp_id = key->lock_addr_or_cgroup;struct cgroup *cgrp =__cgroup__find (&con->cgroups, cgrp_id);
@@ -329,7 +332,8 @@ int lock_contention_read(struct lock_contention *con) ls_key = key.pid;break ;case LOCK_AGGR_ADDR : - ls_key = key.lock_addr; +case LOCK_AGGR_CGROUP : + ls_key = key.lock_addr_or_cgroup;break ;default : goto next;
tools/perf/util/bpf_skel/lock_contention.bpf.c +30 -1
@@ -118,6 +118,9 @@ int needs_callstack;int stack_skip;int lock_owner; +int use_cgroup_v2; +int perf_subsys_id =-1 ; +/* determine the key of lock stat */ int aggr_mode;
@@ -130,6 +133,29 @@ int data_fail;int task_map_full;int data_map_full; +static inline __u64 get_current_cgroup_id (void ) +{ +struct task_struct *task; +struct cgroup *cgrp; + +if (use_cgroup_v2) +return bpf_get_current_cgroup_id (); + + task =bpf_get_current_task_btf (); + +if (perf_subsys_id ==-1 ) { +#if __has_builtin (__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp =BPF_CORE_READ (task, cgroups, subsys[perf_subsys_id], cgroup); +return BPF_CORE_READ (cgrp, kn, id); +} +static inline int can_record (u64 *ctx) {if (has_cpu) {
@@ -364,10 +390,13 @@ int contention_end(u64 *ctx) key.stack_id = pelem->stack_id;break ;case LOCK_AGGR_ADDR : - key.lock_addr = pelem->lock; + key.lock_addr_or_cgroup = pelem->lock;if (needs_callstack) key.stack_id = pelem->stack_id;break ; +case LOCK_AGGR_CGROUP : + key.lock_addr_or_cgroup =get_current_cgroup_id (); +break ;default :/* should not happen */ return 0 ;
tools/perf/util/bpf_skel/lock_data.h +2 -1
@@ -6,7 +6,7 @@struct contention_key {u32 stack_id;u32 pid; -u64 lock_addr; +u64 lock_addr_or_cgroup; };#define TASK_COMM_LEN 16
@@ -39,6 +39,7 @@ enum lock_aggr_mode {LOCK_AGGR_ADDR =0 ,LOCK_AGGR_TASK ,LOCK_AGGR_CALLER , + LOCK_AGGR_CGROUP, }; enum lock_class_sym {
tools/perf/util/lock-contention.h +0 -1
@@ -144,7 +144,6 @@ struct lock_contention {int owner;int nr_filtered;bool save_callstack; -bool use_cgroup; };#ifdef HAVE_BPF_SKEL
Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Namhyung Kim
maintainerPATCH 4/5 perf lock contention: Add -G/--cgroup-filter option
In reply to Namhyung KimThe -G/--cgroup-filter is to limit lock contention collection on the tasks in the specific cgroups only.
$ sudo ./perf lock con -abt -G /user.slice/.../vte-spawn-52221fb8-b33f-4a52-b5c3-e35d1e6fc0e0.scope \
./perf bench sched messaging
# Running 'sched/messaging' benchmark:
# 20 sender and receiver processes per group
# 10 groups == 400 processes runTotal time: 0.174 [sec]
contended total wait max wait avg wait pid comm
4 114.45 us 60.06 us 28.61 us 214847 sched-messaging
2 111.40 us 60.84 us 55.70 us 214848 sched-messaging
2 106.09 us 59.42 us 53.04 us 214837 sched-messaging
1 81.70 us 81.70 us 81.70 us 214709 sched-messaging
68 78.44 us 6.83 us 1.15 us 214633 sched-messaging
69 73.71 us 2.69 us 1.07 us 214632 sched-messaging
4 72.62 us 60.83 us 18.15 us 214850 sched-messaging
2 71.75 us 67.60 us 35.88 us 214840 sched-messaging
2 69.29 us 67.53 us 34.65 us 214804 sched-messaging
2 69.00 us 68.23 us 34.50 us 214826 sched-messaging...
Export cgroup__new() function as it's needed from outside.
--- tools/perf/Documentation/perf-lock.txt | 4 ++ tools/perf/builtin-lock.c | 56 +++++++++++++++++++ tools/perf/util/bpf_lock_contention.c | 15 ++++- .../perf/util/bpf_skel/lock_contention.bpf.c | 17 ++++++ tools/perf/util/cgroup.c | 2 +- tools/perf/util/cgroup.h | 1 + tools/perf/util/lock-contention.h | 2 + 7 files changed, 95 insertions(+), 2 deletions(-)
tools/perf/Documentation/perf-lock.txt +4 -0
@@ -212,6 +212,10 @@ CONTENTION OPTIONS --lock-cgroup:: Show lock contention stat by cgroup. Requires --use-bpf. +-G:: +--cgroup-filter=<value>:: + Show lock contention only in the given cgroups (comma separated list). + SEE ALSO --------
tools/perf/builtin-lock.c +56 -0
80 lines not on this page · raw message
tools/perf/util/bpf_lock_contention.c +14 -1
@@ -21,7 +21,7 @@ static struct lock_contention_bpf *skel;int lock_contention_prepare (struct lock_contention *con) {int i, fd; -int ncpus =1 , ntasks =1 , ntypes =1 , naddrs =1 ; +int ncpus =1 , ntasks =1 , ntypes =1 , naddrs =1 , ncgrps =1 ;struct evlist *evlist = con->evlist;struct target *target = con->target;
@@ -51,6 +51,8 @@ int lock_contention_prepare(struct lock_contention *con) ntasks =perf_thread_map__nr (evlist->core.threads);if (con->filters->nr_types) ntypes = con->filters->nr_types; +if (con->filters->nr_cgrps) + ncgrps = con->filters->nr_cgrps;/* resolve lock name filters to addr */ if (con->filters->nr_syms) {
@@ -85,6 +87,7 @@ int lock_contention_prepare(struct lock_contention *con)bpf_map__set_max_entries (skel->maps.task_filter, ntasks);bpf_map__set_max_entries (skel->maps.type_filter, ntypes);bpf_map__set_max_entries (skel->maps.addr_filter, naddrs); +bpf_map__set_max_entries (skel->maps.cgroup_filter, ncgrps);if (lock_contention_bpf__load (skel) <0 ) {pr_err ("Failed to load lock-contention BPF skeleton\n" );
@@ -146,6 +149,16 @@ int lock_contention_prepare(struct lock_contention *con)bpf_map_update_elem (fd, &con->filters->addrs[i], &val,BPF_ANY ); } +if (con->filters->nr_cgrps) { +u8 val =1 ; + + skel->bss->has_cgroup =1 ; + fd =bpf_map__fd (skel->maps.cgroup_filter); + +for (i =0 ; i < con->filters->nr_cgrps; i++) +bpf_map_update_elem (fd, &con->filters->cgrps[i], &val,BPF_ANY ); + } +/* these don't work well if in the rodata section */ skel->bss->stack_skip = con->stack_skip; skel->bss->aggr_mode = con->aggr_mode;
tools/perf/util/bpf_skel/lock_contention.bpf.c +17 -0
@@ -92,6 +92,13 @@ struct {__uint (max_entries,1 ); } addr_filter SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cgroup_filter SEC(".maps"); +struct rw_semaphore___old {struct task_struct *owner; } __attribute__((preserve_access_index));
@@ -114,6 +121,7 @@ int has_cpu;int has_task;int has_type;int has_addr; +int has_cgroup;int needs_callstack;int stack_skip;int lock_owner;
@@ -194,6 +202,15 @@ static inline int can_record(u64 *ctx)return 0 ; } +if (has_cgroup) { +__u8 *ok; +__u64 cgrp =get_current_cgroup_id (); + + ok =bpf_map_lookup_elem (&cgroup_filter, &cgrp); +if (!ok) +return 0 ; + } +return 1 ; }
tools/perf/util/cgroup.c +1 -1
@@ -114,7 +114,7 @@ static struct cgroup *evlist__find_cgroup(struct evlist *evlist, const char *strreturn NULL ; } -static struct cgroup *cgroup__new (const char *name,bool do_open) +struct cgroup *cgroup__new (const char *name,bool do_open) {struct cgroup *cgroup =zalloc (sizeof (*cgroup));
tools/perf/util/cgroup.h +1 -0
@@ -26,6 +26,7 @@ void cgroup__put(struct cgroup *cgroup);struct evlist ;struct rblist ; +struct cgroup *cgroup__new (const char *name,bool do_open);struct cgroup *evlist__findnew_cgroup (struct evlist *evlist,const char *name);int evlist__expand_cgroup (struct evlist *evlist,const char *cgroups,struct rblist *metric_events,bool open_cgroup);
tools/perf/util/lock-contention.h +2 -0
@@ -9,9 +9,11 @@ struct lock_filter {int nr_types;int nr_addrs;int nr_syms; +int nr_cgrps;unsigned int *types;unsigned long *addrs;char **syms; +u64 *cgrps; };struct lock_stat {
Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Namhyung Kim
maintainerPATCH 5/5 perf test: Improve perf lock contention test
In reply to Namhyung KimAdd cgroup aggregation and filter tests.
$ sudo ./perf test -v contention 84: kernel lock contention analysis test : --- start --- test child forked, pid 222423 Testing perf lock record and perf lock contention Testing perf lock contention --use-bpf Testing perf lock record and perf lock contention at the same time Testing perf lock contention --threads Testing perf lock contention --lock-addr Testing perf lock contention --lock-cgroup Testing perf lock contention --type-filter (w/ spinlock) Testing perf lock contention --lock-filter (w/ tasklist_lock) Testing perf lock contention --callstack-filter (w/ unix_stream) Testing perf lock contention --callstack-filter with task aggregation Testing perf lock contention --cgroup-filter Testing perf lock contention CSV output test child finished with 0 ---- end ---- kernel lock contention analysis test: Ok
--- tools/perf/tests/shell/lock_contention.sh | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+)
tools/perf/tests/shell/lock_contention.sh +45 -0
@@ -123,6 +123,24 @@ test_aggr_addr() fi } +test_aggr_cgroup() +{ + echo "Testing perf lock contention --lock-cgroup" + + if ! perf lock con -b true > /dev/null 2>&1 ; then + echo "[Skip] No BPF support" + return + fi + + # the perf lock contention output goes to the stderr + perf lock con -a -b -g -E 1 -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ "$(cat "${result}" | wc -l)" != "1" ]; then + echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)" + err=1 + exit + fi +} + test_type_filter() { echo "Testing perf lock contention --type-filter (w/ spinlock)"
@@ -232,6 +250,31 @@ test_aggr_task_stack_filter() exit fi } +test_cgroup_filter() +{ + echo "Testing perf lock contention --cgroup-filter" + + if ! perf lock con -b true > /dev/null 2>&1 ; then + echo "[Skip] No BPF support" + return + fi + + perf lock con -a -b -g -E 1 -F wait_total -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ "$(cat "${result}" | wc -l)" != "1" ]; then + echo "[Fail] BPF result should have a cgroup result:" "$(cat "${result}")" + err=1 + exit + fi + + cgroup=$(cat "${result}" | awk '{ print $3 }') + perf lock con -a -b -g -E 1 -G "${cgroup}" -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ "$(cat "${result}" | wc -l)" != "1" ]; then + echo "[Fail] BPF result should have a result with cgroup filter:" "$(cat "${cgroup}")" + err=1 + exit + fi +} + test_csv_output() {
@@ -275,10 +318,12 @@ test_bpf test_record_concurrent test_aggr_task test_aggr_addr +test_aggr_cgroup test_type_filter test_lock_filter test_stack_filter test_aggr_task_stack_filter +test_cgroup_filter test_csv_output exit ${err}
Signature · 1 line
2.42.0.283.g2d96d420d3-goog
Quoted 50 lines · Namhyung Kim
Hello,
The cgroup support comes with two flavors. One is to aggregate the result by cgroups and the other is to filter result for the given cgroups. For now, it only works in BPF mode.
The first one is -g/--lock-cgroup option to show lock stats by cgroups like below. The cgroup names were shortened for brevity:
$ sudo perf lock con -abg perf bench sched messaging contended total wait max wait avg wait cgroup1052 3.34 ms 84.71 us 3.17 us /app-org.gnome.Terminal.slice/vte-spawn-52221fb8-b33f-4a52-b5c3-e35d1e6fc0e0.scope 13 106.60 us 11.48 us 8.20 us /session.slice/[email protected] 12 21.20 us 4.93 us 1.77 us / 3 12.10 us 8.80 us 4.03 us /session-4.scope 2 10.98 us 7.50 us 5.49 us /app-gnome-firefox\x2desr-34054.scope 2 6.04 us 4.88 us 3.02 us /app-gnome-google\x2dchrome-6442.scope 1 5.63 us 5.63 us 5.63 us /app-org.gnome.Terminal.slice/gnome-terminal-server.service 1 3.51 us 3.51 us 3.51 us /pipewire.service 1 2.15 us 2.15 us 2.15 us /pipewire-pulse.service 1 742 ns 742 ns 742 ns /dbus.serviceThe other is -G/--cgroup-filter option to show lock stats only from the given cgroups. It doesn't support cgroup hierarchy and regex matching.
$ sudo perf lock con -abt -G / perf bench sched messaging contended total wait max wait avg wait pid comm2 10.58 us 8.39 us 5.29 us 257552 kworker/4:1 2 9.76 us 7.96 us 4.88 us 0 swapper 4 5.36 us 2.09 us 1.34 us 255462 kworker/0:2 3 3.33 us 1.48 us 1.11 us 257680 kworker/3:1 2 2.59 us 1.46 us 1.29 us 257478 kworker/2:2 1 1.50 us 1.50 us 1.50 us 15 rcu_preemptYou can also use these two options together. :)
The two more test cases were added to the existing lock contention test.
The code is available at 'perf/lock-cgroup-v1' branch in the tree below.
git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.gitThanks,
Namhyung
Quoted 6 lines
Namhyung Kim (5):
perf tools: Addread_all_cgroups()and__cgroup_find()perf lock contention: Prepare to handle cgroups perf lock contention: Add -g/--lock-cgroup option perf lock contention: Add -G/--cgroup-filter option perf test: Improve perf lock contention test
Series:
Thanks,
Ian
Quoted 16 lines
tools/perf/Documentation/perf-lock.txt | 8 ++ tools/perf/builtin-lock.c | 99 ++++++++++++++++++- tools/perf/tests/shell/lock_contention.sh | 45 +++++++++ tools/perf/util/bpf_lock_contention.c | 51 +++++++++- .../perf/util/bpf_skel/lock_contention.bpf.c | 48 ++++++++- tools/perf/util/bpf_skel/lock_data.h | 3 +- tools/perf/util/cgroup.c | 63 ++++++++++-- tools/perf/util/cgroup.h | 5 + tools/perf/util/lock-contention.h | 10 +- 9 files changed, 312 insertions(+), 20 deletions(-)base-commit:
d2045f87154bf67a50ebefe28d2ca0e1e3f8eef1--
2.42.0.283.g2d96d420d3-goog