perf: fix segfault in `lock contention -b`
PATCH 1/1 perf: fix segfault in `lock contention -b`
From: "Tycho Andersen (AMD)" <[email protected]>
When run on a kernel without BTF info, I get:
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', was CONFIG_DEBUG_INFO_BTF enabled?
libbpf: failed to find valid kernel BTF
Program received signal SIGSEGV, Segmentation fault.
0x00005555556915b7 in btf.type_cnt ()
(gdb) bt
#0 0x00005555556915b7 in btf.type_cnt ()
#1 0x0000555555691fbc in btf_find_by_name_kind ()
#2 0x00005555556920d0 in btf.find_by_name_kind ()
#3 0x00005555558a1b7c in init_numa_data (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:125
#4 0x00005555558a264b in lock_contention_prepare (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:313
#5 0x0000555555620702 in __cmd_contention (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2084
#6 0x0000555555622c8d in cmd_lock (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2755
#7 0x0000555555651451 in run_builtin (p=0x555556104f00 <commands+576>, argc=3, argv=0x7fffffffea10)
at perf.c:349
#8 0x00005555556516ed in handle_internal_command (argc=3, argv=0x7fffffffea10) at perf.c:401
#9 0x000055555565184e in run_argv (argcp=0x7fffffffe7fc, argv=0x7fffffffe7f0) at perf.c:445
#10 0x0000555555651b9f in main (argc=3, argv=0x7fffffffea10) at perf.c:553If we really are running -b without BTF info, the error is fatal, so let's propagate it and exit accordingly.
--- tools/perf/util/bpf_lock_contention.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
tools/perf/util/bpf_lock_contention.c +6 -4
@@ -34,7 +34,7 @@ static bool slab_cache_equal(long key1, long key2, void *ctx __maybe_unused)return key1 == key2; } -static void check_slab_cache_iter (struct lock_contention *con) +static int check_slab_cache_iter (struct lock_contention *con) {s32 ret;
@@ -43,19 +43,20 @@ static void check_slab_cache_iter(struct lock_contention *con) con->btf =btf__load_vmlinux_btf ();if (con->btf ==NULL ) {pr_debug ("BTF loading failed: %s\n" ,strerror (errno)); -return ; +return -errno; } ret =btf__find_by_name_kind (con->btf,"bpf_iter__kmem_cache" ,BTF_KIND_STRUCT );if (ret <0 ) {bpf_program__set_autoload (skel->progs.slab_cache_iter, false);pr_debug ("slab cache iterator is not available: %d\n" , ret); -return ; +return ret; } has_slab_iter = true;bpf_map__set_max_entries (skel->maps.slab_caches, con->map_nr_entries); +return 0 ; }static void run_slab_cache_iter (void )
@@ -301,7 +302,8 @@ int lock_contention_prepare(struct lock_contention *con) skel->rodata->use_cgroup_v2 =1 ; } -check_slab_cache_iter (con); +if (check_slab_cache_iter (con) <0 ) +return -1 ;if (con->filters->nr_slabs && has_slab_iter) { skel->rodata->has_slab =1 ;
base-commit: 5572ad8fddecd4a0db19801262072ff5916b7589
Signature · 1 line
2.52.0
Namhyung Kim
maintainerQuoted 23 lines · Tycho Andersen
From: "Tycho Andersen (AMD)" <[email protected]>
When run on a kernel without BTF info, I get:
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', was CONFIG_DEBUG_INFO_BTF enabled? libbpf: failed to find valid kernel BTF Program received signal SIGSEGV, Segmentation fault. 0x00005555556915b7 in btf.type_cnt () (gdb) bt #0 0x00005555556915b7 in btf.type_cnt () #1 0x0000555555691fbc in btf_find_by_name_kind () #2 0x00005555556920d0 in btf.find_by_name_kind () #3 0x00005555558a1b7c in init_numa_data (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:125 #4 0x00005555558a264b in lock_contention_prepare (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:313 #5 0x0000555555620702 in __cmd_contention (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2084 #6 0x0000555555622c8d in cmd_lock (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2755 #7 0x0000555555651451 in run_builtin (p=0x555556104f00 <commands+576>, argc=3, argv=0x7fffffffea10) at perf.c:349 #8 0x00005555556516ed in handle_internal_command (argc=3, argv=0x7fffffffea10) at perf.c:401 #9 0x000055555565184e in run_argv (argcp=0x7fffffffe7fc, argv=0x7fffffffe7f0) at perf.c:445 #10 0x0000555555651b9f in main (argc=3, argv=0x7fffffffea10) at perf.c:553
Quoted 2 lines
If we really are running -b without BTF info, the error is fatal, so let's propagate it and exit accordingly.
Thanks for the patch.
I'm curious how the kernel will react if we want to load the BPF. Probably the verifier will reject as it calls some helpers like bpf_get_current_task_btf().
I think it used to work without BTF - the callstack (default) and task output mode won't need it. It'd be nice if we could run it when the kernel doesn't have BTF. But maybe it's out of scope of this patch.
That said, can you add a NULL check in the init_numa_data() instead and see how far it can go?
Thanks,
Namhyung
Quoted 59 lines
Signed-off-by Tycho Andersen (AMD) <[email protected]>CC Ravi Bangoria <[email protected]>CC K Prateek Nayak <[email protected]>--- tools/perf/util/bpf_lock_contention.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)tools/perf/util/bpf_lock_contention.c +6
-4@@ -34,7 +34,7 @@ static bool slab_cache_equal(long key1, long key2, void *ctx __maybe_unused)return key1 == key2; } -static void check_slab_cache_iter (struct lock_contention *con) +static int check_slab_cache_iter (struct lock_contention *con) {s32 ret;@@ -43,19 +43,20 @@ static void check_slab_cache_iter(struct lock_contention *con) con->btf =btf__load_vmlinux_btf ();if (con->btf ==NULL ) {pr_debug ("BTF loading failed: %s\n" ,strerror (errno)); -return ; +return -errno; } ret =btf__find_by_name_kind (con->btf,"bpf_iter__kmem_cache" ,BTF_KIND_STRUCT );if (ret <0 ) {bpf_program__set_autoload (skel->progs.slab_cache_iter, false);pr_debug ("slab cache iterator is not available: %d\n" , ret); -return ; +return ret; } has_slab_iter = true;bpf_map__set_max_entries (skel->maps.slab_caches, con->map_nr_entries); +return 0 ; }static void run_slab_cache_iter (void )@@ -301,7 +302,8 @@ int lock_contention_prepare(struct lock_contention *con) skel->rodata->use_cgroup_v2 =1 ; } -check_slab_cache_iter (con); +if (check_slab_cache_iter (con) <0 ) +return -1 ;if (con->filters->nr_slabs && has_slab_iter) { skel->rodata->has_slab =1 ;base-commit:
5572ad8fddecd4a0db19801262072ff5916b7589Signature · 1 line
2.52.0
Quoted 33 lines · Namhyung Kim
Quoted 23 lines · Tycho Andersen
From: "Tycho Andersen (AMD)" <[email protected]>
When run on a kernel without BTF info, I get:
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', was CONFIG_DEBUG_INFO_BTF enabled? libbpf: failed to find valid kernel BTF Program received signal SIGSEGV, Segmentation fault. 0x00005555556915b7 in btf.type_cnt () (gdb) bt #0 0x00005555556915b7 in btf.type_cnt () #1 0x0000555555691fbc in btf_find_by_name_kind () #2 0x00005555556920d0 in btf.find_by_name_kind () #3 0x00005555558a1b7c in init_numa_data (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:125 #4 0x00005555558a264b in lock_contention_prepare (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:313 #5 0x0000555555620702 in __cmd_contention (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2084 #6 0x0000555555622c8d in cmd_lock (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2755 #7 0x0000555555651451 in run_builtin (p=0x555556104f00 <commands+576>, argc=3, argv=0x7fffffffea10) at perf.c:349 #8 0x00005555556516ed in handle_internal_command (argc=3, argv=0x7fffffffea10) at perf.c:401 #9 0x000055555565184e in run_argv (argcp=0x7fffffffe7fc, argv=0x7fffffffe7f0) at perf.c:445 #10 0x0000555555651b9f in main (argc=3, argv=0x7fffffffea10) at perf.c:553Quoted 2 lines
If we really are running -b without BTF info, the error is fatal, so let's propagate it and exit accordingly.
Thanks for the patch.
I'm curious how the kernel will react if we want to load the BPF. Probably the verifier will reject as it calls some helpers like
bpf_get_current_task_btf().
Quoted 6 lines
I think it used to work without BTF - the callstack (default) and task output mode won't need it. It'd be nice if we could run it when the kernel doesn't have BTF. But maybe it's out of scope of this patch.
That said, can you add a NULL check in the
init_numa_data()instead and see how far it can go?
Looks like not much further, it fails in lock_contention_bpf__load():
./perf lock contention -b
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', was CONFIG_DEBUG_INFO_BTF enabled?
libbpf: failed to find valid kernel BTF
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', was CONFIG_DEBUG_INFO_BTF enabled?
libbpf: failed to find valid kernel BTF
libbpf: Error loading vmlinux BTF: -ESRCH
libbpf: failed to load BPF skeleton 'lock_contention_bpf': -ESRCH
Failed to load lock-contention BPF skeleton
lock contention BPF setup failed
Tycho
Namhyung Kim
maintainerQuoted 43 lines · Tycho Andersen
Quoted 33 lines · Namhyung Kim
Quoted 23 lines · Tycho Andersen
From: "Tycho Andersen (AMD)" <[email protected]>
When run on a kernel without BTF info, I get:
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', was CONFIG_DEBUG_INFO_BTF enabled? libbpf: failed to find valid kernel BTF Program received signal SIGSEGV, Segmentation fault. 0x00005555556915b7 in btf.type_cnt () (gdb) bt #0 0x00005555556915b7 in btf.type_cnt () #1 0x0000555555691fbc in btf_find_by_name_kind () #2 0x00005555556920d0 in btf.find_by_name_kind () #3 0x00005555558a1b7c in init_numa_data (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:125 #4 0x00005555558a264b in lock_contention_prepare (con=0x7fffffffd0a0) at util/bpf_lock_contention.c:313 #5 0x0000555555620702 in __cmd_contention (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2084 #6 0x0000555555622c8d in cmd_lock (argc=0, argv=0x7fffffffea10) at builtin-lock.c:2755 #7 0x0000555555651451 in run_builtin (p=0x555556104f00 <commands+576>, argc=3, argv=0x7fffffffea10) at perf.c:349 #8 0x00005555556516ed in handle_internal_command (argc=3, argv=0x7fffffffea10) at perf.c:401 #9 0x000055555565184e in run_argv (argcp=0x7fffffffe7fc, argv=0x7fffffffe7f0) at perf.c:445 #10 0x0000555555651b9f in main (argc=3, argv=0x7fffffffea10) at perf.c:553Quoted 2 lines
If we really are running -b without BTF info, the error is fatal, so let's propagate it and exit accordingly.
Thanks for the patch.
I'm curious how the kernel will react if we want to load the BPF. Probably the verifier will reject as it calls some helpers like
bpf_get_current_task_btf().Quoted 6 lines
I think it used to work without BTF - the callstack (default) and task output mode won't need it. It'd be nice if we could run it when the kernel doesn't have BTF. But maybe it's out of scope of this patch.
That said, can you add a NULL check in the
init_numa_data()instead and see how far it can go?Looks like not much further, it fails in
lock_contention_bpf__load():
Quoted 9 lines
./perf lock contention -b
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', wasCONFIG_DEBUG_INFO_BTFenabled?
libbpf: failed to find valid kernel BTF
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', wasCONFIG_DEBUG_INFO_BTFenabled?
libbpf: failed to find valid kernel BTF
libbpf: Error loading vmlinux BTF:-ESRCH
libbpf: failed to load BPF skeleton 'lock_contention_bpf':-ESRCH
Failed to load lock-contention BPF skeleton
lock contention BPF setup failed
Thanks for checking this. I think it's ok to fail like this when BTF is not enabled. Let me see if I can make some changes to run without it later.
Thanks,
Namhyung
Quoted 17 lines · Namhyung Kim
Quoted 6 lines · Tycho Andersen
Quoted 2 lines · Namhyung Kim
That said, can you add a NULL check in the
init_numa_data()instead and see how far it can go?Looks like not much further, it fails in
lock_contention_bpf__load():Quoted 9 lines
./perf lock contention -b
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', wasCONFIG_DEBUG_INFO_BTFenabled?
libbpf: failed to find valid kernel BTF
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', wasCONFIG_DEBUG_INFO_BTFenabled?
libbpf: failed to find valid kernel BTF
libbpf: Error loading vmlinux BTF:-ESRCH
libbpf: failed to load BPF skeleton 'lock_contention_bpf':-ESRCH
Failed to load lock-contention BPF skeleton
lock contention BPF setup failed
Quoted 3 lines
Thanks for checking this. I think it's ok to fail like this when BTF is not enabled. Let me see if I can make some changes to run without it later.
Gentle ping, I was just reminded of this crash again this morning.
Tycho
Namhyung Kim
maintainerHello,
Quoted 18 lines · Tycho Andersen
Quoted 17 lines · Namhyung Kim
Quoted 15 lines · Tycho Andersen
Quoted 2 lines · Namhyung Kim
That said, can you add a NULL check in the
init_numa_data()instead and see how far it can go?Looks like not much further, it fails in
lock_contention_bpf__load():./perf lock contention -b
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', wasCONFIG_DEBUG_INFO_BTFenabled?
libbpf: failed to find valid kernel BTF
libbpf: kernel BTF is missing at '/sys/kernel/btf/vmlinux', wasCONFIG_DEBUG_INFO_BTFenabled?
libbpf: failed to find valid kernel BTF
libbpf: Error loading vmlinux BTF:-ESRCH
libbpf: failed to load BPF skeleton 'lock_contention_bpf':-ESRCH
Failed to load lock-contention BPF skeleton
lock contention BPF setup failed
Quoted 5 lines
Quoted 3 lines
Thanks for checking this. I think it's ok to fail like this when BTF is not enabled. Let me see if I can make some changes to run without it later.
Gentle ping, I was just reminded of this crash again this morning.
Sorry for the delay. Please send a fix with the NULL check in the init_numa_data(). I think it'd be simpler and easier to improve.
Thanks,
Namhyung
Hello Tycho,
Quoted 24 lines · Tycho Andersen
tools/perf/util/bpf_lock_contention.c +3
-2@@ -43,19 +43,20 @@ static void check_slab_cache_iter(struct lock_contention *con) con->btf =btf__load_vmlinux_btf ();if (con->btf ==NULL ) {pr_debug ("BTF loading failed: %s\n" ,strerror (errno)); -return ; +return -errno; } ret =btf__find_by_name_kind (con->btf,"bpf_iter__kmem_cache" ,BTF_KIND_STRUCT );if (ret <0 ) {bpf_program__set_autoload (skel->progs.slab_cache_iter, false);pr_debug ("slab cache iterator is not available: %d\n" , ret); -return ; +return ret; } has_slab_iter = true;bpf_map__set_max_entries (skel->maps.slab_caches, con->map_nr_entries); +return 0 ; }static void run_slab_cache_iter (void )
tools/perf/util/bpf_lock_contention.c int lock_contention_prepare(struct lock_contention *con) · line 308
Quoted 8 lines
tools/perf/util/bpf_lock_contention.c +2
-1@@ -301,7 +302,8 @@ int lock_contention_prepare(struct lock_contention *con) skel->rodata->use_cgroup_v2 =1 ; } -check_slab_cache_iter (con); +if (check_slab_cache_iter (con) <0 ) +return -1 ;if (con->filters->nr_slabs && has_slab_iter) {
nit.
Now that lock_contention_prepare() bails out when check_slab_cache_iter() fails, we no longer need the "has_slab_iter" global indicator.
Both this, and run_slab_cache_iter() that depend on "has_slab_iter" will only run if check_slab_cache_iter() succeeded and the global indicator will be true in both those cases.
Can be separate follow-up cleanup.
Quoted 3 lines
tools/perf/util/bpf_lock_contention.c +0
-0skel->rodata->has_slab =1 ;base-commit:
5572ad8fddecd4a0db19801262072ff5916b7589
Signature · 2 lines
Thanks and Regards, Prateek
Quoted 42 lines · K Prateek Nayak
Hello Tycho,
Quoted 24 lines · Tycho Andersen
tools/perf/util/bpf_lock_contention.c +3
-2@@ -43,19 +43,20 @@ static void check_slab_cache_iter(struct lock_contention *con) con->btf =btf__load_vmlinux_btf ();if (con->btf ==NULL ) {pr_debug ("BTF loading failed: %s\n" ,strerror (errno)); -return ; +return -errno; } ret =btf__find_by_name_kind (con->btf,"bpf_iter__kmem_cache" ,BTF_KIND_STRUCT );if (ret <0 ) {bpf_program__set_autoload (skel->progs.slab_cache_iter, false);pr_debug ("slab cache iterator is not available: %d\n" , ret); -return ; +return ret; } has_slab_iter = true;bpf_map__set_max_entries (skel->maps.slab_caches, con->map_nr_entries); +return 0 ; }static void run_slab_cache_iter (void )tools/perf/util/bpf_lock_contention.c int lock_contention_prepare(struct lock_contention *con) · line 308
Quoted 8 lines
tools/perf/util/bpf_lock_contention.c +2
-1@@ -301,7 +302,8 @@ int lock_contention_prepare(struct lock_contention *con) skel->rodata->use_cgroup_v2 =1 ; } -check_slab_cache_iter (con); +if (check_slab_cache_iter (con) <0 ) +return -1 ;if (con->filters->nr_slabs && has_slab_iter) {nit.
Now that
lock_contention_prepare()bails out whencheck_slab_cache_iter()fails, we no longer need the "has_slab_iter" global indicator.
Quoted 5 lines
Both this, and
run_slab_cache_iter()that depend on "has_slab_iter" will only run ifcheck_slab_cache_iter()succeeded and the global indicator will be true in both those cases.Can be separate follow-up cleanup.
I guess this depends on whether it should work without BTF. Presumably we'd want to keep this check if the goal is to make it work without BTF.
I'm can send an addition cleanup patch if we decide not to go that route.
Tycho