perf tools: Fix crash with non-jited bpf progs
PATCH 1/1 perf tools: Fix crash with non-jited bpf progs
The addr in PERF_RECORD_KSYMBOL events for non-jited bpf progs points to the bpf interpreter, ie. within kernel text section. When processing the unregister event, this causes unexpected removal of vmlinux_map, crashing perf later in cleanup:
# perf record -- timeout --signal=INT 2s /usr/share/bcc/tools/execsnoop PCOMM PID PPID RET ARGS [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.208 MB perf.data (5155 samples) ] perf: tools/include/linux/refcount.h:131: refcount_sub_and_test: Assertion `!(new > val)' failed. Aborted (core dumped)
# perf script -D|grep KSYM 0 0xa40 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_f958f6eb72ef5af6 0 0xab0 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_8c42dee26e8cd4c2 0 0xb20 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_f958f6eb72ef5af6 108563691893 0x33d98 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3b0 len 0 type 1 flags 0x0 name bpf_prog_bc5697a410556fc2_syscall__execve 108568518458 0x34098 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3f0 len 0 type 1 flags 0x0 name bpf_prog_45e2203c2928704d_do_ret_sys_execve 109301967895 0x34830 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3b0 len 0 type 1 flags 0x1 name bpf_prog_bc5697a410556fc2_syscall__execve 109302007356 0x348b0 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3f0 len 0 type 1 flags 0x1 name bpf_prog_45e2203c2928704d_do_ret_sys_execve perf: tools/include/linux/refcount.h:131: refcount_sub_and_test: Assertion `!(new > val)' failed.
Here the addresses match the bpf interpreter:
# grep -e ffffffffa9b6b530 -e ffffffffa9b6b3b0 -e ffffffffa9b6b3f0 /proc/kallsyms ffffffffa9b6b3b0 t __bpf_prog_run224 ffffffffa9b6b3f0 t __bpf_prog_run192 ffffffffa9b6b530 t __bpf_prog_run32
Fix by not allowing vmlinux_map to be removed by PERF_RECORD_KSYMBOL unregister event.
--- tools/perf/util/machine.c | 11 ++++++++++- tools/perf/util/symbol.c | 7 +++++++ tools/perf/util/symbol.h | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-)
tools/perf/util/machine.c +10 -1
@@ -786,11 +786,20 @@ static int machine__process_ksymbol_unregister(struct machine *machine,union perf_event *event, struct perf_sample *sample __maybe_unused) { + struct symbol *sym;struct map *map; map =maps__find (&machine->kmaps, event->ksymbol.addr); -if (map) +if (!map) +return 0 ; + +if (map != machine->vmlinux_map)maps__remove (&machine->kmaps, map); +else { + sym =dso__find_symbol (map->dso, map->map_ip (map, map->start)); +if (sym) +dso__delete_symbol (map->dso, sym); + }return 0 ; }
tools/perf/util/symbol.c +7 -0
@@ -515,6 +515,13 @@ void dso__insert_symbol(struct dso *dso, struct symbol *sym) } } +void dso__delete_symbol (struct dso *dso,struct symbol *sym) +{ +rb_erase_cached (&sym->rb_node, &dso->symbols); +symbol__delete (sym); +dso__reset_find_symbol_cache (dso); +} +struct symbol *dso__find_symbol (struct dso *dso,u64 addr) {if (dso->last_find_result.addr != addr || dso->last_find_result.symbol ==NULL ) {
tools/perf/util/symbol.h +2 -0
@@ -130,6 +130,8 @@ int dso__load_kallsyms(struct dso *dso, const char *filename, struct map *map);void dso__insert_symbol (struct dso *dso,struct symbol *sym); +void dso__delete_symbol (struct dso *dso, +struct symbol *sym);struct symbol *dso__find_symbol (struct dso *dso,u64 addr);struct symbol *dso__find_symbol_by_name (struct dso *dso,const char *name);
Signature · 1 line
2.26.2
Quoted 29 lines · Tommi Rantala
The addr in
PERF_RECORD_KSYMBOLevents for non-jited bpf progs points to the bpf interpreter, ie. within kernel text section. When processing the unregister event, this causes unexpected removal ofvmlinux_map, crashing perf later in cleanup:# perf record -- timeout --signal=INT 2s /usr/share/bcc/tools/execsnoop PCOMM PID PPID RET ARGS [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.208 MB perf.data (5155 samples) ] perf: tools/include/linux/refcount.h:131: refcount_sub_and_test: Assertion `!(new > val)' failed. Aborted (core dumped)# perf script -D|grep KSYM 0 0xa40 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_f958f6eb72ef5af6 0 0xab0 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_8c42dee26e8cd4c2 0 0xb20 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_f958f6eb72ef5af6 108563691893 0x33d98 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3b0 len 0 type 1 flags 0x0 name bpf_prog_bc5697a410556fc2_syscall__execve 108568518458 0x34098 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3f0 len 0 type 1 flags 0x0 name bpf_prog_45e2203c2928704d_do_ret_sys_execve 109301967895 0x34830 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3b0 len 0 type 1 flags 0x1 name bpf_prog_bc5697a410556fc2_syscall__execve 109302007356 0x348b0 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3f0 len 0 type 1 flags 0x1 name bpf_prog_45e2203c2928704d_do_ret_sys_execve perf: tools/include/linux/refcount.h:131: refcount_sub_and_test: Assertion `!(new > val)' failed.Here the addresses match the bpf interpreter:
# grep -e ffffffffa9b6b530 -e ffffffffa9b6b3b0 -e ffffffffa9b6b3f0 /proc/kallsyms ffffffffa9b6b3b0 t __bpf_prog_run224 ffffffffa9b6b3f0 t __bpf_prog_run192 ffffffffa9b6b530 t __bpf_prog_run32
Quoted 4 lines
Fix by not allowing
vmlinux_mapto be removed byPERF_RECORD_KSYMBOLunregister event.Signed-off-by Tommi Rantala <[email protected]>
nice, I almost forgot about non jit mode by now ;-)
Acked/Tested-by: Jiri Olsa <[email protected]>
thanks,
jirka
Quoted 66 lines
--- tools/perf/util/machine.c | 11 ++++++++++- tools/perf/util/symbol.c | 7 +++++++ tools/perf/util/symbol.h | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-)tools/perf/util/machine.c +10
-1@@ -786,11 +786,20 @@ static int machine__process_ksymbol_unregister(struct machine *machine,union perf_event *event, struct perf_sample *sample __maybe_unused) { + struct symbol *sym;struct map *map; map =maps__find (&machine->kmaps, event->ksymbol.addr); -if (map) +if (!map) +return 0 ; + +if (map != machine->vmlinux_map)maps__remove (&machine->kmaps, map); +else { + sym =dso__find_symbol (map->dso, map->map_ip (map, map->start)); +if (sym) +dso__delete_symbol (map->dso, sym); + }return 0 ; }tools/perf/util/symbol.c +7
-0@@ -515,6 +515,13 @@ void dso__insert_symbol(struct dso *dso, struct symbol *sym) } } +void dso__delete_symbol (struct dso *dso,struct symbol *sym) +{ +rb_erase_cached (&sym->rb_node, &dso->symbols); +symbol__delete (sym); +dso__reset_find_symbol_cache (dso); +} +struct symbol *dso__find_symbol (struct dso *dso,u64 addr) {if (dso->last_find_result.addr != addr || dso->last_find_result.symbol ==NULL ) {tools/perf/util/symbol.h +2
-0@@ -130,6 +130,8 @@ int dso__load_kallsyms(struct dso *dso, const char *filename, struct map *map);void dso__insert_symbol (struct dso *dso,struct symbol *sym); +void dso__delete_symbol (struct dso *dso, +struct symbol *sym);struct symbol *dso__find_symbol (struct dso *dso,u64 addr);struct symbol *dso__find_symbol_by_name (struct dso *dso,const char *name);Signature · 1 line
2.26.2
Em Mon, Oct 19, 2020 at 10:53:43PM +0200, Jiri Olsa escreveu:
Quoted 30 lines · Tommi Rantala
Quoted 29 lines
The addr in
PERF_RECORD_KSYMBOLevents for non-jited bpf progs points to the bpf interpreter, ie. within kernel text section. When processing the unregister event, this causes unexpected removal ofvmlinux_map, crashing perf later in cleanup:# perf record -- timeout --signal=INT 2s /usr/share/bcc/tools/execsnoop PCOMM PID PPID RET ARGS [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.208 MB perf.data (5155 samples) ] perf: tools/include/linux/refcount.h:131: refcount_sub_and_test: Assertion `!(new > val)' failed. Aborted (core dumped)# perf script -D|grep KSYM 0 0xa40 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_f958f6eb72ef5af6 0 0xab0 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_8c42dee26e8cd4c2 0 0xb20 [0x48]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b530 len 0 type 1 flags 0x0 name bpf_prog_f958f6eb72ef5af6 108563691893 0x33d98 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3b0 len 0 type 1 flags 0x0 name bpf_prog_bc5697a410556fc2_syscall__execve 108568518458 0x34098 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3f0 len 0 type 1 flags 0x0 name bpf_prog_45e2203c2928704d_do_ret_sys_execve 109301967895 0x34830 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3b0 len 0 type 1 flags 0x1 name bpf_prog_bc5697a410556fc2_syscall__execve 109302007356 0x348b0 [0x58]: PERF_RECORD_KSYMBOL addr ffffffffa9b6b3f0 len 0 type 1 flags 0x1 name bpf_prog_45e2203c2928704d_do_ret_sys_execve perf: tools/include/linux/refcount.h:131: refcount_sub_and_test: Assertion `!(new > val)' failed.Here the addresses match the bpf interpreter:
# grep -e ffffffffa9b6b530 -e ffffffffa9b6b3b0 -e ffffffffa9b6b3f0 /proc/kallsyms ffffffffa9b6b3b0 t __bpf_prog_run224 ffffffffa9b6b3f0 t __bpf_prog_run192 ffffffffa9b6b530 t __bpf_prog_run32
Quoted 8 lines
Quoted 4 lines
Fix by not allowing
vmlinux_mapto be removed byPERF_RECORD_KSYMBOLunregister event.Signed-off-by Tommi Rantala <[email protected]>nice, I almost forgot about non jit mode by now ;-)
Acked/Tested-by: Jiri Olsa <[email protected]>
Thanks, applied.
- Arnaldo
Quoted 70 lines
thanks,
jirkaQuoted 66 lines
--- tools/perf/util/machine.c | 11 ++++++++++- tools/perf/util/symbol.c | 7 +++++++ tools/perf/util/symbol.h | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-)tools/perf/util/machine.c +10
-1@@ -786,11 +786,20 @@ static int machine__process_ksymbol_unregister(struct machine *machine,union perf_event *event, struct perf_sample *sample __maybe_unused) { + struct symbol *sym;struct map *map; map =maps__find (&machine->kmaps, event->ksymbol.addr); -if (map) +if (!map) +return 0 ; + +if (map != machine->vmlinux_map)maps__remove (&machine->kmaps, map); +else { + sym =dso__find_symbol (map->dso, map->map_ip (map, map->start)); +if (sym) +dso__delete_symbol (map->dso, sym); + }return 0 ; }tools/perf/util/symbol.c +7
-0@@ -515,6 +515,13 @@ void dso__insert_symbol(struct dso *dso, struct symbol *sym) } } +void dso__delete_symbol (struct dso *dso,struct symbol *sym) +{ +rb_erase_cached (&sym->rb_node, &dso->symbols); +symbol__delete (sym); +dso__reset_find_symbol_cache (dso); +} +struct symbol *dso__find_symbol (struct dso *dso,u64 addr) {if (dso->last_find_result.addr != addr || dso->last_find_result.symbol ==NULL ) {tools/perf/util/symbol.h +2
-0@@ -130,6 +130,8 @@ int dso__load_kallsyms(struct dso *dso, const char *filename, struct map *map);void dso__insert_symbol (struct dso *dso,struct symbol *sym); +void dso__delete_symbol (struct dso *dso, +struct symbol *sym);struct symbol *dso__find_symbol (struct dso *dso,u64 addr);struct symbol *dso__find_symbol_by_name (struct dso *dso,const char *name);Signature · 1 line
2.26.2
Signature · 1 line
- Arnaldo