bpf: Speed up symbol resolving in kprobe multi link
Subsystems (9)
Jiri Olsa
reviewerv3 cover bpf: Speed up symbol resolving in kprobe multi link
hi,
sending additional fix for symbol resolving in kprobe multi link requested by Alexei and Andrii [1].
This speeds up bpftrace kprobe attachment, when using pure symbols (3344 symbols) to attach:
Before:
# perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* { } i:ms:1 { exit(); }'
...
6.5681 +- 0.0225 seconds time elapsed ( +- 0.34% )After:
# perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* { } i:ms:1 { exit(); }'
...
0.5661 +- 0.0275 seconds time elapsed ( +- 4.85% )v3 changes:
- renamed kallsyms_lookup_names to ftrace_lookup_symbols and moved it to ftrace.c [Masami]
- added ack [Andrii]
- couple small test fixes [Andrii]
v2 changes (first version [2]):
- removed the 2 seconds check [Alexei]
- moving/forcing symbols sorting out of kallsyms_lookup_names function [Alexei]
- skipping one array allocation and copy_from_user [Andrii]
- several small fixes [Masami,Andrii]
- build fix [kernel test robot]
thanks,
jirka
[1] https://lore.kernel.org/bpf/CAEf4BzZtQaiUxQ-sm_hH2qKPRaqGHyOfEsW96DxtBHRaKLoL3Q@mail.gmail.com/
[2] https://lore.kernel.org/bpf/[email protected]/
---
Jiri Olsa (5):
kallsyms: Fully export kallsyms_on_each_symbol function
ftrace: Add ftrace_lookup_symbols function
fprobe: Resolve symbols with ftrace_lookup_symbols
bpf: Resolve symbols with ftrace_lookup_symbols for kprobe multi link selftests/bpf: Add attach bench test
include/linux/ftrace.h | 6 ++++++ include/linux/kallsyms.h | 5 +++++ kernel/kallsyms.c | 3 +-- kernel/trace/bpf_trace.c | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------- kernel/trace/fprobe.c | 32 ++++++++++++-------------------- kernel/trace/ftrace.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/testing/selftests/bpf/progs/kprobe_multi_empty.c | 12 ++++++++++++ 8 files changed, 297 insertions(+), 68 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/kprobe_multi_empty.c
Jiri Olsa
reviewerPATCH v3 1/5 kallsyms: Fully export kallsyms_on_each_symbol function
Fully exporting kallsyms_on_each_symbol function, so it can be used in following changes.
Rather than adding another ifdef option let's export the function completely (when CONFIG_KALLSYMS option is defined).
--- include/linux/kallsyms.h | 5 +++++ kernel/kallsyms.c | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-)
include/linux/kallsyms.h +5 -0
@@ -163,6 +163,11 @@ static inline bool kallsyms_show_value(const struct cred *cred) return false; } +static inline int kallsyms_on_each_symbol (int (*fn)(void *,const char *,struct module *,unsigned long ), +void *data) +{ +return -EOPNOTSUPP ; +}#endif /*CONFIG_KALLSYMS*/ static inline void print_ip_sym (const char *loglvl,unsigned long ip)
kernel/kallsyms.c +0 -2
@@ -228,7 +228,6 @@ unsigned long kallsyms_lookup_name(const char *name)return module_kallsyms_lookup_name (name); } -#ifdef CONFIG_LIVEPATCH /* * Iterate over all symbols in vmlinux. For symbols from modules use * module_kallsyms_on_each_symbol instead.
@@ -251,7 +250,6 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, }return 0 ; } -#endif /* CONFIG_LIVEPATCH */ static unsigned long get_symbol_pos(unsigned long addr, unsigned long *symbolsize,
Signature · 1 line
2.35.1
Re: [PATCHv3 bpf-next 1/5] kallsyms: Fully export kallsyms_on_each_symbol function
Quoted 52 lines · Jiri Olsa
Fully exporting
kallsyms_on_each_symbolfunction, so it can be used in following changes.Rather than adding another ifdef option let's export the function completely (when
CONFIG_KALLSYMSoption is defined).Cc Christoph Hellwig <[email protected]>Signed-off-by Jiri Olsa <[email protected]>--- include/linux/kallsyms.h | 5 +++++ kernel/kallsyms.c | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-)include/linux/kallsyms.h +5
-0@@ -163,6 +163,11 @@ static inline bool kallsyms_show_value(const struct cred *cred) return false; } +static inline int kallsyms_on_each_symbol (int (*fn)(void *,const char *,struct module *,unsigned long ), +void *data) +{ +return -EOPNOTSUPP ; +}#endif /*CONFIG_KALLSYMS*/ static inline void print_ip_sym (const char *loglvl,unsigned long ip)kernel/kallsyms.c +0
-2@@ -228,7 +228,6 @@ unsigned long kallsyms_lookup_name(const char *name)return module_kallsyms_lookup_name (name); } -#ifdef CONFIG_LIVEPATCH /* * Iterate over all symbols in vmlinux. For symbols from modules use * module_kallsyms_on_each_symbol instead. @@ -251,7 +250,6 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, }return 0 ; } -#endif /* CONFIG_LIVEPATCH */ static unsigned long get_symbol_pos(unsigned long addr, unsigned long *symbolsize,Signature · 1 line
2.35.1
got this one wrong, it needs one more change
I'll send new version
jirka
---
include/linux/kallsyms.h +6 -1
@@ -65,11 +65,11 @@ static inline void *dereference_symbol_descriptor(void *ptr)return ptr; } +#ifdef CONFIG_KALLSYMS int kallsyms_on_each_symbol (int (*fn)(void *,const char *,struct module *,unsigned long ),void *data); -#ifdef CONFIG_KALLSYMS /* Lookup the address for a symbol. Returns 0 if not found. */ unsigned long kallsyms_lookup_name (const char *name);
@@ -163,6 +163,11 @@ static inline bool kallsyms_show_value(const struct cred *cred) return false; } +static inline int kallsyms_on_each_symbol (int (*fn)(void *,const char *,struct module *,unsigned long ), +void *data) +{ +return -EOPNOTSUPP ; +}#endif /*CONFIG_KALLSYMS*/ static inline void print_ip_sym (const char *loglvl,unsigned long ip)
kernel/kallsyms.c +0 -2
@@ -228,7 +228,6 @@ unsigned long kallsyms_lookup_name(const char *name)return module_kallsyms_lookup_name (name); } -#ifdef CONFIG_LIVEPATCH /* * Iterate over all symbols in vmlinux. For symbols from modules use * module_kallsyms_on_each_symbol instead.
@@ -251,7 +250,6 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, }return 0 ; } -#endif /* CONFIG_LIVEPATCH */ static unsigned long get_symbol_pos(unsigned long addr, unsigned long *symbolsize,
Jiri Olsa
reviewerPATCH v3 2/5 ftrace: Add ftrace_lookup_symbols function
In reply to Jiri OlsaAdding ftrace_lookup_symbols function that resolves array of symbols with single pass over kallsyms.
The user provides array of string pointers with count and pointer to allocated array for resolved values.
int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt,
unsigned long *addrs)
It iterates all kalsyms symbols and tries to loop up each in provided symbols array with bsearch. The symbols array needs to be sorted by name for this reason.
We also check each symbol to pass ftrace_location, because this API will be used for fprobe symbols resolving.
--- include/linux/ftrace.h | 6 ++++ kernel/kallsyms.c | 1 + kernel/trace/ftrace.c | 62 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+)
include/linux/ftrace.h +6 -0
@@ -303,6 +303,8 @@ int unregister_ftrace_function(struct ftrace_ops *ops);extern void ftrace_stub (unsigned long a0,unsigned long a1,struct ftrace_ops *op,struct ftrace_regs *fregs); + +int ftrace_lookup_symbols (const char **sorted_syms,size_t cnt,unsigned long *addrs);#else /* !CONFIG_FUNCTION_TRACER */ /* * (un)register_ftrace_function must be a macro since the ops parameter
@@ -313,6 +315,10 @@ extern void ftrace_stub(unsigned long a0, unsigned long a1,static inline void ftrace_kill (void ) { }static inline void ftrace_free_init_mem (void ) { }static inline void ftrace_free_mem (struct module *mod,void *start,void *end) { } +static inline int ftrace_lookup_symbols (const char **sorted_syms,size_t cnt,unsigned long *addrs) +{ +return -EOPNOTSUPP ; +}#endif /* CONFIG_FUNCTION_TRACER */ struct ftrace_func_entry {
kernel/kallsyms.c +1 -0
@@ -29,6 +29,7 @@#include <linux/compiler.h> #include <linux/module.h> #include <linux/kernel.h> +#include <linux/bsearch.h> /* * These will be re-linked against their real values
kernel/trace/ftrace.c +62 -0
65 lines not on this page · raw message
Signature · 1 line
2.35.1
Re: [PATCHv3 bpf-next 2/5] ftrace: Add ftrace_lookup_symbols function
Quoted 11 lines · Jiri Olsa
Adding
ftrace_lookup_symbolsfunction that resolves array of symbols with single pass over kallsyms.The user provides array of string pointers with count and pointer to allocated array for resolved values.
int ftrace_lookup_symbols(const char **sorted_syms,
size_tcnt,
unsigned long *addrs)It iterates all kalsyms symbols and tries to loop up each in provided
typo: kallsyms
Quoted 9 lines
symbols array with bsearch. The symbols array needs to be sorted by name for this reason.
We also check each symbol to pass
ftrace_location, because this API will be used for fprobe symbols resolving.Suggested-by Andrii Nakryiko <[email protected]>Signed-off-by Jiri Olsa <[email protected]>---
LGTM.
Quoted 5 lines
include/linux/ftrace.h | 6 ++++ kernel/kallsyms.c | 1 + kernel/trace/ftrace.c | 62 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+)
[...]
Jiri Olsa
reviewerPATCH v3 3/5 fprobe: Resolve symbols with ftrace_lookup_symbols
In reply to Jiri OlsaUsing ftrace_lookup_symbols to speed up symbols lookup in register_fprobe_syms API.
This requires syms array to be alphabetically sorted.
--- kernel/trace/fprobe.c | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-)
kernel/trace/fprobe.c +12 -20
@@ -85,39 +85,31 @@ static void fprobe_exit_handler(struct rethook_node *rh, void *data, }NOKPROBE_SYMBOL (fprobe_exit_handler); +static int symbols_cmp (const void *a,const void *b) +{ +const char **str_a = (const char **) a; +const char **str_b = (const char **) b; + +return strcmp (*str_a, *str_b); +} +/* Convert ftrace location address from symbols */ static unsigned long *get_ftrace_locations (const char **syms,int num) { -unsigned long addr, size;unsigned long *addrs; -int i;/* Convert symbols to symbol address */ addrs =kcalloc (num,sizeof (*addrs),GFP_KERNEL );if (!addrs)return ERR_PTR (-ENOMEM ); -for (i =0 ; i < num; i++) { - addr =kallsyms_lookup_name (syms[i]); -if (!addr)/* Maybe wrong symbol */ - goto error; - -/* Convert symbol address to ftrace location. */ -if (!kallsyms_lookup_size_offset (addr, &size,NULL ) || !size) - goto error; +/* ftrace_lookup_symbols expects sorted symbols */ +sort (syms, num,sizeof (*syms), symbols_cmp,NULL ); - addr =ftrace_location_range (addr, addr + size -1 ); -if (!addr)/* No dynamic ftrace there. */ - goto error; +if (!ftrace_lookup_symbols (syms, num, addrs)) +return addrs; - addrs[i] = addr; - } - -return addrs; - -error:kfree (addrs); -return ERR_PTR (-ENOENT ); }
Signature · 1 line
2.35.1
Jiri Olsa
reviewerPATCH v3 4/5 bpf: Resolve symbols with ftrace_lookup_symbols for kprobe multi link
In reply to Jiri OlsaUsing kallsyms_lookup_names function to speed up symbols lookup in kprobe multi link attachment and replacing with it the current kprobe_multi_resolve_syms function.
This speeds up bpftrace kprobe attachment:
# perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* { } i:ms:1 { exit(); }'
...
6.5681 +- 0.0225 seconds time elapsed ( +- 0.34% )After:
# perf stat -r 5 -e cycles ./src/bpftrace -e 'kprobe:x* { } i:ms:1 { exit(); }'
...
0.5661 +- 0.0275 seconds time elapsed ( +- 4.85% )--- kernel/trace/bpf_trace.c | 112 +++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 46 deletions(-)
kernel/trace/bpf_trace.c +66 -46
@@ -2229,6 +2229,59 @@ struct bpf_kprobe_multi_run_ctx {unsigned long entry_ip; }; +struct user_syms { +const char **syms; +char *buf; +}; + +static int copy_user_syms (struct user_syms *us,unsigned long __user *usyms,u32 cnt) +{ +unsigned long __user usymbol; +const char **syms =NULL ; +char *buf =NULL , *p; +int err = -ENOMEM ; +unsigned int i; + + syms =kvmalloc (cnt *sizeof (*syms),GFP_KERNEL ); +if (!syms) + goto error; + + buf =kvmalloc (cnt *KSYM_NAME_LEN ,GFP_KERNEL ); +if (!buf) + goto error; + +for (p = buf, i =0 ; i < cnt; i++) { +if (__get_user (usymbol, usyms + i)) { + err = -EFAULT ; + goto error; + } + err =strncpy_from_user (p, (const char __user *) usymbol,KSYM_NAME_LEN ); +if (err ==KSYM_NAME_LEN ) + err = -E2BIG ; +if (err <0 ) + goto error; + syms[i] = p; + p += err +1 ; + } + + us->syms = syms; + us->buf = buf; +return 0 ; + +error: +if (err) { +kvfree (syms); +kvfree (buf); + } +return err; +} + +static void free_user_syms (struct user_syms *us) +{ +kvfree (us->syms); +kvfree (us->buf); +} +static void bpf_kprobe_multi_link_release (struct bpf_link *link) {struct bpf_kprobe_multi_link *kmulti_link;
@@ -2349,53 +2402,12 @@ kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip,kprobe_multi_link_prog_run (link, entry_ip, regs); } -static int -kprobe_multi_resolve_syms (const void __user *usyms,u32 cnt, -unsigned long *addrs) +static int symbols_cmp (const void *a,const void *b) { -unsigned long addr, size; -const char __user **syms; -int err = -ENOMEM ; -unsigned int i; -char *func; - - size = cnt *sizeof (*syms); - syms =kvzalloc (size,GFP_KERNEL ); -if (!syms) -return -ENOMEM ; +const char **str_a = (const char **) a; +const char **str_b = (const char **) b; - func =kmalloc (KSYM_NAME_LEN ,GFP_KERNEL ); -if (!func) - goto error; - -if (copy_from_user (syms, usyms, size)) { - err = -EFAULT ; - goto error; - } - -for (i =0 ; i < cnt; i++) { - err =strncpy_from_user (func, syms[i],KSYM_NAME_LEN ); -if (err ==KSYM_NAME_LEN ) - err = -E2BIG ; -if (err <0 ) - goto error; - err = -EINVAL ; - addr =kallsyms_lookup_name (func); -if (!addr) - goto error; -if (!kallsyms_lookup_size_offset (addr, &size,NULL )) - goto error; - addr =ftrace_location_range (addr, addr + size -1 ); -if (!addr) - goto error; - addrs[i] = addr; - } - - err =0 ; -error: -kvfree (syms); -kfree (func); -return err; +return strcmp (*str_a, *str_b); }int bpf_kprobe_multi_link_attach (const union bpf_attr *attr,struct bpf_prog *prog)
@@ -2441,7 +2453,15 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr goto error; } }else { - err =kprobe_multi_resolve_syms (usyms, cnt, addrs); +struct user_syms us; + + err =copy_user_syms (&us, usyms, cnt); +if (err) + goto error; + +sort (us.syms, cnt,sizeof (*us.syms), symbols_cmp,NULL ); + err =ftrace_lookup_symbols (us.syms, cnt, addrs); +free_user_syms (&us);if (err) goto error; }
Signature · 1 line
2.35.1
Jiri Olsa
reviewerPATCH v3 5/5 selftests/bpf: Add attach bench test
In reply to Jiri OlsaAdding test that reads all functions from ftrace available_filter_functions file and attach them all through kprobe_multi API.
It also prints stats info with -v option, like on my setup:
test_bench_attach: found 48712 functions
test_bench_attach: attached in 1.069s
test_bench_attach: detached in 0.373s
--- .../bpf/prog_tests/kprobe_multi_test.c | 133 ++++++++++++++++++ .../selftests/bpf/progs/kprobe_multi_empty.c | 12 ++ 2 files changed, 145 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/kprobe_multi_empty.c
tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +133 -0
@@ -2,6 +2,9 @@#include <test_progs.h> #include "kprobe_multi.skel.h" #include "trace_helpers.h" +#include "kprobe_multi_empty.skel.h" +#include "bpf/libbpf_internal.h" +#include "bpf/hashmap.h" static void kprobe_multi_test_run (struct kprobe_multi *skel,bool test_return) {
@@ -301,6 +304,134 @@ static void test_attach_api_fails(void)kprobe_multi__destroy (skel); } +static inline __u64 get_time_ns (void ) +{ +struct timespec t; + +clock_gettime (CLOCK_MONOTONIC , &t); +return (__u64 ) t.tv_sec *1000000000 + t.tv_nsec; +} + +static size_t symbol_hash (const void *key,void *ctx__maybe_unused ) +{ +return str_hash ((const char *) key); +} + +static bool symbol_equal (const void *key1,const void *key2,void *ctx__maybe_unused ) +{ +return strcmp ((const char *) key1, (const char *) key2) ==0 ; +} + +static int get_syms (char ***symsp,size_t *cntp) +{ +size_t cap =0 , cnt =0 , i; +char *name, **syms =NULL ; +struct hashmap *map; +char buf[256 ]; +FILE *f; +int err; + +/* +* The available_filter_functions contains many duplicates, +* but other than that all symbols are usable in kprobe multi +* interface. +* Filtering out duplicates by using hashmap__add, which won't +* add existing entry. +*/ + f =fopen ("/sys/kernel/debug/tracing/available_filter_functions" ,"r" ); +if (!f) +return -EINVAL ; + + map =hashmap__new (symbol_hash, symbol_equal,NULL ); +if (IS_ERR (map)) + goto error; + +while (fgets (buf,sizeof (buf), f)) { +/* skip modules */ +if (strchr (buf,'[' )) +continue ; +if (sscanf (buf,"%ms$*[^\n]\n" , &name) !=1 ) +continue ; + err =hashmap__add (map, name,NULL ); +if (err) { +free (name); +if (err == -EEXIST ) +continue ; + goto error; + } + err =libbpf_ensure_mem ((void **) &syms, &cap, +sizeof (*syms), cnt +1 ); +if (err) { +free (name); + goto error; + } + syms[cnt] = name; + cnt++; + } + + *symsp = syms; + *cntp = cnt; + +error: +fclose (f); +hashmap__free (map); +if (err) { +for (i =0 ; i < cnt; i++) +free (syms[cnt]); +free (syms); + } +return err; +} + +static void test_bench_attach (void ) +{ +LIBBPF_OPTS (bpf_kprobe_multi_opts, opts); +struct kprobe_multi_empty *skel =NULL ; +long attach_start_ns, attach_end_ns; +long detach_start_ns, detach_end_ns; +double attach_delta, detach_delta; +struct bpf_link *link =NULL ; +char **syms =NULL ; +size_t cnt, i; + +if (!ASSERT_OK (get_syms (&syms, &cnt),"get_syms" )) +return ; + + skel =kprobe_multi_empty__open_and_load (); +if (!ASSERT_OK_PTR (skel,"kprobe_multi_empty__open_and_load" )) + goto cleanup; + + opts.syms = (const char **) syms; + opts.cnt = cnt; + + attach_start_ns =get_time_ns (); + link =bpf_program__attach_kprobe_multi_opts (skel->progs.test_kprobe_empty, +NULL , &opts); + attach_end_ns =get_time_ns (); + +if (!ASSERT_OK_PTR (link,"bpf_program__attach_kprobe_multi_opts" )) + goto cleanup; + + detach_start_ns =get_time_ns (); +bpf_link__destroy (link); + detach_end_ns =get_time_ns (); + + attach_delta = (attach_end_ns - attach_start_ns) /1000000000.0 ; + detach_delta = (detach_end_ns - detach_start_ns) /1000000000.0 ; + +printf ("%s: found %lu functions\n" , __func__, cnt); +printf ("%s: attached in %7.3lfs\n" , __func__, attach_delta); +printf ("%s: detached in %7.3lfs\n" , __func__, detach_delta); + +cleanup: +kprobe_multi_empty__destroy (skel); +if (syms) { +for (i =0 ; i < cnt; i++) +free (syms[i]); +free (syms); + } +} +void test_kprobe_multi_test (void ) {if (!ASSERT_OK (load_kallsyms (),"load_kallsyms" ))
@@ -320,4 +451,6 @@ void test_kprobe_multi_test(void)test_attach_api_syms ();if (test__start_subtest ("attach_api_fails" ))test_attach_api_fails (); +if (test__start_subtest ("bench_attach" )) +test_bench_attach (); }
tools/testing/selftests/bpf/progs/kprobe_multi_empty.c new file +12 -0
@@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("kprobe.multi/") +int test_kprobe_empty (struct pt_regs *ctx) +{ +return 0 ; +}
Signature · 1 line
2.35.1
Re: [PATCHv3 bpf-next 5/5] selftests/bpf: Add attach bench test
Quoted 4 lines · Jiri Olsa
Adding test that reads all functions from ftrace
available_filter_functionsfile and attach them all throughkprobe_multiAPI.
Quoted 8 lines
It also prints stats info with -v option, like on my setup:
test_bench_attach: found 48712 functions
test_bench_attach: attached in 1.069s
test_bench_attach: detached in 0.373sSigned-off-by Jiri Olsa <[email protected]>---
LGTM.
Quoted 5 lines
.../bpf/prog_tests/kprobe_multi_test.c | 133 ++++++++++++++++++ .../selftests/bpf/progs/kprobe_multi_empty.c | 12 ++ 2 files changed, 145 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/kprobe_multi_empty.c
[...]