Skip to contentSkip to conversation

libbpf: kprobe.multi: Filter with available_filter_functions_addrs

PATCH In review Jackie Liu 3 participants 5 messages Jun 25, 2023 – Jun 30, 2023

Subsystems (2)
lore
Share & tools
lore

Jackie Liu

author
#0 lore raw

PATCH v2 1/1 libbpf: kprobe.multi: Filter with available_filter_functions_addrs

From: Jackie Liu <[email protected]>

When using regular expression matching with "kprobe multi", it scans all the functions under "/proc/kallsyms" that can be matched. However, not all of them can be traced by kprobe.multi. If any one of the functions fails to be traced, it will result in the failure of all functions. The best approach is to filter out the functions that cannot be traced to ensure proper tracking of the functions.

Use available_filter_functions_addrs check first, if failed, fallback to kallsyms.

Here is the test eBPF program [1].
[1] https://github.com/JackieLiu1/ketones/tree/master/src/funccount

---
 tools/lib/bpf/libbpf.c | 81 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 75 insertions(+), 6 deletions(-)
tools/lib/bpf/libbpf.c +75 -6
index a27f6e9ccce7..fca5d2e412c5 100644
@@ -10107,6 +10107,12 @@ static const char *tracefs_uprobe_events(void)
 	return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
 }
 
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+	return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" :
+			       TRACEFS"/available_filter_functions_addrs";
+}
+
 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
 					 const char *kfunc_name, size_t offset)
 {
@@ -10422,9 +10428,8 @@ struct kprobe_multi_resolve {
 	size_t cnt;
 };
 
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
-			const char *sym_name, void *ctx)
+static int ftrace_resolve_kprobe_multi_cb(unsigned long long sym_addr,
+					  const char *sym_name, void *ctx)
 {
 	struct kprobe_multi_resolve *res = ctx;
 	int err;
@@ -10441,6 +10446,63 @@ resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
 	return 0;
 }
 
+static int
+kallsyms_resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
+				 const char *sym_name, void *ctx)
+{
+	return ftrace_resolve_kprobe_multi_cb(sym_addr, sym_name, ctx);
+}
+
+typedef int (*available_kprobe_cb_t)(unsigned long long sym_addr,
+				     const char *sym_name, void *ctx);
+
+static int
+libbpf_available_kprobes_parse(available_kprobe_cb_t cb, void *ctx)
+{
+	unsigned long long sym_addr;
+	char sym_name[256];
+	FILE *f;
+	int ret, err = 0;
+	const char *available_path = tracefs_available_filter_functions_addrs();
+
+	f = fopen(available_path, "r");
+	if (!f) {
+		err = -errno;
+		pr_warn("failed to open %s, fallback to /proc/kallsyms.\n",
+			available_path);
+		return err;
+	}
+
+	while (true) {
+		ret = fscanf(f, "%llx %255s%*[^\n]\n", &sym_addr, sym_name);
+		if (ret == EOF && feof(f))
+			break;
+		if (ret != 2) {
+			pr_warn("failed to read available kprobe entry: %d\n",
+				ret);
+			err = -EINVAL;
+			break;
+		}
+
+		err = cb(sym_addr, sym_name, ctx);
+		if (err)
+			break;
+	}
+
+	fclose(f);
+	return err;
+}
+
+static void kprobe_multi_resolve_reinit(struct kprobe_multi_resolve *res)
+{
+	free(res->addrs);
+
+	/* reset to zero, when fallback */
+	res->cap = 0;
+	res->cnt = 0;
+	res->addrs = NULL;
+}
+
 struct bpf_link *
 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
 				      const char *pattern,
@@ -10477,9 +10539,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
 		return libbpf_err_ptr(-EINVAL);
 
 	if (pattern) {
-		err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
-		if (err)
-			goto error;
+		err = libbpf_available_kprobes_parse(ftrace_resolve_kprobe_multi_cb,
+						     &res);
+		if (err) {
+			/* fallback to kallsyms */
+			kprobe_multi_resolve_reinit(&res);
+			err = libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb,
+						    &res);
+			if (err)
+				goto error;
+		}
 		if (!res.cnt) {
 			err = -ENOENT;
 			goto error;
Signature · 1 line
2.25.1
Acked-by on v2 1/1
#1 lore raw
Quoted 9 lines · Jackie Liu

From: Jackie Liu <[email protected]>

When using regular expression matching with "kprobe multi", it scans all the functions under "/proc/kallsyms" that can be matched. However, not all of them can be traced by kprobe.multi. If any one of the functions fails to be traced, it will result in the failure of all functions. The best approach is to filter out the functions that cannot be traced to ensure proper tracking of the functions.

Quoted 8 lines

Use available_filter_functions_addrs check first, if failed, fallback to kallsyms.

Here is the test eBPF program [1].
[1] https://github.com/JackieLiu1/ketones/tree/master/src/funccount

thanks,
jirka

Quoted 120 lines
---
 tools/lib/bpf/libbpf.c | 81 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 75 insertions(+), 6 deletions(-)
tools/lib/bpf/libbpf.c +75 -6
index a27f6e9ccce7..fca5d2e412c5 100644
@@ -10107,6 +10107,12 @@ static const char *tracefs_uprobe_events(void)
 	return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
 }
 
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+	return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" :
+			       TRACEFS"/available_filter_functions_addrs";
+}
+
 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
 					 const char *kfunc_name, size_t offset)
 {
@@ -10422,9 +10428,8 @@ struct kprobe_multi_resolve {
 	size_t cnt;
 };
 
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
-			const char *sym_name, void *ctx)
+static int ftrace_resolve_kprobe_multi_cb(unsigned long long sym_addr,
+					  const char *sym_name, void *ctx)
 {
 	struct kprobe_multi_resolve *res = ctx;
 	int err;
@@ -10441,6 +10446,63 @@ resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
 	return 0;
 }
 
+static int
+kallsyms_resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
+				 const char *sym_name, void *ctx)
+{
+	return ftrace_resolve_kprobe_multi_cb(sym_addr, sym_name, ctx);
+}
+
+typedef int (*available_kprobe_cb_t)(unsigned long long sym_addr,
+				     const char *sym_name, void *ctx);
+
+static int
+libbpf_available_kprobes_parse(available_kprobe_cb_t cb, void *ctx)
+{
+	unsigned long long sym_addr;
+	char sym_name[256];
+	FILE *f;
+	int ret, err = 0;
+	const char *available_path = tracefs_available_filter_functions_addrs();
+
+	f = fopen(available_path, "r");
+	if (!f) {
+		err = -errno;
+		pr_warn("failed to open %s, fallback to /proc/kallsyms.\n",
+			available_path);
+		return err;
+	}
+
+	while (true) {
+		ret = fscanf(f, "%llx %255s%*[^\n]\n", &sym_addr, sym_name);
+		if (ret == EOF && feof(f))
+			break;
+		if (ret != 2) {
+			pr_warn("failed to read available kprobe entry: %d\n",
+				ret);
+			err = -EINVAL;
+			break;
+		}
+
+		err = cb(sym_addr, sym_name, ctx);
+		if (err)
+			break;
+	}
+
+	fclose(f);
+	return err;
+}
+
+static void kprobe_multi_resolve_reinit(struct kprobe_multi_resolve *res)
+{
+	free(res->addrs);
+
+	/* reset to zero, when fallback */
+	res->cap = 0;
+	res->cnt = 0;
+	res->addrs = NULL;
+}
+
 struct bpf_link *
 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
 				      const char *pattern,
@@ -10477,9 +10539,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
 		return libbpf_err_ptr(-EINVAL);
 
 	if (pattern) {
-		err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
-		if (err)
-			goto error;
+		err = libbpf_available_kprobes_parse(ftrace_resolve_kprobe_multi_cb,
+						     &res);
+		if (err) {
+			/* fallback to kallsyms */
+			kprobe_multi_resolve_reinit(&res);
+			err = libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb,
+						    &res);
+			if (err)
+				goto error;
+		}
 		if (!res.cnt) {
 			err = -ENOENT;
 			goto error;
Signature · 1 line
2.25.1
#2 lore raw
In reply to Jackie Liu
Quoted 10 lines · Jackie Liu

From: Jackie Liu <[email protected]>

When using regular expression matching with "kprobe multi", it scans all the functions under "/proc/kallsyms" that can be matched. However, not all of them can be traced by kprobe.multi. If any one of the functions fails to be traced, it will result in the failure of all functions. The best approach is to filter out the functions that cannot be traced to ensure proper tracking of the functions.

Quoted 3 lines

Use available_filter_functions_addrs check first, if failed, fallback to kallsyms.

This is good, but it also doesn't address the original problem. On older kernels that don't have available_filter_functions_addrs, we'll still be trying to attach to non-attachable kprobes?

So I think we'll still need to add available_filter_functions-based filtering on top of kallsyms. I guess we'll have to collect all symbols+addr from kallsyms, sort them, then go over available_filter_functions and do binary search. If element is not found, we'll mark it for removal. Then last pass to filter out marked entries to keep only addrs to be passed to kernel?

Quoted 56 lines

Here is the test eBPF program [1].
[1] https://github.com/JackieLiu1/ketones/tree/master/src/funccount

---
 tools/lib/bpf/libbpf.c | 81 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 75 insertions(+), 6 deletions(-)
tools/lib/bpf/libbpf.c +22 -3
index a27f6e9ccce7..fca5d2e412c5 100644
@@ -10107,6 +10107,12 @@ static const char *tracefs_uprobe_events(void)
        return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
 }
 
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+       return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" :
+                              TRACEFS"/available_filter_functions_addrs";
+}
+
 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
                                         const char *kfunc_name, size_t offset)
 {
@@ -10422,9 +10428,8 @@ struct kprobe_multi_resolve {
        size_t cnt;
 };
 
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
-                       const char *sym_name, void *ctx)
+static int ftrace_resolve_kprobe_multi_cb(unsigned long long sym_addr,
+                                         const char *sym_name, void *ctx)
 {
        struct kprobe_multi_resolve *res = ctx;
        int err;
@@ -10441,6 +10446,63 @@ resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
        return 0;
 }
 
+static int
+kallsyms_resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
+                                const char *sym_name, void *ctx)
+{
+       return ftrace_resolve_kprobe_multi_cb(sym_addr, sym_name, ctx);
+}
+
+typedef int (*available_kprobe_cb_t)(unsigned long long sym_addr,
+                                    const char *sym_name, void *ctx);
+
+static int
+libbpf_available_kprobes_parse(available_kprobe_cb_t cb, void *ctx)
+{
+       unsigned long long sym_addr;

tools/lib/bpf/libbpf.c resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type, · line 10472

Quoted 10 lines
tools/lib/bpf/libbpf.c +10 -0
+       char sym_name[256];
+       FILE *f;
+       int ret, err = 0;
+       const char *available_path = tracefs_available_filter_functions_addrs();
+
+       f = fopen(available_path, "r");
+       if (!f) {
+               err = -errno;
+               pr_warn("failed to open %s, fallback to /proc/kallsyms.\n",
+                       available_path);

if this is expected to happen, we shouldn't pr_warn() and pollute log output

Quoted 38 lines
tools/lib/bpf/libbpf.c +33 -0
+               return err;
+       }
+
+       while (true) {
+               ret = fscanf(f, "%llx %255s%*[^\n]\n", &sym_addr, sym_name);
+               if (ret == EOF && feof(f))
+                       break;
+               if (ret != 2) {
+                       pr_warn("failed to read available kprobe entry: %d\n",
+                               ret);
+                       err = -EINVAL;
+                       break;
+               }
+
+               err = cb(sym_addr, sym_name, ctx);
+               if (err)
+                       break;
+       }
+
+       fclose(f);
+       return err;
+}
+
+static void kprobe_multi_resolve_reinit(struct kprobe_multi_resolve *res)
+{
+       free(res->addrs);
+
+       /* reset to zero, when fallback */
+       res->cap = 0;
+       res->cnt = 0;
+       res->addrs = NULL;
+}
+
 struct bpf_link *
 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
                                      const char *pattern,
@@ -10477,9 +10539,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
                return libbpf_err_ptr(-EINVAL);

tools/lib/bpf/libbpf.c bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · line 10546

Quoted 10 lines
tools/lib/bpf/libbpf.c +5 -3
 
        if (pattern) {
-               err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
-               if (err)
-                       goto error;
+               err = libbpf_available_kprobes_parse(ftrace_resolve_kprobe_multi_cb,
+                                                    &res);
+               if (err) {
+                       /* fallback to kallsyms */
+                       kprobe_multi_resolve_reinit(&res);

instead of try, fail, try something else approach, can we check that tracefs_available_filter_functions_addrs() exists and is readable, and if yes -- commit to it. If something fails -- to bad. If it doesn't exist, fallback to kallsyms. And we won't need ugly kprobe_multi_resolve_reinit() hack.

Quoted 12 lines
tools/lib/bpf/libbpf.c +5 -1
+                       err = libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb,
+                                                   &res);
+                       if (err)
+                               goto error;
+               }
                if (!res.cnt) {
                        err = -ENOENT;
                        goto error;
--

2.25.1

author
#3 lore raw
Quoted 19 lines · Andrii Nakryiko
Quoted 10 lines · Jackie Liu

From: Jackie Liu <[email protected]>

When using regular expression matching with "kprobe multi", it scans all the functions under "/proc/kallsyms" that can be matched. However, not all of them can be traced by kprobe.multi. If any one of the functions fails to be traced, it will result in the failure of all functions. The best approach is to filter out the functions that cannot be traced to ensure proper tracking of the functions.

Quoted 3 lines

Use available_filter_functions_addrs check first, if failed, fallback to kallsyms.

This is good, but it also doesn't address the original problem. On older kernels that don't have available_filter_functions_addrs, we'll still be trying to attach to non-attachable kprobes?

Quoted 6 lines

So I think we'll still need to add available_filter_functions-based filtering on top of kallsyms. I guess we'll have to collect all symbols+addr from kallsyms, sort them, then go over available_filter_functions and do binary search. If element is not found, we'll mark it for removal. Then last pass to filter out marked entries to keep only addrs to be passed to kernel?

The first patch I submitted was to re-search available_filter_functions in the case of kallsyms matching. The link is at:

https://lore.kernel.org/all/[email protected]/,

but it is very slow and takes about 5s to start. If necessary, I can continue to add it to the V3 patch. Maybe you have other better algorithms?

Signature · 1 line
Jackie Liu
Quoted 110 lines
Quoted 58 lines

Here is the test eBPF program [1].
[1] https://github.com/JackieLiu1/ketones/tree/master/src/funccount

---
  tools/lib/bpf/libbpf.c | 81 ++++++++++++++++++++++++++++++++++++++----
  1 file changed, 75 insertions(+), 6 deletions(-)
tools/lib/bpf/libbpf.c +22 -3
index a27f6e9ccce7..fca5d2e412c5 100644
@@ -10107,6 +10107,12 @@ static const char *tracefs_uprobe_events(void)
         return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
  }
 
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+       return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" :
+                              TRACEFS"/available_filter_functions_addrs";
+}
+
  static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
                                          const char *kfunc_name, size_t offset)
  {
@@ -10422,9 +10428,8 @@ struct kprobe_multi_resolve {
         size_t cnt;
  };
 
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
-                       const char *sym_name, void *ctx)
+static int ftrace_resolve_kprobe_multi_cb(unsigned long long sym_addr,
+                                         const char *sym_name, void *ctx)
  {
         struct kprobe_multi_resolve *res = ctx;
         int err;
@@ -10441,6 +10446,63 @@ resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
         return 0;
  }
 
+static int
+kallsyms_resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
+                                const char *sym_name, void *ctx)
+{
+       return ftrace_resolve_kprobe_multi_cb(sym_addr, sym_name, ctx);
+}
+
+typedef int (*available_kprobe_cb_t)(unsigned long long sym_addr,
+                                    const char *sym_name, void *ctx);
+
+static int
+libbpf_available_kprobes_parse(available_kprobe_cb_t cb, void *ctx)
+{
+       unsigned long long sym_addr;

tools/lib/bpf/libbpf.c resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type, · line 10472

Quoted 10 lines
tools/lib/bpf/libbpf.c +10 -0
+       char sym_name[256];
+       FILE *f;
+       int ret, err = 0;
+       const char *available_path = tracefs_available_filter_functions_addrs();
+
+       f = fopen(available_path, "r");
+       if (!f) {
+               err = -errno;
+               pr_warn("failed to open %s, fallback to /proc/kallsyms.\n",
+                       available_path);

if this is expected to happen, we shouldn't pr_warn() and pollute log output

Quoted 39 lines
tools/lib/bpf/libbpf.c +33 -0
+               return err;
+       }
+
+       while (true) {
+               ret = fscanf(f, "%llx %255s%*[^\n]\n", &sym_addr, sym_name);
+               if (ret == EOF && feof(f))
+                       break;
+               if (ret != 2) {
+                       pr_warn("failed to read available kprobe entry: %d\n",
+                               ret);
+                       err = -EINVAL;
+                       break;
+               }
+
+               err = cb(sym_addr, sym_name, ctx);
+               if (err)
+                       break;
+       }
+
+       fclose(f);
+       return err;
+}
+
+static void kprobe_multi_resolve_reinit(struct kprobe_multi_resolve *res)
+{
+       free(res->addrs);
+
+       /* reset to zero, when fallback */
+       res->cap = 0;
+       res->cnt = 0;
+       res->addrs = NULL;
+}
+
  struct bpf_link *
  bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
                                       const char *pattern,
@@ -10477,9 +10539,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
                 return libbpf_err_ptr(-EINVAL);
 
Quoted 15 lines

tools/lib/bpf/libbpf.c bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · line 10546

Quoted 9 lines
tools/lib/bpf/libbpf.c +5 -3
         if (pattern) {
-               err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
-               if (err)
-                       goto error;
+               err = libbpf_available_kprobes_parse(ftrace_resolve_kprobe_multi_cb,
+                                                    &res);
+               if (err) {
+                       /* fallback to kallsyms */
+                       kprobe_multi_resolve_reinit(&res);

instead of try, fail, try something else approach, can we check that tracefs_available_filter_functions_addrs() exists and is readable, and if yes -- commit to it. If something fails -- to bad. If it doesn't exist, fallback to kallsyms. And we won't need ugly kprobe_multi_resolve_reinit() hack.

Sounds good.

Quoted 13 lines
Quoted 13 lines
tools/lib/bpf/libbpf.c +5 -1
+                       err = libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb,
+                                                   &res);
+                       if (err)
+                               goto error;
+               }
                 if (!res.cnt) {
                         err = -ENOENT;
                         goto error;
--

2.25.1

#4 lore raw
Quoted 30 lines · Jackie Liu
Quoted 19 lines · Andrii Nakryiko
Quoted 10 lines · Jackie Liu

From: Jackie Liu <[email protected]>

When using regular expression matching with "kprobe multi", it scans all the functions under "/proc/kallsyms" that can be matched. However, not all of them can be traced by kprobe.multi. If any one of the functions fails to be traced, it will result in the failure of all functions. The best approach is to filter out the functions that cannot be traced to ensure proper tracking of the functions.

Quoted 3 lines

Use available_filter_functions_addrs check first, if failed, fallback to kallsyms.

This is good, but it also doesn't address the original problem. On older kernels that don't have available_filter_functions_addrs, we'll still be trying to attach to non-attachable kprobes?

Quoted 6 lines

So I think we'll still need to add available_filter_functions-based filtering on top of kallsyms. I guess we'll have to collect all symbols+addr from kallsyms, sort them, then go over available_filter_functions and do binary search. If element is not found, we'll mark it for removal. Then last pass to filter out marked entries to keep only addrs to be passed to kernel?

Quoted 8 lines

The first patch I submitted was to re-search available_filter_functions in the case of kallsyms matching. The link is at:

https://lore.kernel.org/all/[email protected]/,

but it is very slow and takes about 5s to start. If necessary, I can continue to add it to the V3 patch. Maybe you have other better algorithms?

Of course it's very slow. For every matched kallsyms symbol you are reopening/reparsing/scanning linearly entire available_filter_functions. You need to make sure that you parse both kallsyms and available_filter_functions just once, and then join them together to keep only symbols in both files.

See my suggestion above about sorting and binary search. That's one way to do it efficiently.

Quoted 145 lines

--
Jackie Liu

Quoted 110 lines
Quoted 58 lines

Here is the test eBPF program [1].
[1] https://github.com/JackieLiu1/ketones/tree/master/src/funccount

---
  tools/lib/bpf/libbpf.c | 81 ++++++++++++++++++++++++++++++++++++++----
  1 file changed, 75 insertions(+), 6 deletions(-)
tools/lib/bpf/libbpf.c +22 -3
index a27f6e9ccce7..fca5d2e412c5 100644
@@ -10107,6 +10107,12 @@ static const char *tracefs_uprobe_events(void)
         return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
  }
 
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+       return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" :
+                              TRACEFS"/available_filter_functions_addrs";
+}
+
  static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
                                          const char *kfunc_name, size_t offset)
  {
@@ -10422,9 +10428,8 @@ struct kprobe_multi_resolve {
         size_t cnt;
  };
 
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
-                       const char *sym_name, void *ctx)
+static int ftrace_resolve_kprobe_multi_cb(unsigned long long sym_addr,
+                                         const char *sym_name, void *ctx)
  {
         struct kprobe_multi_resolve *res = ctx;
         int err;
@@ -10441,6 +10446,63 @@ resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
         return 0;
  }
 
+static int
+kallsyms_resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
+                                const char *sym_name, void *ctx)
+{
+       return ftrace_resolve_kprobe_multi_cb(sym_addr, sym_name, ctx);
+}
+
+typedef int (*available_kprobe_cb_t)(unsigned long long sym_addr,
+                                    const char *sym_name, void *ctx);
+
+static int
+libbpf_available_kprobes_parse(available_kprobe_cb_t cb, void *ctx)
+{
+       unsigned long long sym_addr;

tools/lib/bpf/libbpf.c resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type, · line 10472

Quoted 10 lines
tools/lib/bpf/libbpf.c +10 -0
+       char sym_name[256];
+       FILE *f;
+       int ret, err = 0;
+       const char *available_path = tracefs_available_filter_functions_addrs();
+
+       f = fopen(available_path, "r");
+       if (!f) {
+               err = -errno;
+               pr_warn("failed to open %s, fallback to /proc/kallsyms.\n",
+                       available_path);

if this is expected to happen, we shouldn't pr_warn() and pollute log output

Quoted 39 lines
tools/lib/bpf/libbpf.c +33 -0
+               return err;
+       }
+
+       while (true) {
+               ret = fscanf(f, "%llx %255s%*[^\n]\n", &sym_addr, sym_name);
+               if (ret == EOF && feof(f))
+                       break;
+               if (ret != 2) {
+                       pr_warn("failed to read available kprobe entry: %d\n",
+                               ret);
+                       err = -EINVAL;
+                       break;
+               }
+
+               err = cb(sym_addr, sym_name, ctx);
+               if (err)
+                       break;
+       }
+
+       fclose(f);
+       return err;
+}
+
+static void kprobe_multi_resolve_reinit(struct kprobe_multi_resolve *res)
+{
+       free(res->addrs);
+
+       /* reset to zero, when fallback */
+       res->cap = 0;
+       res->cnt = 0;
+       res->addrs = NULL;
+}
+
  struct bpf_link *
  bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
                                       const char *pattern,
@@ -10477,9 +10539,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
                 return libbpf_err_ptr(-EINVAL);
 
Quoted 15 lines

tools/lib/bpf/libbpf.c bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · line 10546

Quoted 9 lines
tools/lib/bpf/libbpf.c +5 -3
         if (pattern) {
-               err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
-               if (err)
-                       goto error;
+               err = libbpf_available_kprobes_parse(ftrace_resolve_kprobe_multi_cb,
+                                                    &res);
+               if (err) {
+                       /* fallback to kallsyms */
+                       kprobe_multi_resolve_reinit(&res);

instead of try, fail, try something else approach, can we check that tracefs_available_filter_functions_addrs() exists and is readable, and if yes -- commit to it. If something fails -- to bad. If it doesn't exist, fallback to kallsyms. And we won't need ugly kprobe_multi_resolve_reinit() hack.

Sounds good.

Quoted 13 lines
Quoted 13 lines
tools/lib/bpf/libbpf.c +5 -1
+                       err = libbpf_kallsyms_parse(kallsyms_resolve_kprobe_multi_cb,
+                                                   &res);
+                       if (err)
+                               goto error;
+               }
                 if (!res.cnt) {
                         err = -ENOENT;
                         goto error;
--

2.25.1