Skip to contentSkip to conversation

selftests/bpf: Add more uprobe multi tests

PATCH Applied Jiri Olsa 6 participants 11 messages Jul 22, 2024 – Sep 24, 2024

Subsystems (3)
lore
Share & tools
lore

Jiri Olsa

author reviewer
#0 lore raw

v3 cover selftests/bpf: Add more uprobe multi tests

hi,
adding more uprobe multi tests for failed attachments inside the uprobe register code.

v3 changes:
- renamed several variables/functions, null check [Andrii]
- fixed CI issue

thanks,
jirka

---
Jiri Olsa (2):
selftests/bpf: Add uprobe fail tests for uprobe multi selftests/bpf: Add uprobe multi consumers test

 tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c | 331 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c |  39 +++++++++++
 2 files changed, 370 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
author reviewer
#1 lore raw

PATCH v3 1/2 selftests/bpf: Add uprobe fail tests for uprobe multi

Adding tests for checking on recovery after failing to attach uprobe.

---
 .../bpf/prog_tests/uprobe_multi_test.c        | 118 ++++++++++++++++++
 1 file changed, 118 insertions(+)
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +118 -0
index bf6ca8e3eb13..e6255d4df81d 100644
@@ -516,6 +516,122 @@ static void test_attach_api_fails(void)
 	uprobe_multi__destroy(skel);
 }
 
+#ifdef __x86_64__
+noinline void uprobe_multi_error_func(void)
+{
+	/*
+	 * If --fcf-protection=branch is enabled the gcc generates endbr as
+	 * first instruction, so marking the exact address of int3 with the
+	 * symbol to be used in the attach_uprobe_fail_trap test below.
+	 */
+	asm volatile (
+		".globl uprobe_multi_error_func_int3;	\n"
+		"uprobe_multi_error_func_int3:		\n"
+		"int3					\n"
+	);
+}
+
+/*
+ * Attaching uprobe on uprobe_multi_error_func results in error
+ * because it already starts with int3 instruction.
+ */
+static void attach_uprobe_fail_trap(struct uprobe_multi *skel)
+{
+	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
+	const char *syms[4] = {
+		"uprobe_multi_func_1",
+		"uprobe_multi_func_2",
+		"uprobe_multi_func_3",
+		"uprobe_multi_error_func_int3",
+	};
+
+	opts.syms = syms;
+	opts.cnt = ARRAY_SIZE(syms);
+
+	skel->links.uprobe = bpf_program__attach_uprobe_multi(skel->progs.uprobe, -1,
+							      "/proc/self/exe", NULL, &opts);
+	if (!ASSERT_ERR_PTR(skel->links.uprobe, "bpf_program__attach_uprobe_multi")) {
+		bpf_link__destroy(skel->links.uprobe);
+		skel->links.uprobe = NULL;
+	}
+}
+#else
+static void attach_uprobe_fail_trap(struct uprobe_multi *skel) { }
+#endif
+
+short sema_1 __used, sema_2 __used;
+
+static void attach_uprobe_fail_refctr(struct uprobe_multi *skel)
+{
+	unsigned long *tmp_offsets = NULL, *tmp_ref_ctr_offsets = NULL;
+	unsigned long offsets[3], ref_ctr_offsets[3];
+	LIBBPF_OPTS(bpf_link_create_opts, opts);
+	const char *path = "/proc/self/exe";
+	const char *syms[3] = {
+		"uprobe_multi_func_1",
+		"uprobe_multi_func_2",
+	};
+	const char *sema[3] = {
+		"sema_1",
+		"sema_2",
+	};
+	int prog_fd, link_fd, err;
+
+	prog_fd = bpf_program__fd(skel->progs.uprobe_extra);
+
+	err = elf_resolve_syms_offsets("/proc/self/exe", 2, (const char **) &syms,
+				       &tmp_offsets, STT_FUNC);
+	if (!ASSERT_OK(err, "elf_resolve_syms_offsets_func"))
+		return;
+
+	err = elf_resolve_syms_offsets("/proc/self/exe", 2, (const char **) &sema,
+				       &tmp_ref_ctr_offsets, STT_OBJECT);
+	if (!ASSERT_OK(err, "elf_resolve_syms_offsets_sema"))
+		goto cleanup;
+
+	/*
+	 * We attach to 3 uprobes on 2 functions, so 2 uprobes share single function,
+	 * but with different ref_ctr_offset which is not allowed and results in fail.
+	 */
+	offsets[0] = tmp_offsets[0]; /* uprobe_multi_func_1 */
+	offsets[1] = tmp_offsets[1]; /* uprobe_multi_func_2 */
+	offsets[2] = tmp_offsets[1]; /* uprobe_multi_func_2 */
+
+	ref_ctr_offsets[0] = tmp_ref_ctr_offsets[0]; /* sema_1 */
+	ref_ctr_offsets[1] = tmp_ref_ctr_offsets[1]; /* sema_2 */
+	ref_ctr_offsets[2] = tmp_ref_ctr_offsets[0]; /* sema_1, error */
+
+	opts.uprobe_multi.path = path;
+	opts.uprobe_multi.offsets = (const unsigned long *) &offsets;
+	opts.uprobe_multi.ref_ctr_offsets = (const unsigned long *) &ref_ctr_offsets;
+	opts.uprobe_multi.cnt = 3;
+
+	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts);
+	if (!ASSERT_ERR(link_fd, "link_fd"))
+		close(link_fd);
+
+cleanup:
+	free(tmp_ref_ctr_offsets);
+	free(tmp_offsets);
+}
+
+static void test_attach_uprobe_fails(void)
+{
+	struct uprobe_multi *skel = NULL;
+
+	skel = uprobe_multi__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "uprobe_multi__open_and_load"))
+		return;
+
+	/* attach fails due to adding uprobe on trap instruction, x86_64 only */
+	attach_uprobe_fail_trap(skel);
+
+	/* attach fail due to wrong ref_ctr_offs on one of the uprobes */
+	attach_uprobe_fail_refctr(skel);
+
+	uprobe_multi__destroy(skel);
+}
+
 static void __test_link_api(struct child *child)
 {
 	int prog_fd, link1_fd = -1, link2_fd = -1, link3_fd = -1, link4_fd = -1;
@@ -703,4 +819,6 @@ void test_uprobe_multi_test(void)
 		test_bench_attach_usdt();
 	if (test__start_subtest("attach_api_fails"))
 		test_attach_api_fails();
+	if (test__start_subtest("attach_uprobe_fails"))
+		test_attach_uprobe_fails();
 }
Signature · 1 line
2.45.2
author reviewer
#2 lore raw

PATCH v3 2/2 selftests/bpf: Add uprobe multi consumers test

In reply to Jiri Olsa

Adding test that attaches/detaches multiple consumers on single uprobe and verifies all were hit as expected.

---
 .../bpf/prog_tests/uprobe_multi_test.c        | 213 ++++++++++++++++++
 .../bpf/progs/uprobe_multi_consumers.c        |  39 ++++
 2 files changed, 252 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +213 -0
index e6255d4df81d..27708110ea20 100644
@@ -6,6 +6,7 @@
 #include "uprobe_multi.skel.h"
 #include "uprobe_multi_bench.skel.h"
 #include "uprobe_multi_usdt.skel.h"
+#include "uprobe_multi_consumers.skel.h"
 #include "bpf/libbpf_internal.h"
 #include "testing_helpers.h"
 #include "../sdt.h"
@@ -731,6 +732,216 @@ static void test_link_api(void)
 	__test_link_api(child);
 }
 
+static struct bpf_program *
+get_program(struct uprobe_multi_consumers *skel, int prog)
+{
+	switch (prog) {
+	case 0:
+		return skel->progs.uprobe_0;
+	case 1:
+		return skel->progs.uprobe_1;
+	case 2:
+		return skel->progs.uprobe_2;
+	case 3:
+		return skel->progs.uprobe_3;
+	default:
+		ASSERT_FAIL("get_program");
+		return NULL;
+	}
+}
+
+static struct bpf_link **
+get_link(struct uprobe_multi_consumers *skel, int link)
+{
+	switch (link) {
+	case 0:
+		return &skel->links.uprobe_0;
+	case 1:
+		return &skel->links.uprobe_1;
+	case 2:
+		return &skel->links.uprobe_2;
+	case 3:
+		return &skel->links.uprobe_3;
+	default:
+		ASSERT_FAIL("get_link");
+		return NULL;
+	}
+}
+
+static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx)
+{
+	struct bpf_program *prog = get_program(skel, idx);
+	struct bpf_link **link = get_link(skel, idx);
+	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
+
+	if (!prog || !link)
+		return -1;
+
+	/*
+	 * bit/prog: 0,1 uprobe entry
+	 * bit/prog: 2,3 uprobe return
+	 */
+	opts.retprobe = idx == 2 || idx == 3;
+
+	*link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe",
+						"uprobe_consumer_test",
+						&opts);
+	if (!ASSERT_OK_PTR(*link, "bpf_program__attach_uprobe_multi"))
+		return -1;
+	return 0;
+}
+
+static void uprobe_detach(struct uprobe_multi_consumers *skel, int idx)
+{
+	struct bpf_link **link = get_link(skel, idx);
+
+	bpf_link__destroy(*link);
+	*link = NULL;
+}
+
+static bool test_bit(int bit, unsigned long val)
+{
+	return val & (1 << bit);
+}
+
+noinline int
+uprobe_consumer_test(struct uprobe_multi_consumers *skel,
+		     unsigned long before, unsigned long after)
+{
+	int idx;
+
+	/* detach uprobe for each unset programs in 'before' state ... */
+	for (idx = 0; idx < 4; idx++) {
+		if (test_bit(idx, before) && !test_bit(idx, after))
+			uprobe_detach(skel, idx);
+	}
+
+	/* ... and attach all new programs in 'after' state */
+	for (idx = 0; idx < 4; idx++) {
+		if (!test_bit(idx, before) && test_bit(idx, after)) {
+			if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_after"))
+				return -1;
+		}
+	}
+	return 0;
+}
+
+static void consumer_test(struct uprobe_multi_consumers *skel,
+			  unsigned long before, unsigned long after)
+{
+	int err, idx;
+
+	printf("consumer_test before %lu after %lu\n", before, after);
+
+	/* 'before' is each, we attach uprobe for every set idx */
+	for (idx = 0; idx < 4; idx++) {
+		if (test_bit(idx, before)) {
+			if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
+				goto cleanup;
+		}
+	}
+
+	err = uprobe_consumer_test(skel, before, after);
+	if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
+		goto cleanup;
+
+	for (idx = 0; idx < 4; idx++) {
+		const char *fmt = "BUG";
+		__u64 val = 0;
+
+		if (idx < 2) {
+			/*
+			 * uprobe entry
+			 *   +1 if define in 'before'
+			 */
+			if (test_bit(idx, before))
+				val++;
+			fmt = "prog 0/1: uprobe";
+		} else {
+			/*
+			 * uprobe return is tricky ;-)
+			 *
+			 * to trigger uretprobe consumer, the uretprobe needs to be installed,
+			 * which means one of the 'return' uprobes was alive when probe was hit:
+			 *
+			 *   idxs: 2/3 uprobe return in 'installed' mask
+			 *
+			 * in addition if 'after' state removes everything that was installed in
+			 * 'before' state, then uprobe kernel object goes away and return uprobe
+			 * is not installed and we won't hit it even if it's in 'after' state.
+			 */
+			unsigned long had_uretprobes  = before & 0b1100; // is uretprobe installed
+			unsigned long probe_preserved = before & after;  // did uprobe go away
+
+			if (had_uretprobes && probe_preserved && test_bit(idx, after))
+				val++;
+			fmt = "idx 2/3: uretprobe";
+		}
+
+		ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
+		skel->bss->uprobe_result[idx] = 0;
+	}
+
+cleanup:
+	for (idx = 0; idx < 4; idx++)
+		uprobe_detach(skel, idx);
+}
+
+static void test_consumers(void)
+{
+	struct uprobe_multi_consumers *skel;
+	int before, after;
+
+	skel = uprobe_multi_consumers__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "uprobe_multi_consumers__open_and_load"))
+		return;
+
+	/*
+	 * The idea of this test is to try all possible combinations of
+	 * uprobes consumers attached on single function.
+	 *
+	 *  - 2 uprobe entry consumer
+	 *  - 2 uprobe exit consumers
+	 *
+	 * The test uses 4 uprobes attached on single function, but that
+	 * translates into single uprobe with 4 consumers in kernel.
+	 *
+	 * The before/after values present the state of attached consumers
+	 * before and after the probed function:
+	 *
+	 *  bit/prog 0,1 : uprobe entry
+	 *  bit/prog 2,3 : uprobe return
+	 *
+	 * For example for:
+	 *
+	 *   before = 0b0101
+	 *   after  = 0b0110
+	 *
+	 * it means that before we call 'uprobe_consumer_test' we attach
+	 * uprobes defined in 'before' value:
+	 *
+	 *   - bit/prog 0: uprobe entry
+	 *   - bit/prog 2: uprobe return
+	 *
+	 * uprobe_consumer_test is called and inside it we attach and detach
+	 * uprobes based on 'after' value:
+	 *
+	 *   - bit/prog 0: stays untouched
+	 *   - bit/prog 2: uprobe return is detached
+	 *
+	 * uprobe_consumer_test returns and we check counters values increased
+	 * by bpf programs on each uprobe to match the expected count based on
+	 * before/after bits.
+	 */
+
+	for (before = 0; before < 16; before++) {
+		for (after = 0; after < 16; after++)
+			consumer_test(skel, before, after);
+	}
+
+	uprobe_multi_consumers__destroy(skel);
+}
+
 static void test_bench_attach_uprobe(void)
 {
 	long attach_start_ns = 0, attach_end_ns = 0;
@@ -821,4 +1032,6 @@ void test_uprobe_multi_test(void)
 		test_attach_api_fails();
 	if (test__start_subtest("attach_uprobe_fails"))
 		test_attach_uprobe_fails();
+	if (test__start_subtest("consumers"))
+		test_consumers();
 }
tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c new file +39 -0
new file mode 100644
index 000000000000..7e0fdcbbd242
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <stdbool.h>
+#include "bpf_kfuncs.h"
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+__u64 uprobe_result[4];
+
+SEC("uprobe.multi")
+int uprobe_0(struct pt_regs *ctx)
+{
+	uprobe_result[0]++;
+	return 0;
+}
+
+SEC("uprobe.multi")
+int uprobe_1(struct pt_regs *ctx)
+{
+	uprobe_result[1]++;
+	return 0;
+}
+
+SEC("uprobe.multi")
+int uprobe_2(struct pt_regs *ctx)
+{
+	uprobe_result[2]++;
+	return 0;
+}
+
+SEC("uprobe.multi")
+int uprobe_3(struct pt_regs *ctx)
+{
+	uprobe_result[3]++;
+	return 0;
+}
Signature · 1 line
2.45.2
#3 lore raw

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

Quoted 11 lines · Jiri Olsa

Adding test that attaches/detaches multiple consumers on single uprobe and verifies all were hit as expected.

---
 .../bpf/prog_tests/uprobe_multi_test.c        | 213 ++++++++++++++++++
 .../bpf/progs/uprobe_multi_consumers.c        |  39 ++++
 2 files changed, 252 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c

[...]

Quoted 5 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +5 -0
+               } else {
+                       /*
+                        * uprobe return is tricky ;-)
+                        *
+                        * to trigger uretprobe consumer, the uretprobe needs to be installed,

tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c static void test_link_api(void) · line 874

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +10 -0
+                        * which means one of the 'return' uprobes was alive when probe was hit:
+                        *
+                        *   idxs: 2/3 uprobe return in 'installed' mask
+                        *
+                        * in addition if 'after' state removes everything that was installed in
+                        * 'before' state, then uprobe kernel object goes away and return uprobe
+                        * is not installed and we won't hit it even if it's in 'after' state.
+                        */
+                       unsigned long had_uretprobes  = before & 0b1100; // is uretprobe installed
+                       unsigned long probe_preserved = before & after;  // did uprobe go away

fixed C++-style comments, pushed to bpf-next

Quoted 14 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +14 -0
+
+                       if (had_uretprobes && probe_preserved && test_bit(idx, after))
+                               val++;
+                       fmt = "idx 2/3: uretprobe";
+               }
+
+               ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
+               skel->bss->uprobe_result[idx] = 0;
+       }
+
+cleanup:
+       for (idx = 0; idx < 4; idx++)
+               uprobe_detach(skel, idx);
+}

[...]

#4 lore raw

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

In reply to Jiri Olsa
Quoted 29 lines · Jiri Olsa

Adding test that attaches/detaches multiple consumers on single uprobe and verifies all were hit as expected.

---
.../bpf/prog_tests/uprobe_multi_test.c | 213 ++++++++++++++++++ .../bpf/progs/uprobe_multi_consumers.c | 39 ++++

2 files changed, 252 insertions(+)

create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c

tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +1 -0
index e6255d4df81d..27708110ea20 100644
@@ -6,6 +6,7 @@
 #include "uprobe_multi.skel.h"
 #include "uprobe_multi_bench.skel.h"
 #include "uprobe_multi_usdt.skel.h"
+#include "uprobe_multi_consumers.skel.h"
 #include "bpf/libbpf_internal.h"
 #include "testing_helpers.h"
 #include "../sdt.h"
@@ -731,6 +732,216 @@ static void test_link_api(void)
 __test_link_api(child);
 }
 

[...]

Quoted 42 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +42 -0
+
+static void consumer_test(struct uprobe_multi_consumers skel,
+ unsigned long before, unsigned long after)
+{
+ int err, idx;
+
+ printf("consumer_test before %lu after %lu\n", before, after);
+
+ / 'before' is each, we attach uprobe for every set idx */
+ for (idx = 0; idx < 4; idx++) {
+ if (test_bit(idx, before)) {
+ if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
+ goto cleanup;
+ }
+ }
+
+ err = uprobe_consumer_test(skel, before, after);
+ if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
+ goto cleanup;
+
+ for (idx = 0; idx < 4; idx++) {
+ const char fmt = "BUG";
+ __u64 val = 0;
+
+ if (idx < 2) {
+ /
+ * uprobe entry
+ * +1 if define in 'before'
+ /
+ if (test_bit(idx, before))
+ val++;
+ fmt = "prog 0/1: uprobe";
+ } else {
+ /
+ * uprobe return is tricky ;-)
+ *
+ * to trigger uretprobe consumer, the uretprobe needs to be installed,
+ * which means one of the 'return' uprobes was alive when probe was hit:
+ *
+ * idxs: 2/3 uprobe return in 'installed' mask
+ *
+ * in addition if 'after' state removes everything that was installed in

tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c static void test_link_api(void) · line 879

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +10 -0
+ * 'before' state, then uprobe kernel object goes away and return uprobe
+ * is not installed and we won't hit it even if it's in 'after' state.
+ */
+ unsigned long had_uretprobes = before & 0b1100; // is uretprobe installed
+ unsigned long probe_preserved = before & after; // did uprobe go away
+
+ if (had_uretprobes && probe_preserved && test_bit(idx, after))
+ val++;
+ fmt = "idx 2/3: uretprobe";
+ }

Jiri, Andrii,

This test case started failing since upstream got merged into bpf-next, starting from commit https://git.kernel.org/bpf/bpf-next/c/440b65232829

A snippet from the test log:

consumer_test before 4 after 8 uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec consumer_test:PASS:uprobe_attach_before 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec consumer_test:PASS:uprobe_consumer_test 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:idx 2/3: uretprobe 0 nsec
consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0 consumer_test before 4 after 9 uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec consumer_test:PASS:uprobe_attach_before 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec consumer_test:PASS:uprobe_consumer_test 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:idx 2/3: uretprobe 0 nsec
consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0 consumer_test before 4 after 10 uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec consumer_test:PASS:uprobe_attach_before 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec consumer_test:PASS:uprobe_consumer_test 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:idx 2/3: uretprobe 0 nsec
consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0

I couldn't figure out the reason as I have very shallow understanding of what's happening in the test.

Jiri, could you please look into it?

I excluded this test from BPF CI for now.

Thank you!

Quoted 11 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +9 -0
+
+ ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
 
+ skel->bss->uprobe_result[idx] = 0;
 
+ }
+
+cleanup:
+ for (idx = 0; idx < 4; idx++)
+ uprobe_detach(skel, idx);
+}

[...]

#5 lore raw

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

Quoted 131 lines · Ihor Solodrai
Quoted 29 lines · Jiri Olsa

Adding test that attaches/detaches multiple consumers on single uprobe and verifies all were hit as expected.

---
.../bpf/prog_tests/uprobe_multi_test.c | 213 ++++++++++++++++++ .../bpf/progs/uprobe_multi_consumers.c | 39 ++++

2 files changed, 252 insertions(+)

create mode 100644 tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c

tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +1 -0
index e6255d4df81d..27708110ea20 100644
@@ -6,6 +6,7 @@
 #include "uprobe_multi.skel.h"
 #include "uprobe_multi_bench.skel.h"
 #include "uprobe_multi_usdt.skel.h"
+#include "uprobe_multi_consumers.skel.h"
 #include "bpf/libbpf_internal.h"
 #include "testing_helpers.h"
 #include "../sdt.h"
@@ -731,6 +732,216 @@ static void test_link_api(void)
 __test_link_api(child);
 }
 

[...]

Quoted 42 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +42 -0
+
+static void consumer_test(struct uprobe_multi_consumers skel,
+ unsigned long before, unsigned long after)
+{
+ int err, idx;
+
+ printf("consumer_test before %lu after %lu\n", before, after);
+
+ / 'before' is each, we attach uprobe for every set idx */
+ for (idx = 0; idx < 4; idx++) {
+ if (test_bit(idx, before)) {
+ if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))
+ goto cleanup;
+ }
+ }
+
+ err = uprobe_consumer_test(skel, before, after);
+ if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
+ goto cleanup;
+
+ for (idx = 0; idx < 4; idx++) {
+ const char fmt = "BUG";
+ __u64 val = 0;
+
+ if (idx < 2) {
+ /
+ * uprobe entry
+ * +1 if define in 'before'
+ /
+ if (test_bit(idx, before))
+ val++;
+ fmt = "prog 0/1: uprobe";
+ } else {
+ /
+ * uprobe return is tricky ;-)
+ *
+ * to trigger uretprobe consumer, the uretprobe needs to be installed,
+ * which means one of the 'return' uprobes was alive when probe was hit:
+ *
+ * idxs: 2/3 uprobe return in 'installed' mask
+ *
+ * in addition if 'after' state removes everything that was installed in

tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c static void test_link_api(void) · line 879

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +10 -0
+ * 'before' state, then uprobe kernel object goes away and return uprobe
+ * is not installed and we won't hit it even if it's in 'after' state.
+ */
+ unsigned long had_uretprobes = before & 0b1100; // is uretprobe installed
+ unsigned long probe_preserved = before & after; // did uprobe go away
+
+ if (had_uretprobes && probe_preserved && test_bit(idx, after))
+ val++;
+ fmt = "idx 2/3: uretprobe";
+ }

Jiri, Andrii,

This test case started failing since upstream got merged into bpf-next, starting from commit https://git.kernel.org/bpf/bpf-next/c/440b65232829

A snippet from the test log:

consumer_test before 4 after 8 uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec consumer_test:PASS:uprobe_attach_before 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec consumer_test:PASS:uprobe_consumer_test 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:idx 2/3: uretprobe 0 nsec
consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0 consumer_test before 4 after 9 uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec consumer_test:PASS:uprobe_attach_before 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec consumer_test:PASS:uprobe_consumer_test 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:idx 2/3: uretprobe 0 nsec
consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0 consumer_test before 4 after 10 uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec consumer_test:PASS:uprobe_attach_before 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec uprobe_attach:PASS:bpf_program__attach_uprobe_multi 0 nsec uprobe_consumer_test:PASS:uprobe_attach_after 0 nsec consumer_test:PASS:uprobe_consumer_test 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:prog 0/1: uprobe 0 nsec
consumer_test:PASS:idx 2/3: uretprobe 0 nsec
consumer_test:FAIL:idx 2/3: uretprobe unexpected idx 2/3: uretprobe: actual 1 != expected 0

Quoted 6 lines

I couldn't figure out the reason as I have very shallow understanding of what's happening in the test.

Jiri, could you please look into it?

I excluded this test from BPF CI for now.

Thanks for the mitigation! I think this is due to my recent RCU and refcounting changes to uprobes/uretprobes, which went through tip/perf/core initially. And now that tip and bpf-next trees converged, this condition:

Quoted 1 line

unsigned long probe_preserved = before & after; // did uprobe go away

is no longer correct, and uretprobe can be activated if there was *any* uretprobe installed before.

So the test needs adjustment, but I don't think anything really broke. I don't remember exactly (and given the conferencing schedule and quite bad internet can't test quickly), but I think the condition should now be:

unsigned long probe_preserved = after & 0x1100;

(though we might want to also rename the variable to be a bit more meaningful now).

Anyways, I don't think this is critical and we can address this later. But if anyone is willing to send a fix, I'd appreciate it, of course!

Quoted 17 lines

Thank you!

Quoted 11 lines
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +9 -0
+
+ ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);
 
+ skel->bss->uprobe_result[idx] = 0;
 
+ }
+
+cleanup:
+ for (idx = 0; idx < 4; idx++)
+ uprobe_detach(skel, idx);
+}

[...]

#6 lore raw

Re: [PATCHv3 bpf-next 2/2] selftests/bpf: Add uprobe multi consumers test

On Tue, Sep 24, 2024 at 07:44:50AM +0200, Andrii Nakryiko wrote:

SNIP

Quoted 20 lines
Quoted 20 lines
Quoted 19 lines

+ /
+ * uprobe return is tricky ;-)
+ *
+ * to trigger uretprobe consumer, the uretprobe needs to be installed,
+ * which means one of the 'return' uprobes was alive when probe was hit:
+ *
+ * idxs: 2/3 uprobe return in 'installed' mask
+ *
+ * in addition if 'after' state removes everything that was installed in
+ * 'before' state, then uprobe kernel object goes away and return uprobe
+ * is not installed and we won't hit it even if it's in 'after' state.
+ */
+ unsigned long had_uretprobes = before & 0b1100; // is uretprobe installed
+ unsigned long probe_preserved = before & after; // did uprobe go away +
+ if (had_uretprobes && probe_preserved && test_bit(idx, after))
+ val++;
+ fmt = "idx 2/3: uretprobe";
+ }

Quoted 4 lines
Quoted 4 lines

Jiri, Andrii,

This test case started failing since upstream got merged into bpf-next, starting from commit https://git.kernel.org/bpf/bpf-next/c/440b65232829

thanks for the report

SNIP

Quoted 16 lines

Thanks for the mitigation! I think this is due to my recent RCU and refcounting changes to uprobes/uretprobes, which went through tip/perf/core initially. And now that tip and bpf-next trees converged, this condition:

Quoted 1 line

unsigned long probe_preserved = before & after; // did uprobe go away

is no longer correct, and uretprobe can be activated if there was *any* uretprobe installed before.

So the test needs adjustment, but I don't think anything really broke. I don't remember exactly (and given the conferencing schedule and quite bad internet can't test quickly), but I think the condition should now be:

Quoted 7 lines

unsigned long probe_preserved = after & 0x1100;

(though we might want to also rename the variable to be a bit more meaningful now).

Anyways, I don't think this is critical and we can address this later. But if anyone is willing to send a fix, I'd appreciate it, of course!

I think we can remove that check completely.. I sent the patch, let's discuss there ;-)

thanks,
jirka

#7 lore raw
In reply to Jiri Olsa
CI has tested the following submission:

Dear patch submitter,

CI has tested the following submission:

Status:     SUCCESS
Name:       [PATCHv3,bpf-next,0/2] selftests/bpf: Add more uprobe multi tests
Patchwork:  https://patchwork.kernel.org/project/netdevbpf/list/?series=873053&state=*
Matrix:     https://github.com/kernel-patches/bpf/actions/runs/10048022329

No further action is necessary on your part.

Please note: this email is coming from an unmonitored mailbox. If you have questions or feedback, please reach out to the Meta Kernel CI team at [email protected].

#8 lore raw
In reply to Jiri Olsa
CI has tested the following submission:

Dear patch submitter,

CI has tested the following submission:

Status:     FAILURE
Name:       [PATCHv3,bpf-next,0/2] selftests/bpf: Add more uprobe multi tests
Patchwork:  https://patchwork.kernel.org/project/netdevbpf/list/?series=873053&state=*
Matrix:     https://github.com/kernel-patches/bpf/actions/runs/10050702681

Failed jobs:
test_progs_no_alu32-aarch64-gcc: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779250571
test_progs_no_alu32-s390x-gcc: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779381610
test_progs_no_alu32-x86_64-gcc: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779406344
test_progs_no_alu32-x86_64-llvm-17: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779390782
test_progs_no_alu32-x86_64-llvm-18: https://github.com/kernel-patches/bpf/actions/runs/10050702681/job/27779387928

First test_progs failure (test_progs_no_alu32-aarch64-gcc):
#134 libbpf_get_fd_by_id_opts
libbpf: prog 'check_access': BPF program load failed: Invalid argument
libbpf: prog 'check_access': -- BEGIN PROG LOAD LOG --
0: R1=ctx() R10=fp0
; int BPF_PROG(check_access, struct bpf_map *map, fmode_t fmode) @ test_libbpf_get_fd_by_id_opts.c:27
0: (b7) r0 = 0 ; R0_w=0
1: (79) r2 = *(u64 *)(r1 +0)
func 'bpf_lsm_bpf_map' arg0 has btf_id 2072 type STRUCT 'bpf_map'
2: R1=ctx() R2_w=trusted_ptr_bpf_map()
; if (map != (struct bpf_map *)&data_input) @ test_libbpf_get_fd_by_id_opts.c:29
2: (18) r3 = 0xffff0000c341f800 ; R3_w=map_ptr(map=data_input,ks=4,vs=4)
4: (5d) if r2 != r3 goto pc+4 ; R2_w=trusted_ptr_bpf_map() R3_w=map_ptr(map=data_input,ks=4,vs=4)
; int BPF_PROG(check_access, struct bpf_map *map, fmode_t fmode) @ test_libbpf_get_fd_by_id_opts.c:27
5: (79) r0 = *(u64 *)(r1 +8) ; R0_w=scalar() R1=ctx()
; if (fmode & FMODE_WRITE) @ test_libbpf_get_fd_by_id_opts.c:32
6: (67) r0 <<= 62 ; R0_w=scalar(smax=0x4000000000000000,umax=0xc000000000000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xc000000000000000))
7: (c7) r0 s>>= 63 ; R0_w=scalar(smin=smin32=-1,smax=smax32=0)
; @ test_libbpf_get_fd_by_id_opts.c:0
8: (57) r0 &= -13 ; R0_w=scalar(smax=0x7ffffffffffffff3,umax=0xfffffffffffffff3,smax32=0x7ffffff3,umax32=0xfffffff3,var_off=(0x0; 0xfffffffffffffff3))
; int BPF_PROG(check_access, struct bpf_map *map, fmode_t fmode) @ test_libbpf_get_fd_by_id_opts.c:27 9: (95) exit
At program exit the register R0 has smax=9223372036854775795 should have been in [-4095, 0] processed 9 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0 -- END PROG LOAD LOG --
libbpf: prog 'check_access': failed to load: -22
libbpf: failed to load object 'test_libbpf_get_fd_by_id_opts'
libbpf: failed to load BPF skeleton 'test_libbpf_get_fd_by_id_opts': -22 test_libbpf_get_fd_by_id_opts:FAIL:test_libbpf_get_fd_by_id_opts__open_and_load unexpected error: -22

Please note: this email is coming from an unmonitored mailbox. If you have questions or feedback, please reach out to the Meta Kernel CI team at [email protected].

#9 lore raw
In reply to Jiri Olsa
CI has tested the following submission:

Dear patch submitter,

CI has tested the following submission:

Status:     SUCCESS
Name:       [PATCHv3,bpf-next,0/2] selftests/bpf: Add more uprobe multi tests
Patchwork:  https://patchwork.kernel.org/project/netdevbpf/list/?series=873053&state=*
Matrix:     https://github.com/kernel-patches/bpf/actions/runs/10051533401

No further action is necessary on your part.

Please note: this email is coming from an unmonitored mailbox. If you have questions or feedback, please reach out to the Meta Kernel CI team at [email protected].

#10 lore raw
In reply to Jiri Olsa
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:

Hello:

This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:

Quoted 9 lines · On Mon, 22 Jul 2024 22:27:56 +0200 you wrote:

hi,
adding more uprobe multi tests for failed attachments inside the uprobe register code.

v3 changes:
- renamed several variables/functions, null check [Andrii]
- fixed CI issue

[...]

Here is the summary with links:
- [PATCHv3,bpf-next,1/2] selftests/bpf: Add uprobe fail tests for uprobe multi

    https://git.kernel.org/bpf/bpf-next/c/f5ee7d559a7c
  - [PATCHv3,bpf-next,2/2] selftests/bpf: Add uprobe multi consumers test
    https://git.kernel.org/bpf/bpf-next/c/c5ec71c325e7

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html