Skip to contentSkip to conversation

selftests/bpf: Add kptr-xchg benchmark

PATCH In review Chenguang Zhao 3 participants 3 messages Aug 10 – Aug 13

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 selftests/bpf: Add kptr-xchg benchmark

From: Chenguang Zhao <[email protected]>

Add a microbenchmark that repeatedly calls bpf_kptr_xchg() from an fentry program. Architectures that advertise bpf_jit_supports_ptr_xchg() inline the helper to BPF_XCHG; others keep the helper call. This bench makes it easy to compare the two paths under the same workload.

Usage:
sudo ./bench -d 30 -w 5 -p 1 kptr-xchg --nr_loops 256

---
Example results on LoongArch64 (helper vs inlined):

1. kptr xchg is helper:

	./bench -d 30 -w 5 -p 1 kptr-xchg --nr_loops 256
	Summary: throughput   68.612 ± 0.249 M ops/s ( 68.612M ops/prod), latency   14.575 ns/op

kptr xchg is inlined:

	./bench -d 30 -w 5 -p 1 kptr-xchg --nr_loops 256
	Summary: throughput   82.983 ± 0.268 M ops/s ( 82.983M ops/prod), latency   12.051 ns/op

This allows comparing the throughput difference.

2. To check whether bpf_kptr_xchg() was inlined:

bpftool prog show | grep -A4 -B1 'name benchmark'

       xlated 64B  jited 196B  memlock 16384B
   46: tracing  name benchmark  tag a9c8498a6197e8db  gpl
       loaded_at 2026-05-29T16:29:47+0800  uid 0
       xlated 232B  jited 380B  memlock 16384B  map_ids 8,10,9

bpftool prog dump xlated id 46 | grep -A6 -B6 -E 'xchg|atomic|call'

helper:
; __sync_add_and_fetch(&hits, i);
13: (18) r2 = map[id:10][0]+0
15: (db) lock *(u64 *)(r2 +0) += r1
; return 0;
16: (b4) w0 = 0
17: (95) exit
; old = bpf_kptr_xchg(&ptr, NULL);
18: (18) r1 = map[id:9][0]+0
20: (b7) r2 = 0
21: (85) call bpf_kptr_xchg#244684 ------ there call helper function
; if (old)
22: (15) if r0 == 0x0 goto pc-16
; bpf_obj_drop(old);
23: (bf) r1 = r0
24: (18) r2 = 0x0
26: (85) call 0x900000000046760c#92204
27: (05) goto pc-21

inlined:
; __sync_add_and_fetch(&hits, i);
13: (18) r2 = map[id:10][0]+0
15: (db) lock *(u64 *)(r2 +0) += r1
; return 0;
16: (b4) w0 = 0
17: (95) exit
; old = bpf_kptr_xchg(&ptr, NULL);
18: (18) r1 = map[id:9][0]+0
20: (b7) r2 = 0
21: (bf) r0 = r2
22: (db) r0 = atomic64_xchg((u64 *)(r1 +0), r0) ---- there inlining 'kptr xchg'
; if (old)
23: (15) if r0 == 0x0 goto pc-17
; bpf_obj_drop(old);
24: (bf) r1 = r0
25: (18) r2 = 0x0
27: (85) call 0x900000000046760e#92206
28: (05) goto pc-22

 tools/testing/selftests/bpf/Makefile          |  2 +
 tools/testing/selftests/bpf/bench.c           |  2 +
 .../selftests/bpf/benchs/bench_kptr_xchg.c    | 96 +++++++++++++++++++
 .../selftests/bpf/progs/kptr_xchg_bench.c     | 49 ++++++++++
 4 files changed, 149 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c
 create mode 100644 tools/testing/selftests/bpf/progs/kptr_xchg_bench.c
tools/testing/selftests/bpf/Makefile +2 -0
index d3655a706482..76782b442aa9 100644
@@ -988,6 +988,7 @@ $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_tr
 $(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h
 $(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h
 $(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h
+$(OUTPUT)/bench_kptr_xchg.o: $(OUTPUT)/kptr_xchg_bench.skel.h
 $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ)
 $(OUTPUT)/bench: LDLIBS += -lm
 $(OUTPUT)/bench: $(OUTPUT)/bench.o \
@@ -1014,6 +1015,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \
 		 $(OUTPUT)/bench_bpf_timing.o \
 		 $(OUTPUT)/bench_bpf_nop.o \
 		 $(OUTPUT)/bench_xdp_lb.o \
+		 $(OUTPUT)/bench_kptr_xchg.o \
 		 $(OUTPUT)/usdt_1.o \
 		 $(OUTPUT)/usdt_2.o \
 		 #
tools/testing/selftests/bpf/bench.c +2 -0
index b86b73456d3c..96cbd31088b8 100644
@@ -585,6 +585,7 @@ extern const struct bench bench_lpm_trie_delete;
 extern const struct bench bench_lpm_trie_free;
 extern const struct bench bench_bpf_nop;
 extern const struct bench bench_xdp_lb;
+extern const struct bench bench_kptr_xchg;
 
 static const struct bench *benchs[] = {
 	&bench_count_global,
@@ -669,6 +670,7 @@ static const struct bench *benchs[] = {
 	&bench_lpm_trie_free,
 	&bench_bpf_nop,
 	&bench_xdp_lb,
+	&bench_kptr_xchg,
 };
 
 static void find_benchmark(void)
tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c new file +96 -0

96 lines not on this page · raw message

tools/testing/selftests/bpf/progs/kptr_xchg_bench.c new file +49 -0

49 lines not on this page · raw message

Signature · 1 line
2.25.1
#1 lore raw
Should parse_arg() validate the nr_loops value here? Looking at the BPF program, the loop is capped at MAX_XCHG_LOOPS (4096):
Quoted 4 lines
tools/testing/selftests/bpf/Makefile +0 -0
index d3655a706482..76782b442aa9 100644

[ ... ]

Quoted 6 lines
tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c new file +0 -0
new file mode 100644
index 000000000000..b8a0d346fda6
@@ -0,0 +1,96 @@

[ ... ]

Quoted 6 lines
tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c new file +6 -0
+enum {
+	ARG_NR_LOOPS = 7000,
+};
+
+static const struct argp_option opts[] = {
+	{ "nr_loops", ARG_NR_LOOPS, "nr_loops", 0,

tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c line 32

Quoted 10 lines
tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c new file +10 -0
+	  "Set number of bpf_kptr_xchg() calls per trigger"},
+	{},
+};
+
+static error_t parse_arg(int key, char *arg, struct argp_state *state)
+{
+	switch (key) {
+	case ARG_NR_LOOPS:
+		args.nr_loops = strtol(arg, NULL, 10);
+		break;

Should parse_arg() validate the nr_loops value here? Looking at the BPF program, the loop is capped at MAX_XCHG_LOOPS (4096):

tools/testing/selftests/bpf/progs/kptr_xchg_bench.c:benchmark() {
	for (i = 0; i < MAX_XCHG_LOOPS; i++) {
		if (i >= nr_loops)
			break;
		...
	}
}

So passing --nr_loops 100000 silently benchmarks only 4096 iterations, and --nr_loops -1 (which strtol accepts and wraps to 4294967295 when assigned to __u32) also silently runs 4096. The user gets throughput numbers for a different workload than requested.

bench_trigger.c already handles this by rejecting out-of-range values at parse time:

	ret = strtol(arg, NULL, 10);
	if (ret < 1 || ret > MAX_TRIG_BATCH_ITERS) {
		fprintf(stderr, "invalid --trig-batch-iters value ...\n");
		argp_usage(state);
	}

Could bench_kptr_xchg.c add the same range check against MAX_XCHG_LOOPS to make the cap visible rather than silent?

tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c line 43

Quoted 4 lines
tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c new file +4 -0
+static const struct argp bench_kptr_xchg_argp = {
+	.options = opts,
+	.parser = parse_arg,
+};

Is bench_kptr_xchg_argp intended to be static? All other benches export their argp non-static (bench_bpf_loop_argp, bench_xdp_lb_argp, bench_sockmap_argp, etc.) and register them in two places:

1. bench.c declares extern struct argp bench_*_argp
2. bench.c adds them to the static bench_parsers[] child list

This patch adds neither. Consequences:

./bench --help never shows the "kptr-xchg" option group or --nr_loops, because --help is served from the first-pass child list.

--nr_loops only works today because bench_bpf_loop_argp and bench_bpf_for_argp (both already in bench_parsers[]) also declare a long option named "nr_loops", so the first pass consumes the argument into an unrelated bench's args struct rather than erroring out. The second pass then parses it again with the correct parser, so the documented invocation happens to work. Any option name unique to this bench would make the first pass abort with "unrecognized option".

Should bench_kptr_xchg_argp drop static, get an extern declaration in bench.c, and be added to bench_parsers[] alongside the other benches?

Quoted 5 lines
tools/testing/selftests/bpf/progs/kptr_xchg_bench.c new file +0 -0
new file mode 100644
index 000000000000..363883073e2c

[ ... ]

Quoted 17 lines
tools/testing/selftests/bpf/progs/kptr_xchg_bench.c new file +17 -0
+SEC("fentry/" SYS_PREFIX "sys_getpgid")
+int benchmark(void *ctx)
+{
+	struct bin_data *old;
+	u32 i;
+
+	for (i = 0; i < MAX_XCHG_LOOPS; i++) {
+		if (i >= nr_loops)
+			break;
+
+		old = bpf_kptr_xchg(&ptr, NULL);
+		if (old)
+			bpf_obj_drop(old);
+	}
+
+	__sync_add_and_fetch(&hits, i);
+	return 0;

tools/testing/selftests/bpf/progs/kptr_xchg_bench.c line 49

Quoted 10 lines
tools/testing/selftests/bpf/progs/kptr_xchg_bench.c new file +10 -0
+}
+
+/*
+ * BTF FUNC records are not generated for kfuncs referenced only through
+ * optimized paths. Keep bpf_obj_drop() visible to libbpf's kfunc linker.
+ */
+void __btf_root(void)
+{
+	bpf_obj_drop(NULL);
+}

Is __btf_root() needed in this file? The workaround was copied from kptr_xchg_inline.c, but that file actually requires it because the only reference to bpf_obj_drop is inside a __naked function's inline assembly via __imm(bpf_obj_drop), so clang never sees a C-level call and emits no BTF FUNC record. That file's own comment says exactly that: "BTF FUNC records are not generated for kfuncs referenced from inline assembly."

kptr_xchg_bench.c has no inline assembly. benchmark() calls the kfunc with a plain C call:

	old = bpf_kptr_xchg(&ptr, NULL);
	if (old)
		bpf_obj_drop(old);

clang cannot prove old is always NULL (bpf_kptr_xchg is an opaque helper call), so the call site survives -O2 and a BTF FUNC record for bpf_obj_drop is emitted normally. This is confirmed by twenty other programs in progs/ that call bpf_obj_drop() from ordinary C and define no such helper: linked_list.c, refcounted_kptr.c, irq.c, local_kptr_stash.c, rbtree_search.c, bpf_qdisc_fifo.c, etc. kptr_xchg_inline.c is the only file in the tree that actually needs the hack.

The rewritten comment also generalises the original's precise claim ("from inline assembly") into "referenced only through optimized paths", which is not a real libbpf/clang behaviour and does not describe anything happening in this file.

The function is never loaded (it is unreferenced, non-SEC() .text, so libbpf never appends it as a subprog, which is why the bpf_obj_drop(NULL) inside it never reaches the verifier's KF_RELEASE check), so there is no load failure. But it is dead code carrying an incorrect technical rationale that invites copying into future programs.

Should both the function and the comment be removed?

---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md

CI run summary: https://github.com/kernel-patches/bpf/actions/runs/31362294949

#2 lore raw
In reply to Chenguang Zhao
Quoted 7 lines · Chenguang Zhao

From: Chenguang Zhao <[email protected]>

Add a microbenchmark that repeatedly calls bpf_kptr_xchg() from an fentry program. Architectures that advertise bpf_jit_supports_ptr_xchg() inline the helper to BPF_XCHG; others keep the helper call. This bench makes it easy to compare the two paths under the same workload.

but do we need a permanent benchmark for this? Is there any valid risk of regressing performance here? Unless there is a good reason to have this benchmark readily available for everyone to periodically run or something, I'm inclined to reject new benchmarks and minimize amount of code maintenance burden for something like this, sorry.

pw-bot: cr

Quoted 8 lines

Usage:
sudo ./bench -d 30 -w 5 -p 1 kptr-xchg --nr_loops 256

---
Example results on LoongArch64 (helper vs inlined):

[...]