Skip to contentSkip to conversation

bpf: Search and add kfuncs in struct_ops prologue and epilogue

PATCH Superseded Amery Hung 2 participants 4 messages Feb 21, 2025 – Feb 25, 2025

Subsystems (5)

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Amery Hung

author
#0 lore raw

PATCH v3 1/2 bpf: Search and add kfuncs in struct_ops prologue and epilogue

From: Amery Hung <[email protected]>

Currently, add_kfunc_call() is only invoked once before the main verification loop. Therefore, the verifier could not find the bpf_kfunc_btf_tab of a new kfunc call which is not seen in user defined struct_ops operators but introduced in gen_prologue or gen_epilogue during do_misc_fixup(). Fix this by searching kfuncs in the patching instruction buffer and add them to prog->aux->kfunc_tab.

---
 kernel/bpf/verifier.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)
kernel/bpf/verifier.c +24 -1
index 8f1df279e432..212b487fd39d 100644
@@ -3215,6 +3215,21 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 	return res ? &res->func_model : NULL;
 }
 
+static int add_kfunc_in_insns(struct bpf_verifier_env *env,
+			      struct bpf_insn *insn, int cnt)
+{
+	int i, ret;
+
+	for (i = 0; i < cnt; i++, insn++) {
+		if (bpf_pseudo_kfunc_call(insn)) {
+			ret = add_kfunc_call(env, insn->imm, insn->off);
+			if (ret < 0)
+				return ret;
+		}
+	}
+	return 0;
+}
+
 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprog = env->subprog_info;
@@ -20368,7 +20383,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprogs = env->subprog_info;
 	const struct bpf_verifier_ops *ops = env->ops;
-	int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
+	int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
 	const int insn_cnt = env->prog->len;
 	struct bpf_insn *epilogue_buf = env->epilogue_buf;
 	struct bpf_insn *insn_buf = env->insn_buf;
@@ -20397,6 +20412,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 				return -ENOMEM;
 			env->prog = new_prog;
 			delta += cnt - 1;
+
+			ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
+			if (ret < 0)
+				return ret;
 		}
 	}
 
@@ -20417,6 +20436,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 
 			env->prog = new_prog;
 			delta += cnt - 1;
+
+			ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
+			if (ret < 0)
+				return ret;
 		}
 	}
 
Signature · 1 line
2.47.1
author
#1 lore raw

PATCH v3 2/2 selftests/bpf: Test gen_pro/epilogue that generate kfuncs

Test gen_prologue and gen_epilogue that generate kfuncs that have not been seen in the main program.

The main bpf program and return value checks are identical to pro_epilogue.c introduced in commit 47e69431b57a ("selftests/bpf: Test gen_prologue and gen_epilogue"). However, now when bpf_testmod_st_ops detects a program name with prefix "test_kfunc_", it generates slightly different prologue and epilogue: They still add 1000 to args->a in prologue, add 10000 to args->a and set r0 to 2 * args->a in epilogue, but involve kfuncs.

At high level, the alternative version of prologue and epilogue look like this:

cgrp = bpf_cgroup_from_id(0);
if (cgrp)

          bpf_cgroup_release(cgrp);
  else
          /* Perform what original bpf_testmod_st_ops prologue or
           * epilogue does
           */

Since 0 is never a valid cgroup id, the original prologue or epilogue logic will be performed. As a result, the __retval check should expect the exact same return value.

---
 include/linux/filter.h                        | 10 +++
 kernel/bpf/btf.c                              |  1 +
 .../selftests/bpf/prog_tests/pro_epilogue.c   |  2 +
 .../bpf/progs/pro_epilogue_with_kfunc.c       | 87 ++++++++++++++++++
 .../selftests/bpf/test_kmods/bpf_testmod.c    | 90 +++++++++++++++++++
 5 files changed, 190 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c
include/linux/filter.h +10 -0
index a3ea46281595..3ed6eb9e7c73 100644
@@ -469,6 +469,16 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
 		.off   = 0,					\
 		.imm   = BPF_CALL_IMM(FUNC) })
 
+/* Kfunc call */
+
+#define BPF_CALL_KFUNC(OFF, IMM)				\
+	((struct bpf_insn) {					\
+		.code  = BPF_JMP | BPF_CALL,			\
+		.dst_reg = 0,					\
+		.src_reg = BPF_PSEUDO_KFUNC_CALL,		\
+		.off   = OFF,					\
+		.imm   = IMM })
+
 /* Raw code statement block */
 
 #define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
kernel/bpf/btf.c +1 -0
index 7a7e02dcde84..1927d7e53b5a 100644
@@ -606,6 +606,7 @@ s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p)
 	spin_unlock_bh(&btf_idr_lock);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(bpf_find_btf_id);
 
 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
 					       u32 id, u32 *res_id)
tools/testing/selftests/bpf/prog_tests/pro_epilogue.c +2 -0
index 509883e6823a..5d3c00a08a88 100644
@@ -6,6 +6,7 @@
 #include "epilogue_tailcall.skel.h"
 #include "pro_epilogue_goto_start.skel.h"
 #include "epilogue_exit.skel.h"
+#include "pro_epilogue_with_kfunc.skel.h"
 
 struct st_ops_args {
 	__u64 a;
@@ -55,6 +56,7 @@ void test_pro_epilogue(void)
 	RUN_TESTS(pro_epilogue);
 	RUN_TESTS(pro_epilogue_goto_start);
 	RUN_TESTS(epilogue_exit);
+	RUN_TESTS(pro_epilogue_with_kfunc);
 	if (test__start_subtest("tailcall"))
 		test_tailcall();
 }
tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c new file +87 -0

87 lines not on this page · raw message

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +90 -0

114 lines not on this page · raw message

Signature · 1 line
2.47.1
#2 lore raw

Re: [PATCH bpf-next v3 2/2] selftests/bpf: Test gen_pro/epilogue that generate kfuncs

Quoted 7 lines · Amery Hung
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +7 -0
+static int st_ops_gen_epilogue_with_kfunc(struct bpf_insn *insn_buf, const struct bpf_prog *prog,
+					  s16 ctx_stack_off)
+{
+	struct bpf_insn *insn = insn_buf;
+
+	/* r1 = 0;
+	 * r0 = bpf_cgroup_from_id(r1);

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c static int bpf_test_mod_st_ops__test_pro_epilogue(struct st_ops_args *args) · line 1366

Quoted 10 lines
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +10 -0
+	 * if r0 != 0 goto pc+6;
+	 * r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
+	 * r1 = r1[0]; // r1 will be "struct st_ops *args"
+	 * r6 = r1->a;
+	 * r6 += 10000;
+	 * r1->a = r6;
+	 * goto pc+2
+	 * r1 = r0;
+	 * bpf_cgroup_release(r1);
+	 * r0 = r6;

I think r6 is not initialized on the "r0 != 0" case.

Others lgtm.

author
#3 lore raw

Re: [PATCH bpf-next v3 2/2] selftests/bpf: Test gen_pro/epilogue that generate kfuncs

Quoted 13 lines · Martin KaFai Lau
Quoted 11 lines · Amery Hung
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +11 -0
+static int st_ops_gen_epilogue_with_kfunc(struct bpf_insn *insn_buf, const struct bpf_prog *prog,
+                                       s16 ctx_stack_off)
+{
+     struct bpf_insn *insn = insn_buf;
+
+     /* r1 = 0;
+      * r0 = bpf_cgroup_from_id(r1);
+      * if r0 != 0 goto pc+6;
+      * r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
+      * r1 = r1[0]; // r1 will be "struct st_ops *args"
+      * r6 = r1->a;
Quoted 9 lines

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c static int bpf_test_mod_st_ops__test_pro_epilogue(struct st_ops_args *args) · line 1366

Quoted 6 lines
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +6 -0
+      * r6 += 10000;
+      * r1->a = r6;
+      * goto pc+2
+      * r1 = r0;
+      * bpf_cgroup_release(r1);
+      * r0 = r6;

I think r6 is not initialized on the "r0 != 0" case.

I will insert r6 = 0 after the first epilogue instruction.

Thanks,
Amery

Quoted 2 lines

Others lgtm.