Skip to contentSkip to conversation

bpf: Search and add kfuncs in struct_ops prologue and epilogue

PATCH Superseded Amery Hung 2 participants 5 messages Feb 25, 2025

Subsystems (5)

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Amery Hung

author
#0 lore raw

PATCH v4 1/2 bpf: Search and add kfuncs in struct_ops prologue and epilogue

From: Amery Hung <[email protected]>

Currently, add_kfunc_call() is only invoked once before the main verification loop. Therefore, the verifier could not find the bpf_kfunc_btf_tab of a new kfunc call which is not seen in user defined struct_ops operators but introduced in gen_prologue or gen_epilogue during do_misc_fixup(). Fix this by searching kfuncs in the patching instruction buffer and add them to prog->aux->kfunc_tab.

---
 kernel/bpf/verifier.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)
kernel/bpf/verifier.c +24 -1
index 8f1df279e432..212b487fd39d 100644
@@ -3215,6 +3215,21 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 	return res ? &res->func_model : NULL;
 }
 
+static int add_kfunc_in_insns(struct bpf_verifier_env *env,
+			      struct bpf_insn *insn, int cnt)
+{
+	int i, ret;
+
+	for (i = 0; i < cnt; i++, insn++) {
+		if (bpf_pseudo_kfunc_call(insn)) {
+			ret = add_kfunc_call(env, insn->imm, insn->off);
+			if (ret < 0)
+				return ret;
+		}
+	}
+	return 0;
+}
+
 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprog = env->subprog_info;
@@ -20368,7 +20383,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprogs = env->subprog_info;
 	const struct bpf_verifier_ops *ops = env->ops;
-	int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
+	int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
 	const int insn_cnt = env->prog->len;
 	struct bpf_insn *epilogue_buf = env->epilogue_buf;
 	struct bpf_insn *insn_buf = env->insn_buf;
@@ -20397,6 +20412,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 				return -ENOMEM;
 			env->prog = new_prog;
 			delta += cnt - 1;
+
+			ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
+			if (ret < 0)
+				return ret;
 		}
 	}
 
@@ -20417,6 +20436,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 
 			env->prog = new_prog;
 			delta += cnt - 1;
+
+			ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
+			if (ret < 0)
+				return ret;
 		}
 	}
 
Signature · 1 line
2.47.1
author
#1 lore raw

PATCH v4 2/2 selftests/bpf: Test gen_pro/epilogue that generate kfuncs

Test gen_prologue and gen_epilogue that generate kfuncs that have not been seen in the main program.

The main bpf program and return value checks are identical to pro_epilogue.c introduced in commit 47e69431b57a ("selftests/bpf: Test gen_prologue and gen_epilogue"). However, now when bpf_testmod_st_ops detects a program name with prefix "test_kfunc_", it generates slightly different prologue and epilogue: They still add 1000 to args->a in prologue, add 10000 to args->a and set r0 to 2 * args->a in epilogue, but involve kfuncs.

At high level, the alternative version of prologue and epilogue look like this:

cgrp = bpf_cgroup_from_id(0);
if (cgrp)

          bpf_cgroup_release(cgrp);
  else
          /* Perform what original bpf_testmod_st_ops prologue or
           * epilogue does
           */

Since 0 is never a valid cgroup id, the original prologue or epilogue logic will be performed. As a result, the __retval check should expect the exact same return value.

---
 include/linux/filter.h                        | 10 ++
 kernel/bpf/btf.c                              |  1 +
 .../selftests/bpf/prog_tests/pro_epilogue.c   |  2 +
 .../bpf/progs/pro_epilogue_with_kfunc.c       | 88 ++++++++++++++++++
 .../selftests/bpf/test_kmods/bpf_testmod.c    | 92 +++++++++++++++++++
 5 files changed, 193 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c
include/linux/filter.h +10 -0
index a3ea46281595..3ed6eb9e7c73 100644
@@ -469,6 +469,16 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
 		.off   = 0,					\
 		.imm   = BPF_CALL_IMM(FUNC) })
 
+/* Kfunc call */
+
+#define BPF_CALL_KFUNC(OFF, IMM)				\
+	((struct bpf_insn) {					\
+		.code  = BPF_JMP | BPF_CALL,			\
+		.dst_reg = 0,					\
+		.src_reg = BPF_PSEUDO_KFUNC_CALL,		\
+		.off   = OFF,					\
+		.imm   = IMM })
+
 /* Raw code statement block */
 
 #define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
kernel/bpf/btf.c +1 -0
index 7a7e02dcde84..1927d7e53b5a 100644
@@ -606,6 +606,7 @@ s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p)
 	spin_unlock_bh(&btf_idr_lock);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(bpf_find_btf_id);
 
 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
 					       u32 id, u32 *res_id)
tools/testing/selftests/bpf/prog_tests/pro_epilogue.c +2 -0
index 509883e6823a..5d3c00a08a88 100644
@@ -6,6 +6,7 @@
 #include "epilogue_tailcall.skel.h"
 #include "pro_epilogue_goto_start.skel.h"
 #include "epilogue_exit.skel.h"
+#include "pro_epilogue_with_kfunc.skel.h"
 
 struct st_ops_args {
 	__u64 a;
@@ -55,6 +56,7 @@ void test_pro_epilogue(void)
 	RUN_TESTS(pro_epilogue);
 	RUN_TESTS(pro_epilogue_goto_start);
 	RUN_TESTS(epilogue_exit);
+	RUN_TESTS(pro_epilogue_with_kfunc);
 	if (test__start_subtest("tailcall"))
 		test_tailcall();
 }
tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c new file +88 -0

88 lines not on this page · raw message

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +92 -0

116 lines not on this page · raw message

Signature · 1 line
2.47.1
reviewer Acked-by on v4 2/2
#2 lore raw

Re: [PATCH bpf-next v4 2/2] selftests/bpf: Test gen_pro/epilogue that generate kfuncs

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c static void st_ops_unreg(void *kdata, struct bpf_link *link) · line 1503

Quoted 9 lines · Amery Hung
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +5 -0
@@ -1411,6 +1496,13 @@ static void st_ops_unreg(void *kdata, struct bpf_link *link)
 
  static int st_ops_init(struct btf *btf)
  {
+	struct btf *kfunc_btf;
+
+	bpf_cgroup_from_id_id = bpf_find_btf_id("bpf_cgroup_from_id", BTF_KIND_FUNC, &kfunc_btf);
+	bpf_cgroup_release_id = bpf_find_btf_id("bpf_cgroup_release", BTF_KIND_FUNC, &kfunc_btf);
+	if (!bpf_cgroup_from_id_id || !bpf_cgroup_release_id)

Just noticed this. This should be "< 0" check. No need for "== 0" check because "id == 0" is reserved for "void" which is not BTF_KIND_FUNC.

With that,

author
#3 lore raw

Re: [PATCH bpf-next v4 2/2] selftests/bpf: Test gen_pro/epilogue that generate kfuncs

Quoted 4 lines · Martin KaFai Lau
Quoted 2 lines · Amery Hung
(quoted diff) +0 -0
@@ -1411,6 +1496,13 @@ static void st_ops_unreg(void *kdata, struct bpf_link *link)
 
Quoted 11 lines

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c static void st_ops_unreg(void *kdata, struct bpf_link *link) · line 1503

Quoted 7 lines
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +5 -0
  static int st_ops_init(struct btf *btf)
  {
+     struct btf *kfunc_btf;
+
+     bpf_cgroup_from_id_id = bpf_find_btf_id("bpf_cgroup_from_id", BTF_KIND_FUNC, &kfunc_btf);
+     bpf_cgroup_release_id = bpf_find_btf_id("bpf_cgroup_release", BTF_KIND_FUNC, &kfunc_btf);
+     if (!bpf_cgroup_from_id_id || !bpf_cgroup_release_id)

Just noticed this. This should be "< 0" check. No need for "== 0" check because "id == 0" is reserved for "void" which is not BTF_KIND_FUNC.

I should be more careful about this. I will send another version.

Thank you,
Amery

Quoted 3 lines

With that,

reviewer Acked-by on v4 1/2
#4 lore raw
In reply to Amery Hung
Quoted 8 lines · Amery Hung

From: Amery Hung <[email protected]>

Currently, add_kfunc_call() is only invoked once before the main verification loop. Therefore, the verifier could not find the bpf_kfunc_btf_tab of a new kfunc call which is not seen in user defined struct_ops operators but introduced in gen_prologue or gen_epilogue during do_misc_fixup(). Fix this by searching kfuncs in the patching instruction buffer and add them to prog->aux->kfunc_tab.