Skip to contentSkip to conversation

support bpf_fastcall patterns for calls to kfuncs

PATCH Superseded Eduard Zingerman 3 participants 15 messages Aug 17, 2024 – Aug 22, 2024

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Eduard Zingerman

author maintainer
#0 lore raw

v2 cover support bpf_fastcall patterns for calls to kfuncs

As an extension of [1], allow bpf_fastcall patterns for kfuncs:
- pattern rules are the same as for helpers;
- spill/fill removal is allowed only for kfuncs listed in the is_fastcall_kfunc_call (under assumption that such kfuncs would always be members of special_kfunc_list).

Allow bpf_fastcall rewrite for bpf_cast_to_kern_ctx() and bpf_rdonly_cast() in order to conjure selftests for this feature.

After this patch-set verifier would rewrite the program below:

r2 = 1
*(u64 *)(r10 - 32) = r2
call %[bpf_cast_to_kern_ctx]
r2 = *(u64 *)(r10 - 32)
r0 = r2;"

As follows:

r2 = 1 /* spill/fill at r10[-32] is removed */
r0 = r1 /* replacement for bpf_cast_to_kern_ctx() */ r0 = r2
exit

Also, attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [2]). This patch-set replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] no_caller_saved_registers attribute for helper calls

    https://lore.kernel.org/bpf/[email protected]/
[2] [BPF] introduce __attribute__((bpf_fastcall))
    https://github.com/llvm/llvm-project/pull/101228

Changes v1->v2:
- added two patches replacing all mentions of nocsr by bpf_fastcall (suggested by Andrii);
- removed KF_NOCSR flag (suggested by Yonghong).

v1: https://lore.kernel.org/bpf/[email protected]/

Eduard Zingerman (5):
bpf: rename nocsr -> bpf_fastcall in verifier selftests/bpf: rename nocsr -> bpf_fastcall in selftests
bpf: support bpf_fastcall patterns for kfuncs
bpf: allow bpf_fastcall for bpf_cast_to_kern_ctx and bpf_rdonly_cast selftests/bpf: check if bpf_fastcall is recognized for kfuncs

 include/linux/bpf.h                           |   6 +-
 include/linux/bpf_verifier.h                  |  18 +-
 kernel/bpf/helpers.c                          |   2 +-
 kernel/bpf/verifier.c                         | 183 +++++++++++-------
 .../selftests/bpf/prog_tests/verifier.c       |   4 +-
 ...rifier_nocsr.c => verifier_bpf_fastcall.c} |  76 ++++++--
 6 files changed, 188 insertions(+), 101 deletions(-)
 rename tools/testing/selftests/bpf/progs/{verifier_nocsr.c => verifier_bpf_fastcall.c} (90%)
Signature · 1 line
2.45.2
author maintainer
#1 lore raw

PATCH v2 1/5 bpf: rename nocsr -> bpf_fastcall in verifier

Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] https://github.com/llvm/llvm-project/pull/101228

---
 include/linux/bpf.h          |   6 +-
 include/linux/bpf_verifier.h |  18 ++---
 kernel/bpf/helpers.c         |   2 +-
 kernel/bpf/verifier.c        | 143 +++++++++++++++++------------------
 4 files changed, 84 insertions(+), 85 deletions(-)
include/linux/bpf.h +3 -3
index b9425e410bcb..b9ac23adf849 100644
@@ -808,12 +808,12 @@ struct bpf_func_proto {
 	bool gpl_only;
 	bool pkt_access;
 	bool might_sleep;
-	/* set to true if helper follows contract for gcc/llvm
-	 * attribute no_caller_saved_registers:
+	/* set to true if helper follows contract for llvm
+	 * attribute bpf_fastcall:
 	 * - void functions do not scratch r0
 	 * - functions taking N arguments scratch only registers r1-rN
 	 */
-	bool allow_nocsr;
+	bool allow_fastcall;
 	enum bpf_return_type ret_type;
 	union {
 		struct {
include/linux/bpf_verifier.h +9 -9
index 5cea15c81b8a..634a302a39e3 100644
@@ -577,13 +577,13 @@ struct bpf_insn_aux_data {
 	bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
 	u8 alu_state; /* used in combination with alu_limit */
 	/* true if STX or LDX instruction is a part of a spill/fill
-	 * pattern for a no_caller_saved_registers call.
+	 * pattern for a bpf_fastcall call.
 	 */
-	u8 nocsr_pattern:1;
+	u8 fastcall_pattern:1;
 	/* for CALL instructions, a number of spill/fill pairs in the
-	 * no_caller_saved_registers pattern.
+	 * bpf_fastcall pattern.
 	 */
-	u8 nocsr_spills_num:3;
+	u8 fastcall_spills_num:3;
 
 	/* below fields are initialized once */
 	unsigned int orig_idx; /* original instruction index */
@@ -653,10 +653,10 @@ struct bpf_subprog_info {
 	u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
 	u16 stack_depth; /* max. stack depth used by this function */
 	u16 stack_extra;
-	/* offsets in range [stack_depth .. nocsr_stack_off)
-	 * are used for no_caller_saved_registers spills and fills.
+	/* offsets in range [stack_depth .. fastcall_stack_off)
+	 * are used for bpf_fastcall spills and fills.
 	 */
-	s16 nocsr_stack_off;
+	s16 fastcall_stack_off;
 	bool has_tail_call: 1;
 	bool tail_call_reachable: 1;
 	bool has_ld_abs: 1;
@@ -664,8 +664,8 @@ struct bpf_subprog_info {
 	bool is_async_cb: 1;
 	bool is_exception_cb: 1;
 	bool args_cached: 1;
-	/* true if nocsr stack region is used by functions that can't be inlined */
-	bool keep_nocsr_stack: 1;
+	/* true if bpf_fastcall stack region is used by functions that can't be inlined */
+	bool keep_fastcall_stack: 1;
 
 	u8 arg_cnt;
 	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
kernel/bpf/helpers.c +1 -1
index d02ae323996b..9595a758645c 100644
@@ -158,7 +158,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
 	.func		= bpf_get_smp_processor_id,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.allow_nocsr	= true,
+	.allow_fastcall	= true,
 };
 
 BPF_CALL_0(bpf_get_numa_node_id)
kernel/bpf/verifier.c +71 -72

318 lines not on this page · raw message

Signature · 1 line
2.45.2
reviewer Acked-by on v2 1/5
#2 lore raw

Re: [PATCH bpf-next v2 1/5] bpf: rename nocsr -> bpf_fastcall in verifier

Quoted 8 lines · Eduard Zingerman

Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] https://github.com/llvm/llvm-project/pull/101228

reviewer
#3 lore raw

Re: [PATCH bpf-next v2 1/5] bpf: rename nocsr -> bpf_fastcall in verifier

In reply to Eduard Zingerman
Quoted 6 lines · Eduard Zingerman

Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] https://github.com/llvm/llvm-project/pull/101228

Let us change this link to

  https://github.com/llvm/llvm-project/pull/105417

Quoted 8 lines
---
  include/linux/bpf.h          |   6 +-
  include/linux/bpf_verifier.h |  18 ++---
  kernel/bpf/helpers.c         |   2 +-
  kernel/bpf/verifier.c        | 143 +++++++++++++++++------------------
  4 files changed, 84 insertions(+), 85 deletions(-)

[...]

author maintainer
#4 lore raw

Re: [PATCH bpf-next v2 1/5] bpf: rename nocsr -> bpf_fastcall in verifier

Quoted 11 lines · Yonghong Song
Quoted 6 lines · Eduard Zingerman

Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] https://github.com/llvm/llvm-project/pull/101228

Let us change this link to

  https://github.com/llvm/llvm-project/pull/105417

Sure, will do, thank you for the review.

[...]

author maintainer
#5 lore raw

PATCH v2 2/5 selftests/bpf: rename nocsr -> bpf_fastcall in selftests

In reply to Eduard Zingerman

Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and selftests parts in sync.

[1] https://github.com/llvm/llvm-project/pull/101228

---
 .../selftests/bpf/prog_tests/verifier.c       |  4 +--
 ...rifier_nocsr.c => verifier_bpf_fastcall.c} | 26 +++++++++----------
 2 files changed, 15 insertions(+), 15 deletions(-)
 rename tools/testing/selftests/bpf/progs/{verifier_nocsr.c => verifier_bpf_fastcall.c} (95%)
tools/testing/selftests/bpf/prog_tests/verifier.c +2 -2
index f8f546eba488..c3b5a4164b92 100644
@@ -53,7 +53,7 @@
 #include "verifier_movsx.skel.h"
 #include "verifier_netfilter_ctx.skel.h"
 #include "verifier_netfilter_retcode.skel.h"
-#include "verifier_nocsr.skel.h"
+#include "verifier_bpf_fastcall.skel.h"
 #include "verifier_or_jmp32_k.skel.h"
 #include "verifier_precision.skel.h"
 #include "verifier_prevent_map_lookup.skel.h"
@@ -176,7 +176,7 @@ void test_verifier_meta_access(void)          { RUN(verifier_meta_access); }
 void test_verifier_movsx(void)                 { RUN(verifier_movsx); }
 void test_verifier_netfilter_ctx(void)        { RUN(verifier_netfilter_ctx); }
 void test_verifier_netfilter_retcode(void)    { RUN(verifier_netfilter_retcode); }
-void test_verifier_nocsr(void)                { RUN(verifier_nocsr); }
+void test_verifier_bpf_fastcall(void)         { RUN(verifier_bpf_fastcall); }
 void test_verifier_or_jmp32_k(void)           { RUN(verifier_or_jmp32_k); }
 void test_verifier_precision(void)            { RUN(verifier_precision); }
 void test_verifier_prevent_map_lookup(void)   { RUN(verifier_prevent_map_lookup); }
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c renamed from tools/testing/selftests/bpf/progs/verifier_nocsr.c +13 -13
similarity index 95%
rename from tools/testing/selftests/bpf/progs/verifier_nocsr.c
rename to tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
index a7fe277e5167..f75cd5e3fffe 100644
@@ -39,7 +39,7 @@ __naked void simple(void)
 	: __clobber_all);
 }
 
-/* The logic for detecting and verifying nocsr pattern is the same for
+/* The logic for detecting and verifying bpf_fastcall pattern is the same for
  * any arch, however x86 differs from arm64 or riscv64 in a way
  * bpf_get_smp_processor_id is rewritten:
  * - on x86 it is done by verifier
@@ -52,7 +52,7 @@ __naked void simple(void)
  *
  * It is really desirable to check instruction indexes in the xlated
  * patterns, so add this canary test to check that function rewrite by
- * jit is correctly processed by nocsr logic, keep the rest of the
+ * jit is correctly processed by bpf_fastcall logic, keep the rest of the
  * tests as x86.
  */
 SEC("raw_tp")
@@ -430,7 +430,7 @@ __naked static void bad_write_in_subprog_aux(void)
 {
 	asm volatile (
 	"r0 = 1;"
-	"*(u64 *)(r1 - 0) = r0;"	/* invalidates nocsr contract for caller: */
+	"*(u64 *)(r1 - 0) = r0;"	/* invalidates bpf_fastcall contract for caller: */
 	"exit;"				/* caller stack at -8 used outside of the pattern */
 	::: __clobber_all);
 }
@@ -445,7 +445,7 @@ __naked void bad_helper_write(void)
 {
 	asm volatile (
 	"r1 = 1;"
-	/* nocsr pattern with stack offset -8 */
+	/* bpf_fastcall pattern with stack offset -8 */
 	"*(u64 *)(r10 - 8) = r1;"
 	"call %[bpf_get_smp_processor_id];"
 	"r1 = *(u64 *)(r10 - 8);"
@@ -453,7 +453,7 @@ __naked void bad_helper_write(void)
 	"r1 += -8;"
 	"r2 = 1;"
 	"r3 = 42;"
-	/* read dst is fp[-8], thus nocsr rewrite not applied */
+	/* read dst is fp[-8], thus bpf_fastcall rewrite not applied */
 	"call %[bpf_probe_read_kernel];"
 	"exit;"
 	:
@@ -553,7 +553,7 @@ __arch_x86_64
 __log_level(4) __msg("stack depth 8")
 __xlated("2: r0 = &(void __percpu *)(r0)")
 __success
-__naked void helper_call_does_not_prevent_nocsr(void)
+__naked void helper_call_does_not_prevent_bpf_fastcall(void)
 {
 	asm volatile (
 	"r1 = 1;"
@@ -640,7 +640,7 @@ __naked int bpf_loop_interaction1(void)
 {
 	asm volatile (
 	"r1 = 1;"
-	/* nocsr stack region at -16, but could be removed */
+	/* bpf_fastcall stack region at -16, but could be removed */
 	"*(u64 *)(r10 - 16) = r1;"
 	"call %[bpf_get_smp_processor_id];"
 	"r1 = *(u64 *)(r10 - 16);"
@@ -680,7 +680,7 @@ __naked int bpf_loop_interaction2(void)
 {
 	asm volatile (
 	"r1 = 42;"
-	/* nocsr stack region at -16, cannot be removed */
+	/* bpf_fastcall stack region at -16, cannot be removed */
 	"*(u64 *)(r10 - 16) = r1;"
 	"call %[bpf_get_smp_processor_id];"
 	"r1 = *(u64 *)(r10 - 16);"
@@ -710,8 +710,8 @@ __msg("stack depth 512+0")
 __xlated("r0 = &(void __percpu *)(r0)")
 __success
 /* cumulative_stack_depth() stack usage is MAX_BPF_STACK,
- * called subprogram uses an additional slot for nocsr spill/fill,
- * since nocsr spill/fill could be removed the program still fits
+ * called subprogram uses an additional slot for bpf_fastcall spill/fill,
+ * since bpf_fastcall spill/fill could be removed the program still fits
  * in MAX_BPF_STACK and should be accepted.
  */
 __naked int cumulative_stack_depth(void)
@@ -749,7 +749,7 @@ __xlated("3: r0 = &(void __percpu *)(r0)")
 __xlated("4: r0 = *(u32 *)(r0 +0)")
 __xlated("5: exit")
 __success
-__naked int nocsr_max_stack_ok(void)
+__naked int bpf_fastcall_max_stack_ok(void)
 {
 	asm volatile(
 	"r1 = 42;"
@@ -771,7 +771,7 @@ __arch_x86_64
 __log_level(4)
 __msg("stack depth 520")
 __failure
-__naked int nocsr_max_stack_fail(void)
+__naked int bpf_fastcall_max_stack_fail(void)
 {
 	asm volatile(
 	"r1 = 42;"
@@ -779,7 +779,7 @@ __naked int nocsr_max_stack_fail(void)
 	"*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
 	"call %[bpf_get_smp_processor_id];"
 	"r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
-	/* call to prandom blocks nocsr rewrite */
+	/* call to prandom blocks bpf_fastcall rewrite */
 	"*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
 	"call %[bpf_get_prandom_u32];"
 	"r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
Signature · 1 line
2.45.2
reviewer Acked-by on v2 2/5
#6 lore raw

Re: [PATCH bpf-next v2 2/5] selftests/bpf: rename nocsr -> bpf_fastcall in selftests

Quoted 6 lines · Eduard Zingerman

Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and selftests parts in sync.

[1] https://github.com/llvm/llvm-project/pull/101228

Let us change the link to https://github.com/llvm/llvm-project/pull/105417

Quoted 2 lines
author maintainer
#7 lore raw

PATCH v2 3/5 bpf: support bpf_fastcall patterns for kfuncs

In reply to Eduard Zingerman

Recognize bpf_fastcall patterns around kfunc calls.
For example, suppose bpf_cast_to_kern_ctx() follows bpf_fastcall contract (which it does), in such a case allow verifier to rewrite BPF program below:

  r2 = 1;
  *(u64 *)(r10 - 32) = r2;
  call %[bpf_cast_to_kern_ctx];
  r2 = *(u64 *)(r10 - 32);
  r0 = r2;

By removing the spill/fill pair:

  r2 = 1;
  call %[bpf_cast_to_kern_ctx];
  r0 = r2;
---
 kernel/bpf/verifier.c | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)
kernel/bpf/verifier.c +36 -1
index b18a21bb5e6e..5dafcfff729e 100644
@@ -16112,7 +16112,7 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,
  */
 static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
 {
-	u8 mask;
+	u32 mask;
 	int i;
 
 	mask = 0;
@@ -16140,6 +16140,28 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
 	}
 }
 
+/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
+static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
+{
+	const struct btf_param *params;
+	u32 vlen, i, mask;
+
+	params = btf_params(meta->func_proto);
+	vlen = btf_type_vlen(meta->func_proto);
+	mask = 0;
+	if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
+		mask |= BIT(BPF_REG_0);
+	for (i = 0; i < vlen; ++i)
+		mask |= BIT(BPF_REG_1 + i);
+	return mask;
+}
+
+/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
+static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
+{
+	return false;
+}
+
 /* LLVM define a bpf_fastcall function attribute.
  * This attribute means that function scratches only some of
  * the caller saved registers defined by ABI.
@@ -16237,6 +16259,19 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
 				  bpf_jit_inlines_helper_call(call->imm));
 	}
 
+	if (bpf_pseudo_kfunc_call(call)) {
+		struct bpf_kfunc_call_arg_meta meta;
+		int err;
+
+		err = fetch_kfunc_meta(env, call, &meta, NULL);
+		if (err < 0)
+			/* error would be reported later */
+			return;
+
+		clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);
+		can_be_inlined = is_fastcall_kfunc_call(&meta);
+	}
+
 	if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)
 		return;
 
Signature · 1 line
2.45.2
#8 lore raw

Re: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs

kernel test robot noticed the following build warnings:

Hi Eduard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Eduard-Zingerman/bpf-rename-nocsr-bpf_fastcall-in-verifier/20240817-095340
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20240817015140.1039351-4-eddyz87%40gmail.com
patch subject: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs
config: arc-randconfig-002-20240817 (https://download.01.org/0day-ci/archive/20240818/[email protected]/config)
compiler: arceb-elf-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240818/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

kernel/bpf/verifier.c: In function 'kfunc_fastcall_clobber_mask':

>> kernel/bpf/verifier.c:16146:33: warning: variable 'params' set but not used [-Wunused-but-set-variable]

16146 | const struct btf_param *params;
| ^~~~~~

vim +/params +16146 kernel/bpf/verifier.c

16142
16143 /* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
16144 static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
16145 {
16146 const struct btf_param *params;
16147 u32 vlen, i, mask;
16148
16149 params = btf_params(meta->func_proto);
16150 vlen = btf_type_vlen(meta->func_proto);
16151 mask = 0;
16152 if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
16153 mask |= BIT(BPF_REG_0);
16154 for (i = 0; i < vlen; ++i)
16155 mask |= BIT(BPF_REG_1 + i);
16156 return mask;
16157 }
16158

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
author maintainer
#9 lore raw

Re: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs

Quoted 19 lines · kernel test robot

Hi Eduard,

kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Eduard-Zingerman/bpf-rename-nocsr-bpf_fastcall-in-verifier/20240817-095340
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20240817015140.1039351-4-eddyz87%40gmail.com
patch subject: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs
config: arc-randconfig-002-20240817 (https://download.01.org/0day-ci/archive/20240818/[email protected]/config)
compiler: arceb-elf-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240818/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags

Quoted 2 lines
Quoted 7 lines

All warnings (new ones prefixed by >>):

kernel/bpf/verifier.c: In function 'kfunc_fastcall_clobber_mask':

Quoted 1 line
Quoted 1 line

kernel/bpf/verifier.c:16146:33: warning: variable 'params' set but not used [-Wunused-but-set-variable]

16146 | const struct btf_param *params;
| ^~~~~~

Missed this warning.
The params variable is not needed indeed.
Will wait for comments before resubmitting.

[...]

reviewer Acked-by on v2 3/5
#10 lore raw

Re: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs

In reply to Eduard Zingerman
Quoted 11 lines · Eduard Zingerman

Recognize bpf_fastcall patterns around kfunc calls.
For example, suppose bpf_cast_to_kern_ctx() follows bpf_fastcall contract (which it does), in such a case allow verifier to rewrite BPF program below:

   r2 = 1;
   *(u64 *)(r10 - 32) = r2;
   call %[bpf_cast_to_kern_ctx];
   r2 = *(u64 *)(r10 - 32);
   r0 = r2;
Quoted 7 lines

By removing the spill/fill pair:

   r2 = 1;
   call %[bpf_cast_to_kern_ctx];
   r0 = r2;
author maintainer
#11 lore raw

PATCH v2 4/5 bpf: allow bpf_fastcall for bpf_cast_to_kern_ctx and bpf_rdonly_cast

In reply to Eduard Zingerman

do_misc_fixups() relaces bpf_cast_to_kern_ctx() and bpf_rdonly_cast() by a single instruction "r0 = r1". This follows bpf_fastcall contract. This commit allows bpf_fastcall pattern rewrite for these two functions in order to use them in bpf_fastcall selftests.

---
 kernel/bpf/verifier.c | 3 +++
 1 file changed, 3 insertions(+)
kernel/bpf/verifier.c +3 -0
index 5dafcfff729e..720163772719 100644
@@ -16159,6 +16159,9 @@ static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
 /* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
 static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
 {
+	if (meta->btf == btf_vmlinux)
+		return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
+		       meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];
 	return false;
 }
 
Signature · 1 line
2.45.2
reviewer Acked-by on v2 4/5
#12 lore raw

Re: [PATCH bpf-next v2 4/5] bpf: allow bpf_fastcall for bpf_cast_to_kern_ctx and bpf_rdonly_cast

Quoted 6 lines · Eduard Zingerman

do_misc_fixups() relaces bpf_cast_to_kern_ctx() and bpf_rdonly_cast() by a single instruction "r0 = r1". This follows bpf_fastcall contract. This commit allows bpf_fastcall pattern rewrite for these two functions in order to use them in bpf_fastcall selftests.

author maintainer
#13 lore raw

PATCH v2 5/5 selftests/bpf: check if bpf_fastcall is recognized for kfuncs

In reply to Eduard Zingerman

Use kfunc_bpf_cast_to_kern_ctx() and kfunc_bpf_rdonly_cast() to verify that bpf_fastcall pattern is recognized for kfunc calls.

---
 .../bpf/progs/verifier_bpf_fastcall.c         | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +50 -0
index f75cd5e3fffe..97c2420ccb38 100644
@@ -2,8 +2,11 @@
 
 #include <linux/bpf.h>
 #include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
 #include "../../../include/linux/filter.h"
 #include "bpf_misc.h"
+#include <stdbool.h>
+#include "bpf_kfuncs.h"
 
 SEC("raw_tp")
 __arch_x86_64
@@ -793,4 +796,51 @@ __naked int bpf_fastcall_max_stack_fail(void)
 	);
 }
 
+SEC("cgroup/getsockname_unix")
+__xlated("0: r2 = 1")
+/* bpf_cast_to_kern_ctx is replaced by a single assignment */
+__xlated("1: r0 = r1")
+__xlated("2: r0 = r2")
+__xlated("3: exit")
+__success
+__naked void kfunc_bpf_cast_to_kern_ctx(void)
+{
+	asm volatile (
+	"r2 = 1;"
+	"*(u64 *)(r10 - 32) = r2;"
+	"call %[bpf_cast_to_kern_ctx];"
+	"r2 = *(u64 *)(r10 - 32);"
+	"r0 = r2;"
+	"exit;"
+	:
+	: __imm(bpf_cast_to_kern_ctx)
+	: __clobber_all);
+}
+
+SEC("raw_tp")
+__xlated("3: r3 = 1")
+/* bpf_rdonly_cast is replaced by a single assignment */
+__xlated("4: r0 = r1")
+__xlated("5: r0 = r3")
+void kfunc_bpf_rdonly_cast(void)
+{
+	asm volatile (
+	"r2 = %[btf_id];"
+	"r3 = 1;"
+	"*(u64 *)(r10 - 32) = r3;"
+	"call %[bpf_rdonly_cast];"
+	"r3 = *(u64 *)(r10 - 32);"
+	"r0 = r3;"
+	:
+	: __imm(bpf_rdonly_cast),
+	 [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr))
+	: __clobber_common);
+}
+
+void kfunc_root(void)
+{
+	bpf_cast_to_kern_ctx(0);
+	bpf_rdonly_cast(0, 0);
+}
+
 char _license[] SEC("license") = "GPL";
Signature · 1 line
2.45.2
reviewer Acked-by on v2 5/5
#14 lore raw

Re: [PATCH bpf-next v2 5/5] selftests/bpf: check if bpf_fastcall is recognized for kfuncs

Quoted 4 lines · Eduard Zingerman

Use kfunc_bpf_cast_to_kern_ctx() and kfunc_bpf_rdonly_cast() to verify that bpf_fastcall pattern is recognized for kfunc calls.

LGTM with a nit below.

Quoted 56 lines
---
  .../bpf/progs/verifier_bpf_fastcall.c         | 50 +++++++++++++++++++
  1 file changed, 50 insertions(+)
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +35 -0
index f75cd5e3fffe..97c2420ccb38 100644
@@ -2,8 +2,11 @@
 
  #include <linux/bpf.h>
  #include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
  #include "../../../include/linux/filter.h"
  #include "bpf_misc.h"
+#include <stdbool.h>
+#include "bpf_kfuncs.h"
 
  SEC("raw_tp")
  __arch_x86_64
@@ -793,4 +796,51 @@ __naked int bpf_fastcall_max_stack_fail(void)
  	);
  }
 
+SEC("cgroup/getsockname_unix")
+__xlated("0: r2 = 1")
+/* bpf_cast_to_kern_ctx is replaced by a single assignment */
+__xlated("1: r0 = r1")
+__xlated("2: r0 = r2")
+__xlated("3: exit")
+__success
+__naked void kfunc_bpf_cast_to_kern_ctx(void)
+{
+	asm volatile (
+	"r2 = 1;"
+	"*(u64 *)(r10 - 32) = r2;"
+	"call %[bpf_cast_to_kern_ctx];"
+	"r2 = *(u64 *)(r10 - 32);"
+	"r0 = r2;"
+	"exit;"
+	:
+	: __imm(bpf_cast_to_kern_ctx)
+	: __clobber_all);
+}
+
+SEC("raw_tp")
+__xlated("3: r3 = 1")
+/* bpf_rdonly_cast is replaced by a single assignment */
+__xlated("4: r0 = r1")
+__xlated("5: r0 = r3")
+void kfunc_bpf_rdonly_cast(void)
+{
+	asm volatile (
+	"r2 = %[btf_id];"
+	"r3 = 1;"
+	"*(u64 *)(r10 - 32) = r3;"

tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c __naked int bpf_fastcall_max_stack_fail(void) · line 840

Quoted 10 lines
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +10 -0
+	"call %[bpf_rdonly_cast];"
+	"r3 = *(u64 *)(r10 - 32);"
+	"r0 = r3;"
+	:
+	: __imm(bpf_rdonly_cast),
+	 [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr))
+	: __clobber_common);
+}
+
+void kfunc_root(void)

It would be good to add some comments for this function to indicate that this func intends to enable BTF generation for kfuncs bpf_cast_to_kern_ctx() and bpf_rdonly_cast() which are necessary for the above two programs.

Quoted 6 lines
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +5 -0
+{
+	bpf_cast_to_kern_ctx(0);
+	bpf_rdonly_cast(0, 0);
+}
+
  char _license[] SEC("license") = "GPL";