support bpf_fastcall patterns for calls to kfuncs
Subsystems (4)
Eduard Zingerman
maintainerv2 cover support bpf_fastcall patterns for calls to kfuncs
As an extension of [1], allow bpf_fastcall patterns for kfuncs:
- pattern rules are the same as for helpers;
- spill/fill removal is allowed only for kfuncs listed in the is_fastcall_kfunc_call (under assumption that such kfuncs would always be members of special_kfunc_list).
Allow bpf_fastcall rewrite for bpf_cast_to_kern_ctx() and bpf_rdonly_cast() in order to conjure selftests for this feature.
After this patch-set verifier would rewrite the program below:
r2 = 1
*(u64 *)(r10 - 32) = r2
call %[bpf_cast_to_kern_ctx]
r2 = *(u64 *)(r10 - 32)
r0 = r2;"
As follows:
r2 = 1 /* spill/fill at r10[-32] is removed */
r0 = r1 /* replacement for bpf_cast_to_kern_ctx() */ r0 = r2
exit
Also, attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [2]). This patch-set replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.
[1] no_caller_saved_registers attribute for helper calls
https://lore.kernel.org/bpf/[email protected]/ [2] [BPF] introduce __attribute__((bpf_fastcall)) https://github.com/llvm/llvm-project/pull/101228
Changes v1->v2:
- added two patches replacing all mentions of nocsr by bpf_fastcall (suggested by Andrii);
- removed KF_NOCSR flag (suggested by Yonghong).
v1: https://lore.kernel.org/bpf/[email protected]/
Eduard Zingerman (5):
bpf: rename nocsr -> bpf_fastcall in verifier selftests/bpf: rename nocsr -> bpf_fastcall in selftests
bpf: support bpf_fastcall patterns for kfuncs
bpf: allow bpf_fastcall for bpf_cast_to_kern_ctx and bpf_rdonly_cast selftests/bpf: check if bpf_fastcall is recognized for kfuncs
include/linux/bpf.h | 6 +-
include/linux/bpf_verifier.h | 18 +-
kernel/bpf/helpers.c | 2 +-
kernel/bpf/verifier.c | 183 +++++++++++-------
.../selftests/bpf/prog_tests/verifier.c | 4 +-
...rifier_nocsr.c => verifier_bpf_fastcall.c} | 76 ++++++--
6 files changed, 188 insertions(+), 101 deletions(-)
rename tools/testing/selftests/bpf/progs/{verifier_nocsr.c => verifier_bpf_fastcall.c} (90%)Signature · 1 line
2.45.2
Eduard Zingerman
maintainerPATCH v2 1/5 bpf: rename nocsr -> bpf_fastcall in verifier
Attribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and Kernel parts in sync.
[1] https://github.com/llvm/llvm-project/pull/101228
--- include/linux/bpf.h | 6 +- include/linux/bpf_verifier.h | 18 ++--- kernel/bpf/helpers.c | 2 +- kernel/bpf/verifier.c | 143 +++++++++++++++++------------------ 4 files changed, 84 insertions(+), 85 deletions(-)
include/linux/bpf.h +3 -3
@@ -808,12 +808,12 @@ struct bpf_func_proto {bool gpl_only;bool pkt_access;bool might_sleep; -/* set to true if helper follows contract for gcc/llvm -* attribute no_caller_saved_registers: +/* set to true if helper follows contract for llvm +* attribute bpf_fastcall: * - void functions do not scratch r0 * - functions taking N arguments scratch only registers r1-rN */ -bool allow_nocsr; +bool allow_fastcall;enum bpf_return_type ret_type; union { struct {
include/linux/bpf_verifier.h +9 -9
@@ -577,13 +577,13 @@ struct bpf_insn_aux_data {bool call_with_percpu_alloc_ptr;/* {this,per}_cpu_ptr() with prog percpu alloc */ u8 alu_state;/* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill -* pattern for a no_caller_saved_registers call. +* pattern for a bpf_fastcall call. */ - u8 nocsr_pattern:1; + u8 fastcall_pattern:1;/* for CALL instructions, a number of spill/fill pairs in the -* no_caller_saved_registers pattern. +* bpf_fastcall pattern. */ - u8 nocsr_spills_num:3; + u8 fastcall_spills_num:3;/* below fields are initialized once */ unsigned int orig_idx;/* original instruction index */
@@ -653,10 +653,10 @@ struct bpf_subprog_info {u32 linfo_idx;/* The idx to the main_prog->aux->linfo */ u16 stack_depth;/* max. stack depth used by this function */ u16 stack_extra; -/* offsets in range [stack_depth .. nocsr_stack_off) -* are used for no_caller_saved_registers spills and fills. +/* offsets in range [stack_depth .. fastcall_stack_off) +* are used for bpf_fastcall spills and fills. */ -s16 nocsr_stack_off; +s16 fastcall_stack_off; bool has_tail_call: 1; bool tail_call_reachable: 1; bool has_ld_abs: 1;
@@ -664,8 +664,8 @@ struct bpf_subprog_info { bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; -/* true if nocsr stack region is used by functions that can't be inlined */ - bool keep_nocsr_stack: 1; +/* true if bpf_fastcall stack region is used by functions that can't be inlined */ + bool keep_fastcall_stack: 1;u8 arg_cnt;struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS ];
kernel/bpf/helpers.c +1 -1
@@ -158,7 +158,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = { .func = bpf_get_smp_processor_id, .gpl_only = false, .ret_type = RET_INTEGER, - .allow_nocsr = true, + .allow_fastcall = true, };BPF_CALL_0 (bpf_get_numa_node_id)
kernel/bpf/verifier.c +71 -72
318 lines not on this page · raw message
Signature · 1 line
2.45.2
Re: [PATCH bpf-next v2 1/5] bpf: rename nocsr -> bpf_fastcall in verifier
Quoted 8 lines · Eduard Zingerman
Attribute used by LLVM implementation of the feature had been changed from
no_caller_saved_registerstobpf_fastcall(see [1]). This commit replaces references to nocsr by references tobpf_fastcallto keep LLVM and Kernel parts in sync.[1] https://github.com/llvm/llvm-project/pull/101228
Signed-off-by Eduard Zingerman <[email protected]>
Yonghong Song
reviewerRe: [PATCH bpf-next v2 1/5] bpf: rename nocsr -> bpf_fastcall in verifier
In reply to Eduard ZingermanQuoted 6 lines · Eduard Zingerman
Attribute used by LLVM implementation of the feature had been changed from
no_caller_saved_registerstobpf_fastcall(see [1]). This commit replaces references to nocsr by references tobpf_fastcallto keep LLVM and Kernel parts in sync.
Let us change this link to
https://github.com/llvm/llvm-project/pull/105417
Quoted 8 lines
Signed-off-by Eduard Zingerman <[email protected]>--- include/linux/bpf.h | 6 +- include/linux/bpf_verifier.h | 18 ++--- kernel/bpf/helpers.c | 2 +- kernel/bpf/verifier.c | 143 +++++++++++++++++------------------ 4 files changed, 84 insertions(+), 85 deletions(-)
[...]
Eduard Zingerman
maintainerRe: [PATCH bpf-next v2 1/5] bpf: rename nocsr -> bpf_fastcall in verifier
Quoted 11 lines · Yonghong Song
Quoted 6 lines · Eduard Zingerman
Attribute used by LLVM implementation of the feature had been changed from
no_caller_saved_registerstobpf_fastcall(see [1]). This commit replaces references to nocsr by references tobpf_fastcallto keep LLVM and Kernel parts in sync.Let us change this link to
Sure, will do, thank you for the review.
[...]
Eduard Zingerman
maintainerPATCH v2 2/5 selftests/bpf: rename nocsr -> bpf_fastcall in selftests
In reply to Eduard ZingermanAttribute used by LLVM implementation of the feature had been changed from no_caller_saved_registers to bpf_fastcall (see [1]). This commit replaces references to nocsr by references to bpf_fastcall to keep LLVM and selftests parts in sync.
[1] https://github.com/llvm/llvm-project/pull/101228
---
.../selftests/bpf/prog_tests/verifier.c | 4 +--
...rifier_nocsr.c => verifier_bpf_fastcall.c} | 26 +++++++++----------
2 files changed, 15 insertions(+), 15 deletions(-)
rename tools/testing/selftests/bpf/progs/{verifier_nocsr.c => verifier_bpf_fastcall.c} (95%)tools/testing/selftests/bpf/prog_tests/verifier.c +2 -2
@@ -53,7 +53,7 @@#include "verifier_movsx.skel.h" #include "verifier_netfilter_ctx.skel.h" #include "verifier_netfilter_retcode.skel.h" -#include "verifier_nocsr.skel.h" +#include "verifier_bpf_fastcall.skel.h" #include "verifier_or_jmp32_k.skel.h" #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h"
@@ -176,7 +176,7 @@ void test_verifier_meta_access(void) { RUN(verifier_meta_access); }void test_verifier_movsx (void ) {RUN (verifier_movsx); }void test_verifier_netfilter_ctx (void ) {RUN (verifier_netfilter_ctx); }void test_verifier_netfilter_retcode (void ) {RUN (verifier_netfilter_retcode); } -void test_verifier_nocsr (void ) {RUN (verifier_nocsr); } +void test_verifier_bpf_fastcall (void ) {RUN (verifier_bpf_fastcall); }void test_verifier_or_jmp32_k (void ) {RUN (verifier_or_jmp32_k); }void test_verifier_precision (void ) {RUN (verifier_precision); }void test_verifier_prevent_map_lookup (void ) {RUN (verifier_prevent_map_lookup); }
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c renamed from tools/testing/selftests/bpf/progs/verifier_nocsr.c +13 -13
@@ -39,7 +39,7 @@ __naked void simple(void) : __clobber_all); } -/* The logic for detecting and verifying nocsr pattern is the same for +/* The logic for detecting and verifying bpf_fastcall pattern is the same for * any arch, however x86 differs from arm64 or riscv64 in a way * bpf_get_smp_processor_id is rewritten: * - on x86 it is done by verifier
@@ -52,7 +52,7 @@ __naked void simple(void)* * It is really desirable to check instruction indexes in the xlated * patterns, so add this canary test to check that function rewrite by -* jit is correctly processed by nocsr logic, keep the rest of the +* jit is correctly processed by bpf_fastcall logic, keep the rest of the * tests as x86. */ SEC ("raw_tp" )
@@ -430,7 +430,7 @@ __naked static void bad_write_in_subprog_aux(void) { asmvolatile ("r0 = 1;" -"*(u64 *)(r1 - 0) = r0;" /* invalidates nocsr contract for caller: */ +"*(u64 *)(r1 - 0) = r0;" /* invalidates bpf_fastcall contract for caller: */ "exit;" /* caller stack at -8 used outside of the pattern */ ::: __clobber_all); }
@@ -445,7 +445,7 @@ __naked void bad_helper_write(void) { asmvolatile ("r1 = 1;" -/* nocsr pattern with stack offset -8 */ +/* bpf_fastcall pattern with stack offset -8 */ "*(u64 *)(r10 - 8) = r1;" "call %[bpf_get_smp_processor_id];" "r1 = *(u64 *)(r10 - 8);"
@@ -453,7 +453,7 @@ __naked void bad_helper_write(void)"r1 += -8;" "r2 = 1;" "r3 = 42;" -/* read dst is fp[-8], thus nocsr rewrite not applied */ +/* read dst is fp[-8], thus bpf_fastcall rewrite not applied */ "call %[bpf_probe_read_kernel];" "exit;" :
@@ -553,7 +553,7 @@ __arch_x86_64 __log_level(4) __msg("stack depth 8") __xlated("2: r0 = &(void __percpu *)(r0)") __success -__naked void helper_call_does_not_prevent_nocsr (void ) +__naked void helper_call_does_not_prevent_bpf_fastcall (void ) { asmvolatile ( "r1 = 1;"
@@ -640,7 +640,7 @@ __naked int bpf_loop_interaction1(void) { asmvolatile ("r1 = 1;" -/* nocsr stack region at -16, but could be removed */ +/* bpf_fastcall stack region at -16, but could be removed */ "*(u64 *)(r10 - 16) = r1;" "call %[bpf_get_smp_processor_id];" "r1 = *(u64 *)(r10 - 16);"
@@ -680,7 +680,7 @@ __naked int bpf_loop_interaction2(void) { asmvolatile ("r1 = 42;" -/* nocsr stack region at -16, cannot be removed */ +/* bpf_fastcall stack region at -16, cannot be removed */ "*(u64 *)(r10 - 16) = r1;" "call %[bpf_get_smp_processor_id];" "r1 = *(u64 *)(r10 - 16);"
@@ -710,8 +710,8 @@ __msg("stack depth 512+0") __xlated("r0 = &(void __percpu *)(r0)")__success /* cumulative_stack_depth() stack usage is MAX_BPF_STACK, -* called subprogram uses an additional slot for nocsr spill/fill, -* since nocsr spill/fill could be removed the program still fits +* called subprogram uses an additional slot for bpf_fastcall spill/fill, +* since bpf_fastcall spill/fill could be removed the program still fits * in MAX_BPF_STACK and should be accepted. */ __naked int cumulative_stack_depth(void)
@@ -749,7 +749,7 @@ __xlated("3: r0 = &(void __percpu *)(r0)") __xlated("4: r0 = *(u32 *)(r0 +0)") __xlated("5: exit") __success -__naked int nocsr_max_stack_ok (void ) +__naked int bpf_fastcall_max_stack_ok (void ) { asmvolatile ( "r1 = 42;"
@@ -771,7 +771,7 @@ __arch_x86_64 __log_level(4) __msg("stack depth 520") __failure -__naked int nocsr_max_stack_fail (void ) +__naked int bpf_fastcall_max_stack_fail (void ) { asmvolatile ( "r1 = 42;"
@@ -779,7 +779,7 @@ __naked int nocsr_max_stack_fail(void)"*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;" "call %[bpf_get_smp_processor_id];" "r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);" -/* call to prandom blocks nocsr rewrite */ +/* call to prandom blocks bpf_fastcall rewrite */ "*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;" "call %[bpf_get_prandom_u32];" "r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
Signature · 1 line
2.45.2
Re: [PATCH bpf-next v2 2/5] selftests/bpf: rename nocsr -> bpf_fastcall in selftests
Quoted 6 lines · Eduard Zingerman
Attribute used by LLVM implementation of the feature had been changed from
no_caller_saved_registerstobpf_fastcall(see [1]). This commit replaces references to nocsr by references tobpf_fastcallto keep LLVM and selftests parts in sync.
Let us change the link to https://github.com/llvm/llvm-project/pull/105417
Quoted 2 lines
Signed-off-by Eduard Zingerman <[email protected]>
Eduard Zingerman
maintainerPATCH v2 3/5 bpf: support bpf_fastcall patterns for kfuncs
In reply to Eduard ZingermanRecognize bpf_fastcall patterns around kfunc calls.
For example, suppose bpf_cast_to_kern_ctx() follows bpf_fastcall contract (which it does), in such a case allow verifier to rewrite BPF program below:
r2 = 1; *(u64 *)(r10 - 32) = r2; call %[bpf_cast_to_kern_ctx]; r2 = *(u64 *)(r10 - 32); r0 = r2;
By removing the spill/fill pair:
r2 = 1; call %[bpf_cast_to_kern_ctx]; r0 = r2;
--- kernel/bpf/verifier.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-)
kernel/bpf/verifier.c +36 -1
@@ -16112,7 +16112,7 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,*/ static u32 helper_fastcall_clobber_mask (const struct bpf_func_proto *fn) { -u8 mask; +u32 mask;int i; mask =0 ;
@@ -16140,6 +16140,28 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) } } +/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */ +static u32 kfunc_fastcall_clobber_mask (struct bpf_kfunc_call_arg_meta *meta) +{ +const struct btf_param *params; +u32 vlen, i, mask; + + params =btf_params (meta->func_proto); + vlen =btf_type_vlen (meta->func_proto); + mask =0 ; +if (!btf_type_is_void (btf_type_by_id (meta->btf, meta->func_proto->type))) + mask |=BIT (BPF_REG_0 ); +for (i =0 ; i < vlen; ++i) + mask |=BIT (BPF_REG_1 + i); +return mask; +} + +/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */ +static bool is_fastcall_kfunc_call (struct bpf_kfunc_call_arg_meta *meta) +{ +return false; +} +/* LLVM define a bpf_fastcall function attribute. * This attribute means that function scratches only some of * the caller saved registers defined by ABI.
@@ -16237,6 +16259,19 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, bpf_jit_inlines_helper_call(call->imm)); } +if (bpf_pseudo_kfunc_call (call)) { +struct bpf_kfunc_call_arg_meta meta; +int err; + + err =fetch_kfunc_meta (env, call, &meta,NULL ); +if (err <0 ) +/* error would be reported later */ +return ; + + clobbered_regs_mask =kfunc_fastcall_clobber_mask (&meta); + can_be_inlined =is_fastcall_kfunc_call (&meta); + } +if (clobbered_regs_mask ==ALL_CALLER_SAVED_REGS )return ;
Signature · 1 line
2.45.2
kernel test robot
build botRe: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs
kernel test robot noticed the following build warnings:
Hi Eduard,
kernel test robot noticed the following build warnings:
[auto build test WARNING on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Eduard-Zingerman/bpf-rename-nocsr-bpf_fastcall-in-verifier/20240817-095340 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20240817015140.1039351-4-eddyz87%40gmail.com patch subject: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs config: arc-randconfig-002-20240817 (https://download.01.org/0day-ci/archive/20240818/[email protected]/config) compiler: arceb-elf-gcc (GCC) 13.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240818/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
kernel/bpf/verifier.c: In function 'kfunc_fastcall_clobber_mask':
>> kernel/bpf/verifier.c:16146:33: warning: variable 'params' set but not used [-Wunused-but-set-variable]
16146 | const struct btf_param *params;
| ^~~~~~
vim +/params +16146 kernel/bpf/verifier.c
16142
16143 /* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
16144 static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
16145 {
16146 const struct btf_param *params;
16147 u32 vlen, i, mask;
16148
16149 params = btf_params(meta->func_proto);
16150 vlen = btf_type_vlen(meta->func_proto);
16151 mask = 0;
16152 if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
16153 mask |= BIT(BPF_REG_0);
16154 for (i = 0; i < vlen; ++i)
16155 mask |= BIT(BPF_REG_1 + i);
16156 return mask;
16157 }
16158
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Eduard Zingerman
maintainerRe: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs
Quoted 19 lines · kernel test robot
Hi Eduard,
kernel test robot noticed the following build warnings:
[auto build test WARNING on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Eduard-Zingerman/bpf-rename-nocsr-bpf_fastcall-in-verifier/20240817-095340 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20240817015140.1039351-4-eddyz87%40gmail.com patch subject: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs config: arc-randconfig-002-20240817 (https://download.01.org/0day-ci/archive/20240818/[email protected]/config) compiler: arceb-elf-gcc (GCC) 13.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240818/[email protected]/reproduce)If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
Quoted 2 lines
Quoted 7 lines
All warnings (new ones prefixed by >>):
kernel/bpf/verifier.c: In function 'kfunc_fastcall_clobber_mask':Quoted 1 line
Quoted 1 line
kernel/bpf/verifier.c:16146:33: warning: variable 'params' set but not used [-Wunused-but-set-variable]16146 | const struct
btf_param*params;
| ^~~~~~
Missed this warning.
The params variable is not needed indeed.
Will wait for comments before resubmitting.
[...]
Re: [PATCH bpf-next v2 3/5] bpf: support bpf_fastcall patterns for kfuncs
In reply to Eduard ZingermanQuoted 11 lines · Eduard Zingerman
Recognize
bpf_fastcallpatterns around kfunc calls.
For example, supposebpf_cast_to_kern_ctx()followsbpf_fastcallcontract (which it does), in such a case allow verifier to rewrite BPF program below:r2 = 1; *(u64 *)(r10 - 32) = r2; call %[bpf_cast_to_kern_ctx]; r2 = *(u64 *)(r10 - 32); r0 = r2;
Quoted 7 lines
By removing the spill/fill pair:
r2 = 1; call %[bpf_cast_to_kern_ctx]; r0 = r2;Signed-off-by Eduard Zingerman <[email protected]>
Eduard Zingerman
maintainerPATCH v2 4/5 bpf: allow bpf_fastcall for bpf_cast_to_kern_ctx and bpf_rdonly_cast
In reply to Eduard Zingermando_misc_fixups() relaces bpf_cast_to_kern_ctx() and bpf_rdonly_cast() by a single instruction "r0 = r1". This follows bpf_fastcall contract. This commit allows bpf_fastcall pattern rewrite for these two functions in order to use them in bpf_fastcall selftests.
--- kernel/bpf/verifier.c | 3 +++ 1 file changed, 3 insertions(+)
kernel/bpf/verifier.c +3 -0
@@ -16159,6 +16159,9 @@ static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */ static bool is_fastcall_kfunc_call (struct bpf_kfunc_call_arg_meta *meta) { +if (meta->btf == btf_vmlinux) +return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];return false; }
Signature · 1 line
2.45.2
Re: [PATCH bpf-next v2 4/5] bpf: allow bpf_fastcall for bpf_cast_to_kern_ctx and bpf_rdonly_cast
Quoted 6 lines · Eduard Zingerman
do_misc_fixups()relacesbpf_cast_to_kern_ctx()andbpf_rdonly_cast()by a single instruction "r0 = r1". This followsbpf_fastcallcontract. This commit allowsbpf_fastcallpattern rewrite for these two functions in order to use them inbpf_fastcallselftests.Signed-off-by Eduard Zingerman <[email protected]>
Eduard Zingerman
maintainerPATCH v2 5/5 selftests/bpf: check if bpf_fastcall is recognized for kfuncs
In reply to Eduard ZingermanUse kfunc_bpf_cast_to_kern_ctx() and kfunc_bpf_rdonly_cast() to verify that bpf_fastcall pattern is recognized for kfunc calls.
--- .../bpf/progs/verifier_bpf_fastcall.c | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+)
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +50 -0
@@ -2,8 +2,11 @@#include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" +#include <stdbool.h> +#include "bpf_kfuncs.h" SEC("raw_tp") __arch_x86_64
@@ -793,4 +796,51 @@ __naked int bpf_fastcall_max_stack_fail(void) ); } +SEC("cgroup/getsockname_unix") +__xlated ("0: r2 = 1" ) +/* bpf_cast_to_kern_ctx is replaced by a single assignment */ +__xlated("1: r0 = r1") +__xlated("2: r0 = r2") +__xlated("3: exit") +__success +__naked void kfunc_bpf_cast_to_kern_ctx (void ) +{ + asmvolatile ( +"r2 = 1;" +"*(u64 *)(r10 - 32) = r2;" +"call %[bpf_cast_to_kern_ctx];" +"r2 = *(u64 *)(r10 - 32);" +"r0 = r2;" +"exit;" + : + : __imm(bpf_cast_to_kern_ctx) + : __clobber_all); +} + +SEC("raw_tp") +__xlated ("3: r3 = 1" ) +/* bpf_rdonly_cast is replaced by a single assignment */ +__xlated("4: r0 = r1") +__xlated("5: r0 = r3") +void kfunc_bpf_rdonly_cast (void ) +{ + asmvolatile ( +"r2 = %[btf_id];" +"r3 = 1;" +"*(u64 *)(r10 - 32) = r3;" +"call %[bpf_rdonly_cast];" +"r3 = *(u64 *)(r10 - 32);" +"r0 = r3;" + : + : __imm(bpf_rdonly_cast), + [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr)) + : __clobber_common); +} + +void kfunc_root (void ) +{ +bpf_cast_to_kern_ctx (0 ); +bpf_rdonly_cast (0 ,0 ); +} + char _license[] SEC("license") = "GPL";
Signature · 1 line
2.45.2
Re: [PATCH bpf-next v2 5/5] selftests/bpf: check if bpf_fastcall is recognized for kfuncs
Quoted 4 lines · Eduard Zingerman
Use
kfunc_bpf_cast_to_kern_ctx()andkfunc_bpf_rdonly_cast()to verify thatbpf_fastcallpattern is recognized for kfunc calls.Signed-off-by Eduard Zingerman <[email protected]>
LGTM with a nit below.
Quoted 56 lines
--- .../bpf/progs/verifier_bpf_fastcall.c | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+)tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +35
-0@@ -2,8 +2,11 @@#include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" +#include <stdbool.h> +#include "bpf_kfuncs.h" SEC("raw_tp") __arch_x86_64@@ -793,4 +796,51 @@ __naked int bpf_fastcall_max_stack_fail(void) ); } +SEC("cgroup/getsockname_unix") +__xlated ("0: r2 = 1" ) +/* bpf_cast_to_kern_ctx is replaced by a single assignment */ +__xlated("1: r0 = r1") +__xlated("2: r0 = r2") +__xlated("3: exit") +__success +__naked void kfunc_bpf_cast_to_kern_ctx (void ) +{ + asmvolatile ( +"r2 = 1;" +"*(u64 *)(r10 - 32) = r2;" +"call %[bpf_cast_to_kern_ctx];" +"r2 = *(u64 *)(r10 - 32);" +"r0 = r2;" +"exit;" + : + : __imm(bpf_cast_to_kern_ctx) + : __clobber_all); +} + +SEC("raw_tp") +__xlated ("3: r3 = 1" ) +/* bpf_rdonly_cast is replaced by a single assignment */ +__xlated("4: r0 = r1") +__xlated("5: r0 = r3") +void kfunc_bpf_rdonly_cast (void ) +{ + asmvolatile ( +"r2 = %[btf_id];" +"r3 = 1;" + "*(u64 *)(r10 - 32) = r3;"
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c __naked int bpf_fastcall_max_stack_fail(void) · line 840
Quoted 10 lines
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +10
-0+ "call %[bpf_rdonly_cast];" + "r3 = *(u64 *)(r10 - 32);" + "r0 = r3;" + : + : __imm(bpf_rdonly_cast), + [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr)) + : __clobber_common); +} + +void kfunc_root (void )
It would be good to add some comments for this function to indicate that this func intends to enable BTF generation for kfuncs bpf_cast_to_kern_ctx() and bpf_rdonly_cast() which are necessary for the above two programs.
Quoted 6 lines
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +5
-0+{ +bpf_cast_to_kern_ctx (0 ); +bpf_rdonly_cast (0 ,0 ); +} + char _license[] SEC("license") = "GPL";