bpf: Save struct_ops instance pointer in bpf_prog_aux
Subsystems (5)
PATCH 1/4 bpf: Save struct_ops instance pointer in bpf_prog_aux
Allows struct_ops implementors to infer the calling struct_ops instance inside a kfunc through prog->aux->this_st_ops. A new field, flags, is added to bpf_struct_ops. If BPF_STRUCT_OPS_F_THIS_PTR is set in flags, a pointer to the struct_ops structure registered to the kernel (i.e., kvalue->data) will be saved to prog->aux->this_st_ops. To access it in a kfunc, use BPF_STRUCT_OPS_F_THIS_PTR with __prog argument [0]. The verifier will fixup the argument with a pointer to prog->aux. this_st_ops is protected by rcu and is valid until a struct_ops map is unregistered updated.
For a struct_ops map with BPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the same this_st_ops, cmpxchg is used. Only if a program is not already used in another struct_ops map also with BPF_STRUCT_OPS_F_THIS_PTR can it be assigned to the current struct_ops map.
[0]
commit bc049387b41f ("bpf: Add support for __prog argument suffix to pass in prog->aux")
https://lore.kernel.org/r/[email protected]
--- include/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_struct_ops.c | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+)
include/linux/bpf.h +10 -0
@@ -1640,6 +1640,11 @@ struct bpf_prog_aux {struct work_struct work;struct rcu_head rcu; }; +/* pointer to struct_ops struct registered to the kernel. +* Only valid when BPF_STRUCT_OPS_F_THIS_PTR is set in +* bpf_struct_ops.flags +*/ +void __rcu *this_st_ops; };struct bpf_prog {
@@ -1788,6 +1793,10 @@ struct bpf_token {struct bpf_struct_ops_value ;struct btf_member ; +#define BPF_STRUCT_OPS_F_THIS_PTR BIT(0) + +#define BPF_STRUCT_OPS_FLAG_MASK BPF_STRUCT_OPS_F_THIS_PTR +#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64/** * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
@@ -1853,6 +1862,7 @@ struct bpf_struct_ops {struct module *owner;const char *name;struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS ]; +u32 flags; };/* Every member of a struct_ops type has an instance even a member is not
kernel/bpf/bpf_struct_ops.c +38 -0
@@ -533,6 +533,17 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) } } +static void bpf_struct_ops_map_clear_this_ptr (struct bpf_struct_ops_map *st_map) +{ +u32 i; + +for (i =0 ; i < st_map->funcs_cnt; i++) { +if (!st_map->links[i]) +break ; +RCU_INIT_POINTER (st_map->links[i]->prog->aux->this_st_ops,NULL ); + } +} +static void bpf_struct_ops_map_free_image (struct bpf_struct_ops_map *st_map) {int i;
@@ -695,6 +706,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,if (flags)return -EINVAL ; +if (st_ops->flags & ~BPF_STRUCT_OPS_FLAG_MASK ) +return -EINVAL ; +if (*(u32 *)key !=0 )return -E2BIG ;
@@ -801,6 +815,19 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto reset_unlock; } +if (st_ops->flags &BPF_STRUCT_OPS_F_THIS_PTR ) { +/* Make sure a struct_ops map will not have programs with +* different this_st_ops. Once a program is associated with +* a struct_ops map, it cannot be used in another struct_ops +* map also with BPF_STRUCT_OPS_F_THIS_PTR +*/ +if (cmpxchg (&prog->aux->this_st_ops,NULL , kdata)) { +bpf_prog_put (prog); + err = -EINVAL ; + goto reset_unlock; + } + } + link =kzalloc (sizeof (*link),GFP_USER );if (!link) {bpf_prog_put (prog);
@@ -894,6 +921,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,bpf_struct_ops_map_free_ksyms (st_map);bpf_struct_ops_map_free_image (st_map);bpf_struct_ops_map_put_progs (st_map); +if (st_ops->flags &BPF_STRUCT_OPS_F_THIS_PTR ) +bpf_struct_ops_map_clear_this_ptr (st_map);memset (uvalue,0 , map->value_size);memset (kvalue,0 , map->value_size); unlock:
@@ -919,6 +948,8 @@ static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)switch (prev_state) {case BPF_STRUCT_OPS_STATE_INUSE : st_map->st_ops_desc->st_ops->unreg (&st_map->kvalue.data,NULL ); +if (st_map->st_ops_desc->st_ops->flags &BPF_STRUCT_OPS_F_THIS_PTR ) +bpf_struct_ops_map_clear_this_ptr (st_map);bpf_map_put (map);return 0 ;case BPF_STRUCT_OPS_STATE_TOBEFREE :
@@ -1194,6 +1225,8 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)rcu_dereference_protected (st_link->map, true);if (st_map) { st_map->st_ops_desc->st_ops->unreg (&st_map->kvalue.data, link); +if (st_map->st_ops_desc->st_ops->flags &BPF_STRUCT_OPS_F_THIS_PTR ) +bpf_struct_ops_map_clear_this_ptr (st_map);bpf_map_put (&st_map->map); }kfree (st_link);
@@ -1268,6 +1301,9 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_mapif (err) goto err_out; +if (st_map->st_ops_desc->st_ops->flags &BPF_STRUCT_OPS_F_THIS_PTR ) +bpf_struct_ops_map_clear_this_ptr (st_map); +bpf_map_inc (new_map);rcu_assign_pointer (st_link->map, new_map);bpf_map_put (old_map);
@@ -1294,6 +1330,8 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) st_map = container_of(map, struct bpf_struct_ops_map, map); st_map->st_ops_desc->st_ops->unreg (&st_map->kvalue.data, link); +if (st_map->st_ops_desc->st_ops->flags &BPF_STRUCT_OPS_F_THIS_PTR ) +bpf_struct_ops_map_clear_this_ptr (st_map);RCU_INIT_POINTER (st_link->map,NULL );/* Pair with bpf_map_get() in bpf_struct_ops_link_create() or
Signature · 1 line
2.47.1
PATCH 2/4 bpf: Allow verifier to fixup kernel module kfuncs
Allow verifier to fixup kfuncs in kernel module to support kfuncs with __prog arguments. Currently, special kfuncs and kfuncs with __prog arguments are kernel kfuncs. As there is no safety reason that prevents a kernel module kfunc from accessing prog->aux, allow it by removing the kernel BTF check.
--- kernel/bpf/verifier.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
kernel/bpf/verifier.c +1 -2
@@ -21467,8 +21467,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,if (!bpf_jit_supports_far_kfunc_call ()) insn->imm =BPF_CALL_IMM (desc->addr); -if (insn->off) -return 0 ; +if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] || desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
Signature · 1 line
2.47.1
PATCH 3/4 selftests/bpf: Test accessing struct_ops this pointer
In reply to Amery HungTest struct_ops this pointer using bpf_testmod_ops3. Add an integer field, data, to bpf_testmod_ops3 and a kfunc that reads data through aux->this_st_ops. Check if the kfunc reads the correct value of data.
--- .../bpf/prog_tests/test_struct_ops_this_ptr.c | 10 ++++++ .../progs/struct_ops_private_stack_recur.c | 3 +- .../selftests/bpf/progs/struct_ops_this_ptr.c | 30 ++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 36 +++++++++++++++++-- .../selftests/bpf/test_kmods/bpf_testmod.h | 1 + .../bpf/test_kmods/bpf_testmod_kfunc.h | 3 ++ 6 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_this_ptr.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_this_ptr.c
tools/testing/selftests/bpf/prog_tests/test_struct_ops_this_ptr.c new file +10 -0
@@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <test_progs.h> + +#include "struct_ops_this_ptr.skel.h" + +void serial_test_struct_ops_this_ptr (void ) +{ +RUN_TESTS (struct_ops_this_ptr); +}
tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c +1 -2
@@ -4,6 +4,7 @@#include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" char _license[] SEC("license") = "GPL";
@@ -13,8 +14,6 @@ bool skip __attribute((__section__(".data"))) = false;bool skip = true;#endif -void bpf_testmod_ops3_call_test_1(void) __ksym; -int val_i, val_j; __noinline static int subprog2(int *a, int *b)
tools/testing/selftests/bpf/progs/struct_ops_this_ptr.c new file +30 -0
@@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +int BPF_PROG (test1 ) +{ +return bpf_kfunc_st_ops_test_this_ptr_impl (NULL ); +} + +SEC("syscall") +__success __retval(1234) +int syscall_this_ptr(void *ctx) +{ +return bpf_testmod_ops3_call_test_1 (); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_this_ptr = { + .test_1 = (void *)test1, + .data =1234 , +}; + +
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +33 -3
@@ -272,9 +272,9 @@ static void bpf_testmod_test_struct_ops3(void) st_ops3->test_1 (); } -__bpf_kfunc void bpf_testmod_ops3_call_test_1(void) +__bpf_kfunc int bpf_testmod_ops3_call_test_1(void) { - st_ops3->test_1 (); +return st_ops3->test_1 (); } __bpf_kfunc void bpf_testmod_ops3_call_test_2(void)
@@ -1057,6 +1057,23 @@ __bpf_kfunc int bpf_kfunc_st_ops_inc10(struct st_ops_args *args)return args->a; } +__bpf_kfunc int bpf_kfunc_st_ops_test_this_ptr_impl(void *aux__prog) +{ +struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__prog; +struct bpf_testmod_ops3 *ops; +int data =-1 ; + +rcu_read_lock (); + ops =rcu_dereference (aux->this_st_ops); +if (!ops) + goto out; + + data = ops->data; +out: +rcu_read_unlock (); +return data; +} + BTF_KFUNCS_START(bpf_testmod_check_kfunc_ids) BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) BTF_ID_FLAGS(func, bpf_kfunc_call_test1)
@@ -1097,6 +1114,7 @@ BTF_ID_FLAGS(func, bpf_kfunc_st_ops_test_prologue, KF_TRUSTED_ARGS | KF_SLEEPABL BTF_ID_FLAGS(func, bpf_kfunc_st_ops_test_epilogue, KF_TRUSTED_ARGS | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_kfunc_st_ops_test_pro_epilogue, KF_TRUSTED_ARGS | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_kfunc_st_ops_inc10, KF_TRUSTED_ARGS) +BTF_ID_FLAGS(func, bpf_kfunc_st_ops_test_this_ptr_impl)BTF_KFUNCS_END (bpf_testmod_check_kfunc_ids) static int bpf_testmod_ops_init(struct btf *btf)
@@ -1269,6 +1287,17 @@ static void test_1_recursion_detected(struct bpf_prog *prog) u64_stats_read(&stats->misses)); } +static int st_ops3_init_member (const struct btf_type *t, +const struct btf_member *member, +void *kdata,const void *udata) +{ +if (member->offset == offsetof(struct bpf_testmod_ops3 , data) *8 ) { + ((struct bpf_testmod_ops3 *)kdata)->data = ((struct bpf_testmod_ops3 *)udata)->data; +return 1 ; + } +return 0 ; +} +static int st_ops3_check_member (const struct btf_type *t,const struct btf_member *member,const struct bpf_prog *prog)
@@ -1289,13 +1318,14 @@ static int st_ops3_check_member(const struct btf_type *t,struct bpf_struct_ops bpf_testmod_ops3 = { .verifier_ops = &bpf_testmod_verifier_ops3, .init = bpf_testmod_ops_init, - .init_member = bpf_testmod_ops_init_member, + .init_member = st_ops3_init_member, .reg = st_ops3_reg, .unreg = st_ops3_unreg, .check_member = st_ops3_check_member, .cfi_stubs = &__bpf_testmod_ops3, .name ="bpf_testmod_ops3" , .owner =THIS_MODULE , + .flags =BPF_STRUCT_OPS_F_THIS_PTR , };static int bpf_test_mod_st_ops__test_prologue (struct st_ops_args *args)
tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +1 -0
@@ -103,6 +103,7 @@ struct bpf_testmod_ops2 {struct bpf_testmod_ops3 {int (*test_1)(void );int (*test_2)(void ); +int data; };struct st_ops_args {
tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +3 -0
@@ -159,4 +159,7 @@ void bpf_kfunc_trusted_task_test(struct task_struct *ptr) __ksym; void bpf_kfunc_trusted_num_test(int *ptr) __ksym; void bpf_kfunc_rcu_task_test(struct task_struct *ptr) __ksym; +int bpf_testmod_ops3_call_test_1(void) __ksym; +int bpf_kfunc_st_ops_test_this_ptr_impl(void *aux__prog) __ksym; +#endif /* _BPF_TESTMOD_KFUNC_H */
Signature · 1 line
2.47.1
PATCH 4/4 selftests/bpf: Test accessing struct_ops this pointer in timer callback
In reply to Amery HungCheck accessing aux->this_st_ops in timer callback. Make sure a kfunc in a timer callback can get a correct this pointer when a struct_ops map is attached, and NULL after the map is detached.
--- .../bpf/prog_tests/test_struct_ops_this_ptr.c | 51 ++++++++++++++++ .../bpf/progs/struct_ops_this_ptr_in_timer.c | 60 +++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 1 + 3 files changed, 112 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_this_ptr_in_timer.c
tools/testing/selftests/bpf/prog_tests/test_struct_ops_this_ptr.c +51 -0
@@ -3,8 +3,59 @@#include <test_progs.h> #include "struct_ops_this_ptr.skel.h" +#include "struct_ops_this_ptr_in_timer.skel.h" + +static void test_struct_ops_this_ptr_in_timer_common (int timer_nsec,int expected_data) +{ +struct struct_ops_this_ptr_in_timer *skel; +LIBBPF_OPTS (bpf_test_run_opts, topts); +struct bpf_link *link; +int err, prog_fd; + + skel =struct_ops_this_ptr_in_timer__open_and_load (); +if (!ASSERT_OK_PTR (skel,"skel_open_and_load" )) +return ; + + skel->bss->timer_nsec = timer_nsec; + + link =bpf_map__attach_struct_ops (skel->maps.testmod_this_ptr); +if (!ASSERT_OK_PTR (link,"bpf_map__attach_struct_ops" )) + goto out; + + prog_fd =bpf_program__fd (skel->progs.syscall_this_ptr_in_timer); + err =bpf_prog_test_run_opts (prog_fd, &topts); +ASSERT_OK (err,"bpf_prog_test_run_opts" ); + +bpf_link__destroy (link); + +/* Check st_ops3_data after timer_cb runs */ +while (!READ_ONCE (skel->bss->st_ops3_data)) +sched_yield (); +ASSERT_EQ (skel->bss->st_ops3_data, expected_data,"st_ops->data" ); +out: +struct_ops_this_ptr_in_timer__destroy (skel); +} + +static void test_struct_ops_this_ptr_in_timer (void ) +{ +/* Run timer callback immediately */ +test_struct_ops_this_ptr_in_timer_common (0 ,1234 ); +} + +static void test_struct_ops_this_ptr_in_timer_after_detach (void ) +{ +/* +* Run timer callback 0.1s after test run. By then the struct_ops map +* should have been detached. +*/ +test_struct_ops_this_ptr_in_timer_common (100000000 ,-1 ); +}void serial_test_struct_ops_this_ptr (void ) {RUN_TESTS (struct_ops_this_ptr); +if (test__start_subtest ("struct_ops_this_ptr_in_timer" )) +test_struct_ops_this_ptr_in_timer (); +if (test__start_subtest ("struct_ops_this_ptr_in_timer_after_detach" )) +test_struct_ops_this_ptr_in_timer_after_detach (); }
tools/testing/selftests/bpf/progs/struct_ops_this_ptr_in_timer.c new file +60 -0
60 lines not on this page · raw message
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +1 -0
@@ -1157,6 +1157,7 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { };static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = { + .get_func_proto = bpf_base_func_proto, .is_valid_access = bpf_testmod_ops_is_valid_access, };
Signature · 1 line
2.47.1
A message that is not in the archive (sent off-list or not yet received): 8c4943cb40967d152abe032b4208a7cdd89b539da26783afaa61e37eb6663cbf@mail.kernel.org
Re: [PATCH bpf-next v1 4/4] selftests/bpf: Test accessing struct_ops this pointer in timer callback
Quoted 17 lines · On Mon, Jun 9, 2025 at 5:42 PM <[email protected]> wrote:
Dear patch submitter,
CI has tested the following submission:
Status: FAILURE Name: [bpf-next,v1,4/4] selftests/bpf: Test accessing struct_ops this pointer in timer callback Patchwork: https://patchwork.kernel.org/project/netdevbpf/list/?series=970053&state=* Matrix: https://github.com/kernel-patches/bpf/actions/runs/15547147788Failed jobs:
test_progs-aarch64-gcc-14: https://github.com/kernel-patches/bpf/actions/runs/15547147788/job/43771213307
test_progs_cpuv4-aarch64-gcc-14: https://github.com/kernel-patches/bpf/actions/runs/15547147788/job/43771213280
test_progs_no_alu32-aarch64-gcc-14: https://github.com/kernel-patches/bpf/actions/runs/15547147788/job/43771213268
test_progs-s390x-gcc-14: https://github.com/kernel-patches/bpf/actions/runs/15547147788/job/43771045529
test_progs_cpuv4-s390x-gcc-14: https://github.com/kernel-patches/bpf/actions/runs/15547147788/job/43771045522
test_progs_no_alu32-s390x-gcc-14: https://github.com/kernel-patches/bpf/actions/runs/15547147788/job/43771045519
Quoted 14 lines
First
test_progsfailure (test_progs-aarch64-gcc-14):
#409struct_ops_this_ptr
tester_init:PASS:tester_log_buf 0 nsec process_subtest:PASS:obj_open_mem 0 nsec process_subtest:PASS:specs_alloc 0 nsec
#409/1 struct_ops_this_ptr/test1 run_subtest:PASS:obj_open_mem 0 nsec
libbpf: prog 'test1': BPF program load failed:-EACCES
libbpf: prog 'test1': failed to load:-EACCES
libbpf: failed to load object 'struct_ops_this_ptr' run_subtest:FAIL:unexpected_load_failure unexpected error: -13 (errno 13) VERIFIER LOG:
=============
Private stack not supported by jit
The selftests failed to pass CI since bpf_testmod_ops3 is using private stack. I will change selftests to be based on other struct_ops in bpf_testmod.c
Quoted 40 lines
processed 0 insns (limit 1000000)
max_states_per_insn0total_states0peak_states0mark_read0
=============
#409/2 struct_ops_this_ptr/syscall_this_ptr
run_subtest:PASS:obj_open_mem 0 nsec
libbpf: prog 'test1': BPF program load failed:-EACCES
libbpf: prog 'test1': -- BEGIN PROG LOAD LOG --
Private stack not supported by jit
processed 0 insns (limit 1000000)max_states_per_insn0total_states0peak_states0mark_read0 -- END PROG LOAD LOG --
libbpf: prog 'test1': failed to load:-EACCES
libbpf: failed to load object 'struct_ops_this_ptr' run_subtest:FAIL:unexpected_load_failure unexpected error: -13 (errno 13) VERIFIER LOG:
=============
=============
#409/3 struct_ops_this_ptr/struct_ops_this_ptr_in_timer
libbpf: prog 'test1': BPF program load failed:-EACCES
libbpf: prog 'test1': -- BEGIN PROG LOAD LOG --
Private stack not supported by jit
processed 0 insns (limit 1000000)max_states_per_insn0total_states0peak_states0mark_read0 -- END PROG LOAD LOG --
libbpf: prog 'test1': failed to load:-EACCES
libbpf: failed to load object 'struct_ops_this_ptr_in_timer'
libbpf: failed to load BPF skeleton 'struct_ops_this_ptr_in_timer':-EACCEStest_struct_ops_this_ptr_in_timer_common:FAIL:skel_open_and_load unexpected error: -13 #409/4 struct_ops_this_ptr/struct_ops_this_ptr_in_timer_after_detach
libbpf: prog 'test1': BPF program load failed:-EACCES
libbpf: prog 'test1': -- BEGIN PROG LOAD LOG --
Private stack not supported by jit
processed 0 insns (limit 1000000)max_states_per_insn0total_states0peak_states0mark_read0 -- END PROG LOAD LOG --
libbpf: prog 'test1': failed to load:-EACCES
libbpf: failed to load object 'struct_ops_this_ptr_in_timer'
libbpf: failed to load BPF skeleton 'struct_ops_this_ptr_in_timer':-EACCEStest_struct_ops_this_ptr_in_timer_common:FAIL:skel_open_and_load unexpected error: -13Please note: this email is coming from an unmonitored mailbox. If you have questions or feedback, please reach out to the Meta Kernel CI team at [email protected].
Quoted 16 lines · Amery Hung
Allows
struct_opsimplementors to infer the callingstruct_opsinstance inside a kfunc throughprog->aux->this_st_ops. A new field, flags, is added tobpf_struct_ops. IfBPF_STRUCT_OPS_F_THIS_PTRis set in flags, a pointer to thestruct_opsstructure registered to the kernel (i.e.,kvalue->data) will be saved toprog->aux->this_st_ops. To access it in a kfunc, useBPF_STRUCT_OPS_F_THIS_PTRwith__progargument [0]. The verifier will fixup the argument with a pointer toprog->aux.this_st_opsis protected by rcu and is valid until astruct_opsmap is unregistered updated.For a
struct_opsmap withBPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the samethis_st_ops, cmpxchg is used. Only if a program is not already used in anotherstruct_opsmap also withBPF_STRUCT_OPS_F_THIS_PTRcan it be assigned to the currentstruct_opsmap.
Quoted 6 lines
[0]
commitbc049387b41f("bpf: Add support for__progargument suffix to pass inprog->aux")
https://lore.kernel.org/r/[email protected]Signed-off-by Amery Hung <[email protected]>
FWIW, this looks great from sched_ext POV.
Thanks.
Signature · 1 line
tejun
kernel test robot
build botkernel test robot noticed the following build warnings:
Hi Amery,
kernel test robot noticed the following build warnings:
[auto build test WARNING on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Amery-Hung/bpf-Allow-verifier-to-fixup-kernel-module-kfuncs/20250610-072957 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20250609232746.1030044-1-ameryhung%40gmail.com patch subject: [PATCH bpf-next v1 1/4] bpf: Save struct_ops instance pointer in bpf_prog_aux config: x86_64-randconfig-121-20250611 (https://download.01.org/0day-ci/archive/20250611/[email protected]/config) compiler: gcc-12 (Debian 12.2.0-14) 12.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250611/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
sparse warnings: (new ones prefixed by >>)
>> kernel/bpf/bpf_struct_ops.c:824:29: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected void [noderef] __rcu *__new @@ got void *[assigned] kdata @@
kernel/bpf/bpf_struct_ops.c:824:29: sparse: expected void [noderef] __rcu *__new
kernel/bpf/bpf_struct_ops.c:824:29: sparse: got void *[assigned] kdata
vim +824 kernel/bpf/bpf_struct_ops.c
686
687 static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
688 void *value, u64 flags)
689 {
690 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
691 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
692 const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
693 struct bpf_struct_ops_value *uvalue, *kvalue;
694 const struct btf_type *module_type;
695 const struct btf_member *member;
696 const struct btf_type *t = st_ops_desc->type;
697 struct bpf_tramp_links *tlinks;
698 void *udata, *kdata;
699 int prog_fd, err;
700 u32 i, trampoline_start, image_off = 0;
701 void *cur_image = NULL, *image = NULL;
702 struct bpf_link **plink;
703 struct bpf_ksym **pksym;
704 const char *tname, *mname;
705
706 if (flags)
707 return -EINVAL;
708
709 if (st_ops->flags & ~BPF_STRUCT_OPS_FLAG_MASK)
710 return -EINVAL;
711
712 if (*(u32 *)key != 0)
713 return -E2BIG;
714
715 err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);
716 if (err)
717 return err;
718
719 uvalue = value;
720 err = check_zero_holes(st_map->btf, t, uvalue->data);
721 if (err)
722 return err;
723
724 if (uvalue->common.state || refcount_read(&uvalue->common.refcnt))
725 return -EINVAL;
726
727 tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
728 if (!tlinks)
729 return -ENOMEM;
730
731 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
732 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
733
734 mutex_lock(&st_map->lock);
735
736 if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) {
737 err = -EBUSY;
738 goto unlock;
739 }
740
741 memcpy(uvalue, value, map->value_size);
742
743 udata = &uvalue->data;
744 kdata = &kvalue->data;
745
746 plink = st_map->links;
747 pksym = st_map->ksyms;
748 tname = btf_name_by_offset(st_map->btf, t->name_off);
749 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
750 for_each_member(i, t, member) {
751 const struct btf_type *mtype, *ptype;
752 struct bpf_prog *prog;
753 struct bpf_tramp_link *link;
754 struct bpf_ksym *ksym;
755 u32 moff;
756
757 moff = __btf_member_bit_offset(t, member) / 8;
758 mname = btf_name_by_offset(st_map->btf, member->name_off);
759 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
760 if (ptype == module_type) {
761 if (*(void **)(udata + moff))
762 goto reset_unlock;
763 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
764 continue;
765 }
766
767 err = st_ops->init_member(t, member, kdata, udata);
768 if (err < 0)
769 goto reset_unlock;
770
771 /* The ->init_member() has handled this member */
772 if (err > 0)
773 continue;
774
775 /* If st_ops->init_member does not handle it,
776 * we will only handle func ptrs and zero-ed members
777 * here. Reject everything else.
778 */
779
780 /* All non func ptr member must be 0 */
781 if (!ptype || !btf_type_is_func_proto(ptype)) {
782 u32 msize;
783
784 mtype = btf_type_by_id(st_map->btf, member->type);
785 mtype = btf_resolve_size(st_map->btf, mtype, &msize);
786 if (IS_ERR(mtype)) {
787 err = PTR_ERR(mtype);
788 goto reset_unlock;
789 }
790
791 if (memchr_inv(udata + moff, 0, msize)) {
792 err = -EINVAL;
793 goto reset_unlock;
794 }
795
796 continue;
797 }
798
799 prog_fd = (int)(*(unsigned long *)(udata + moff));
800 /* Similar check as the attr->attach_prog_fd */
801 if (!prog_fd)
802 continue;
803
804 prog = bpf_prog_get(prog_fd);
805 if (IS_ERR(prog)) {
806 err = PTR_ERR(prog);
807 goto reset_unlock;
808 }
809
810 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
811 prog->aux->attach_btf_id != st_ops_desc->type_id ||
812 prog->expected_attach_type != i) {
813 bpf_prog_put(prog);
814 err = -EINVAL;
815 goto reset_unlock;
816 }
817
818 if (st_ops->flags & BPF_STRUCT_OPS_F_THIS_PTR) {
819 /* Make sure a struct_ops map will not have programs with
820 * different this_st_ops. Once a program is associated with
821 * a struct_ops map, it cannot be used in another struct_ops
822 * map also with BPF_STRUCT_OPS_F_THIS_PTR
823 */
> 824 if (cmpxchg(&prog->aux->this_st_ops, NULL, kdata)) { 825 bpf_prog_put(prog);
826 err = -EINVAL;
827 goto reset_unlock;
828 }
829 }
830
831 link = kzalloc(sizeof(*link), GFP_USER);
832 if (!link) {
833 bpf_prog_put(prog);
834 err = -ENOMEM;
835 goto reset_unlock;
836 }
837 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
838 &bpf_struct_ops_link_lops, prog);
839 *plink++ = &link->link;
840
841 ksym = kzalloc(sizeof(*ksym), GFP_USER);
842 if (!ksym) {
843 err = -ENOMEM;
844 goto reset_unlock;
845 }
846 *pksym++ = ksym;
847
848 trampoline_start = image_off;
849 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
850 &st_ops->func_models[i],
851 *(void **)(st_ops->cfi_stubs + moff),
852 &image, &image_off,
853 st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);
854 if (err)
855 goto reset_unlock;
856
857 if (cur_image != image) {
858 st_map->image_pages[st_map->image_pages_cnt++] = image;
859 cur_image = image;
860 trampoline_start = 0;
861 }
862
863 *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
864
865 /* put prog_id to udata */
866 *(unsigned long *)(udata + moff) = prog->aux->id;
867
868 /* init ksym for this trampoline */
869 bpf_struct_ops_ksym_init(tname, mname,
870 image + trampoline_start,
871 image_off - trampoline_start,
872 ksym);
873 }
874
875 if (st_ops->validate) {
876 err = st_ops->validate(kdata);
877 if (err)
878 goto reset_unlock;
879 }
880 for (i = 0; i < st_map->image_pages_cnt; i++) {
881 err = arch_protect_bpf_trampoline(st_map->image_pages[i],
882 PAGE_SIZE);
883 if (err)
884 goto reset_unlock;
885 }
886
887 if (st_map->map.map_flags & BPF_F_LINK) {
888 err = 0;
889 /* Let bpf_link handle registration & unregistration.
890 *
891 * Pair with smp_load_acquire() during lookup_elem().
892 */
893 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY);
894 goto unlock;
895 }
896
897 err = st_ops->reg(kdata, NULL);
898 if (likely(!err)) {
899 /* This refcnt increment on the map here after
900 * 'st_ops->reg()' is secure since the state of the
901 * map must be set to INIT at this moment, and thus
902 * bpf_struct_ops_map_delete_elem() can't unregister
903 * or transition it to TOBEFREE concurrently.
904 */
905 bpf_map_inc(map);
906 /* Pair with smp_load_acquire() during lookup_elem().
907 * It ensures the above udata updates (e.g. prog->aux->id)
908 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
909 */
910 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE);
911 goto unlock;
912 }
913
914 /* Error during st_ops->reg(). Can happen if this struct_ops needs to be
915 * verified as a whole, after all init_member() calls. Can also happen if
916 * there was a race in registering the struct_ops (under the same name) to
917 * a sub-system through different struct_ops's maps.
918 */
919
920 reset_unlock:
921 bpf_struct_ops_map_free_ksyms(st_map);
922 bpf_struct_ops_map_free_image(st_map);
923 bpf_struct_ops_map_put_progs(st_map);
924 if (st_ops->flags & BPF_STRUCT_OPS_F_THIS_PTR)
925 bpf_struct_ops_map_clear_this_ptr(st_map);
926 memset(uvalue, 0, map->value_size);
927 memset(kvalue, 0, map->value_size);
928 unlock:
929 kfree(tlinks);
930 mutex_unlock(&st_map->lock);
931 if (!err)
932 bpf_struct_ops_map_add_ksyms(st_map);
933 return err;
934 }
935
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Quoted 11 lines · Amery Hung
Allows
struct_opsimplementors to infer the callingstruct_opsinstance inside a kfunc throughprog->aux->this_st_ops. A new field, flags, is added tobpf_struct_ops. IfBPF_STRUCT_OPS_F_THIS_PTRis set in flags, a pointer to thestruct_opsstructure registered to the kernel (i.e.,kvalue->data) will be saved toprog->aux->this_st_ops. To access it in a kfunc, useBPF_STRUCT_OPS_F_THIS_PTRwith__progargument [0]. The verifier will fixup the argument with a pointer toprog->aux.this_st_opsis protected by rcu and is valid until astruct_opsmap is unregistered updated.
Quoted 6 lines
For a
struct_opsmap withBPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the samethis_st_ops, cmpxchg is used. Only if a program is not already used in anotherstruct_opsmap also withBPF_STRUCT_OPS_F_THIS_PTRcan it be assigned to the currentstruct_opsmap.
Have you considered an alternative to storing this_st_ops in bpf_prog_aux by setting it at runtime (in struct_ops trampoline) into bpf_run_ctx (which I think all struct ops programs have set), and then letting any struct_ops kfunc just access it through current (see other uses of bpf_run_ctx, if you are unfamiliar). This would avoid all this business with extra flags and passing bpf_prog_aux as an extra argument.
There will be no "only one struct_ops for this BPF program" limitation
either: technically, you could have the same BPF program used from two struct_ops maps just fine (even at the same time). Then depending on which struct_ops is currently active, you'd have a corresponding bpf_run_ctx's struct_ops pointer. It feels like a cleaner approach to me.
And in the trampoline itself it would be a hard-coded single word assignment on the stack, so should be basically a no-op from performance point of view.
Quoted 11 lines
[0]
commitbc049387b41f("bpf: Add support for__progargument suffix to pass inprog->aux")
https://lore.kernel.org/r/[email protected]Signed-off-by Amery Hung <[email protected]>--- include/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_struct_ops.c | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+)
[...]
Quoted 20 lines · Andrii Nakryiko
Quoted 11 lines · Amery Hung
Allows
struct_opsimplementors to infer the callingstruct_opsinstance inside a kfunc throughprog->aux->this_st_ops. A new field, flags, is added tobpf_struct_ops. IfBPF_STRUCT_OPS_F_THIS_PTRis set in flags, a pointer to thestruct_opsstructure registered to the kernel (i.e.,kvalue->data) will be saved toprog->aux->this_st_ops. To access it in a kfunc, useBPF_STRUCT_OPS_F_THIS_PTRwith__progargument [0]. The verifier will fixup the argument with a pointer toprog->aux.this_st_opsis protected by rcu and is valid until astruct_opsmap is unregistered updated.Quoted 6 lines
For a
struct_opsmap withBPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the samethis_st_ops, cmpxchg is used. Only if a program is not already used in anotherstruct_opsmap also withBPF_STRUCT_OPS_F_THIS_PTRcan it be assigned to the currentstruct_opsmap.
Quoted 8 lines
Have you considered an alternative to storing
this_st_opsinbpf_prog_auxby setting it at runtime (instruct_opstrampoline) intobpf_run_ctx(which I think all struct ops programs have set), and then letting anystruct_opskfunc just access it through current (see other uses ofbpf_run_ctx, if you are unfamiliar). This would avoid all this business with extra flags and passingbpf_prog_auxas an extra argument.
I didn't know this. Thanks for suggesting an alternative!
Quoted 7 lines
There will be no "only one
struct_opsfor this BPF program" limitation
either: technically, you could have the same BPF program used from twostruct_opsmaps just fine (even at the same time). Then depending on whichstruct_opsis currently active, you'd have a corresponding bpf_run_ctx'sstruct_opspointer. It feels like a cleaner approach to me.
This is a cleaner approach for struct_ops operators. To make it work for kfuncs called in timer callback, I think prog->aux->st_ops is still needed, but at least we can unify how to get this_st_ops in kfunc, in a way that does not requires adding __prog to every kfuncs.
+enum bpf_run_ctx_type {
+ BPF_CG_RUN_CTX = 0, + BPF_TRACE_RUN_CTX, + BPF_TRAMP_RUN_CTX, + BPF_TIMER_RUN_CTX,
+};
struct bpf_run_ctx {
+ enum bpf_run_ctx_type type;
};
+struct bpf_timer_run_ctx {
+ struct bpf_prog_aux *aux;
+};
struct bpf_tramp_run_ctx {
...+ void *st_ops;
};
In bpf_struct_ops_prepare_trampoline(), the st_ops assignment will be emitted to the trampoline.
In bpf_timer_cb(), prepare bpf_timer_run_ctx, where st_ops comes from prog->aux->this_st_ops and set current->bpf_ctx.
Finally, in kfuncs that want to know the current struct_ops, call this new function below:
+void *bpf_struct_ops_current_st_ops(void)
+{
+ struct bpf_prog_aux aux;
+
+ if (!current->bpf_ctx)
+ return NULL;
+
+ switch(current->bpf_ctx->type) {
+ case BPF_TRAMP_RUN_CTX:
+ return (struct bpf_tramp_run_ctx *)(current->bpf_ctx)->st_ops;
+ case BPF_TIMER_RUN_CTX:
+ aux = (struct bpf_timer_run_ctx *)(current->bpf_ctx)->aux;
+ return rcu_dereference(aux->this_st_ops);
+ }
+ return NULL;
+}
What do you think?
Quoted 17 lines
And in the trampoline itself it would be a hard-coded single word assignment on the stack, so should be basically a no-op from performance point of view.
Quoted 11 lines
[0]
commitbc049387b41f("bpf: Add support for__progargument suffix to pass inprog->aux")
https://lore.kernel.org/r/[email protected]Signed-off-by Amery Hung <[email protected]>--- include/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_struct_ops.c | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+)[...]
Quoted 93 lines · Amery Hung
Quoted 20 lines · Andrii Nakryiko
Quoted 11 lines · Amery Hung
Allows
struct_opsimplementors to infer the callingstruct_opsinstance inside a kfunc throughprog->aux->this_st_ops. A new field, flags, is added tobpf_struct_ops. IfBPF_STRUCT_OPS_F_THIS_PTRis set in flags, a pointer to thestruct_opsstructure registered to the kernel (i.e.,kvalue->data) will be saved toprog->aux->this_st_ops. To access it in a kfunc, useBPF_STRUCT_OPS_F_THIS_PTRwith__progargument [0]. The verifier will fixup the argument with a pointer toprog->aux.this_st_opsis protected by rcu and is valid until astruct_opsmap is unregistered updated.Quoted 6 lines
For a
struct_opsmap withBPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the samethis_st_ops, cmpxchg is used. Only if a program is not already used in anotherstruct_opsmap also withBPF_STRUCT_OPS_F_THIS_PTRcan it be assigned to the currentstruct_opsmap.Quoted 8 lines
Have you considered an alternative to storing
this_st_opsinbpf_prog_auxby setting it at runtime (instruct_opstrampoline) intobpf_run_ctx(which I think all struct ops programs have set), and then letting anystruct_opskfunc just access it through current (see other uses ofbpf_run_ctx, if you are unfamiliar). This would avoid all this business with extra flags and passingbpf_prog_auxas an extra argument.I didn't know this. Thanks for suggesting an alternative!
Quoted 7 lines
There will be no "only one
struct_opsfor this BPF program" limitation
either: technically, you could have the same BPF program used from twostruct_opsmaps just fine (even at the same time). Then depending on whichstruct_opsis currently active, you'd have a corresponding bpf_run_ctx'sstruct_opspointer. It feels like a cleaner approach to me.This is a cleaner approach for
struct_opsoperators. To make it work for kfuncs called in timer callback, I thinkprog->aux->st_opsis still needed, but at least we can unify how to getthis_st_opsin kfunc, in a way that does not requires adding__progto every kfuncs.(quoted diff) +6
-0+enum bpf_run_ctx_type { + BPF_CG_RUN_CTX = 0, + BPF_TRACE_RUN_CTX, + BPF_TRAMP_RUN_CTX, + BPF_TIMER_RUN_CTX, +};struct
bpf_run_ctx{
+ enumbpf_run_ctx_typetype;
};(quoted diff) +3
-0+struct bpf_timer_run_ctx { + struct bpf_prog_aux *aux; +};struct bpf_tramp_run_ctx { ...+ void
*st_ops;
};In
bpf_struct_ops_prepare_trampoline(), thest_opsassignment will be emitted to the trampoline.In
bpf_timer_cb(), preparebpf_timer_run_ctx, wherest_opscomes fromprog->aux->this_st_opsand setcurrent->bpf_ctx.Finally, in kfuncs that want to know the current
struct_ops, call this new function below:(quoted diff) +16
-0+void *bpf_struct_ops_current_st_ops(void) +{ + struct bpf_prog_aux aux; + + if (!current->bpf_ctx) + return NULL; + + switch(current->bpf_ctx->type) { + case BPF_TRAMP_RUN_CTX: + return (struct bpf_tramp_run_ctx *)(current->bpf_ctx)->st_ops; + case BPF_TIMER_RUN_CTX: + aux = (struct bpf_timer_run_ctx *)(current->bpf_ctx)->aux; + return rcu_dereference(aux->this_st_ops); + } + return NULL; +}
Quoted 1 line
What do you think?
I'm not sure I particularly like different ways to get this st_ops pointer depending whether we are in an async callback or not. So if bpf_prog_aux is inevitable, I'd just stick to that. As far as bpf_run_ctx, though, instead of bpf_run_ctx_type, shall we just put struct bpf_prog_aux * pointer into common struct bpf_run_ctx and always set it for all program types that support run_ctx?
Quoted 18 lines
Quoted 18 lines
And in the trampoline itself it would be a hard-coded single word assignment on the stack, so should be basically a no-op from performance point of view.
Quoted 11 lines
[0]
commitbc049387b41f("bpf: Add support for__progargument suffix to pass inprog->aux")
https://lore.kernel.org/r/[email protected]Signed-off-by Amery Hung <[email protected]>--- include/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_struct_ops.c | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+)[...]
Quoted 95 lines · Andrii Nakryiko
Quoted 93 lines · Amery Hung
Quoted 20 lines · Andrii Nakryiko
Quoted 11 lines · Amery Hung
Allows
struct_opsimplementors to infer the callingstruct_opsinstance inside a kfunc throughprog->aux->this_st_ops. A new field, flags, is added tobpf_struct_ops. IfBPF_STRUCT_OPS_F_THIS_PTRis set in flags, a pointer to thestruct_opsstructure registered to the kernel (i.e.,kvalue->data) will be saved toprog->aux->this_st_ops. To access it in a kfunc, useBPF_STRUCT_OPS_F_THIS_PTRwith__progargument [0]. The verifier will fixup the argument with a pointer toprog->aux.this_st_opsis protected by rcu and is valid until astruct_opsmap is unregistered updated.Quoted 6 lines
For a
struct_opsmap withBPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the samethis_st_ops, cmpxchg is used. Only if a program is not already used in anotherstruct_opsmap also withBPF_STRUCT_OPS_F_THIS_PTRcan it be assigned to the currentstruct_opsmap.Quoted 8 lines
Have you considered an alternative to storing
this_st_opsinbpf_prog_auxby setting it at runtime (instruct_opstrampoline) intobpf_run_ctx(which I think all struct ops programs have set), and then letting anystruct_opskfunc just access it through current (see other uses ofbpf_run_ctx, if you are unfamiliar). This would avoid all this business with extra flags and passingbpf_prog_auxas an extra argument.I didn't know this. Thanks for suggesting an alternative!
Quoted 7 lines
There will be no "only one
struct_opsfor this BPF program" limitation
either: technically, you could have the same BPF program used from twostruct_opsmaps just fine (even at the same time). Then depending on whichstruct_opsis currently active, you'd have a corresponding bpf_run_ctx'sstruct_opspointer. It feels like a cleaner approach to me.This is a cleaner approach for
struct_opsoperators. To make it work for kfuncs called in timer callback, I thinkprog->aux->st_opsis still needed, but at least we can unify how to getthis_st_opsin kfunc, in a way that does not requires adding__progto every kfuncs.(quoted diff) +6
-0+enum bpf_run_ctx_type { + BPF_CG_RUN_CTX = 0, + BPF_TRACE_RUN_CTX, + BPF_TRAMP_RUN_CTX, + BPF_TIMER_RUN_CTX, +};struct
bpf_run_ctx{
+ enumbpf_run_ctx_typetype;
};(quoted diff) +3
-0+struct bpf_timer_run_ctx { + struct bpf_prog_aux *aux; +};struct bpf_tramp_run_ctx { ...+ void
*st_ops;
};In
bpf_struct_ops_prepare_trampoline(), thest_opsassignment will be emitted to the trampoline.In
bpf_timer_cb(), preparebpf_timer_run_ctx, wherest_opscomes fromprog->aux->this_st_opsand setcurrent->bpf_ctx.Finally, in kfuncs that want to know the current
struct_ops, call this new function below:(quoted diff) +16
-0+void *bpf_struct_ops_current_st_ops(void) +{ + struct bpf_prog_aux aux; + + if (!current->bpf_ctx) + return NULL; + + switch(current->bpf_ctx->type) { + case BPF_TRAMP_RUN_CTX: + return (struct bpf_tramp_run_ctx *)(current->bpf_ctx)->st_ops; + case BPF_TIMER_RUN_CTX: + aux = (struct bpf_timer_run_ctx *)(current->bpf_ctx)->aux; + return rcu_dereference(aux->this_st_ops); + } + return NULL; +}
Quoted 8 lines
Quoted 1 line
What do you think?
I'm not sure I particularly like different ways to get this
st_opspointer depending whether we are in an async callback or not. So ifbpf_prog_auxis inevitable, I'd just stick to that. As far asbpf_run_ctx, though, instead ofbpf_run_ctx_type, shall we just putstruct bpf_prog_aux *pointer into common structbpf_run_ctxand always set it for all program types that supportrun_ctx?
No. This is death by a thousand cuts.
bpf trampoline is getting slower and slower.
scx's 'this' case is not a common pattern.
Quoted 107 lines · Alexei Starovoitov
Quoted 95 lines · Andrii Nakryiko
Quoted 93 lines · Amery Hung
Quoted 20 lines · Andrii Nakryiko
Quoted 11 lines · Amery Hung
Allows
struct_opsimplementors to infer the callingstruct_opsinstance inside a kfunc throughprog->aux->this_st_ops. A new field, flags, is added tobpf_struct_ops. IfBPF_STRUCT_OPS_F_THIS_PTRis set in flags, a pointer to thestruct_opsstructure registered to the kernel (i.e.,kvalue->data) will be saved toprog->aux->this_st_ops. To access it in a kfunc, useBPF_STRUCT_OPS_F_THIS_PTRwith__progargument [0]. The verifier will fixup the argument with a pointer toprog->aux.this_st_opsis protected by rcu and is valid until astruct_opsmap is unregistered updated.Quoted 6 lines
For a
struct_opsmap withBPF_STRUCT_OPS_F_THIS_PTR, to make sure all programs in it have the samethis_st_ops, cmpxchg is used. Only if a program is not already used in anotherstruct_opsmap also withBPF_STRUCT_OPS_F_THIS_PTRcan it be assigned to the currentstruct_opsmap.Quoted 8 lines
Have you considered an alternative to storing
this_st_opsinbpf_prog_auxby setting it at runtime (instruct_opstrampoline) intobpf_run_ctx(which I think all struct ops programs have set), and then letting anystruct_opskfunc just access it through current (see other uses ofbpf_run_ctx, if you are unfamiliar). This would avoid all this business with extra flags and passingbpf_prog_auxas an extra argument.I didn't know this. Thanks for suggesting an alternative!
Quoted 7 lines
There will be no "only one
struct_opsfor this BPF program" limitation
either: technically, you could have the same BPF program used from twostruct_opsmaps just fine (even at the same time). Then depending on whichstruct_opsis currently active, you'd have a corresponding bpf_run_ctx'sstruct_opspointer. It feels like a cleaner approach to me.This is a cleaner approach for
struct_opsoperators. To make it work for kfuncs called in timer callback, I thinkprog->aux->st_opsis still needed, but at least we can unify how to getthis_st_opsin kfunc, in a way that does not requires adding__progto every kfuncs.(quoted diff) +6
-0+enum bpf_run_ctx_type { + BPF_CG_RUN_CTX = 0, + BPF_TRACE_RUN_CTX, + BPF_TRAMP_RUN_CTX, + BPF_TIMER_RUN_CTX, +};struct
bpf_run_ctx{
+ enumbpf_run_ctx_typetype;
};(quoted diff) +3
-0+struct bpf_timer_run_ctx { + struct bpf_prog_aux *aux; +};struct bpf_tramp_run_ctx { ...+ void
*st_ops;
};In
bpf_struct_ops_prepare_trampoline(), thest_opsassignment will be emitted to the trampoline.In
bpf_timer_cb(), preparebpf_timer_run_ctx, wherest_opscomes fromprog->aux->this_st_opsand setcurrent->bpf_ctx.Finally, in kfuncs that want to know the current
struct_ops, call this new function below:(quoted diff) +16
-0+void *bpf_struct_ops_current_st_ops(void) +{ + struct bpf_prog_aux aux; + + if (!current->bpf_ctx) + return NULL; + + switch(current->bpf_ctx->type) { + case BPF_TRAMP_RUN_CTX: + return (struct bpf_tramp_run_ctx *)(current->bpf_ctx)->st_ops; + case BPF_TIMER_RUN_CTX: + aux = (struct bpf_timer_run_ctx *)(current->bpf_ctx)->aux; + return rcu_dereference(aux->this_st_ops); + } + return NULL; +}Quoted 8 lines
Quoted 1 line
What do you think?
I'm not sure I particularly like different ways to get this
st_opspointer depending whether we are in an async callback or not. So ifbpf_prog_auxis inevitable, I'd just stick to that. As far asbpf_run_ctx, though, instead ofbpf_run_ctx_type, shall we just putstruct bpf_prog_aux *pointer into common structbpf_run_ctxand always set it for all program types that supportrun_ctx?
Quoted 3 lines
No. This is death by a thousand cuts.
bpf trampoline is getting slower and slower.
scx's 'this' case is not a common pattern.
ok, no big deal