Attach a cookie to a tracing program.
Subsystems (12)
v7 cover Attach a cookie to a tracing program.
Allow users to attach a 64-bits cookie to a bpf_link of fentry, fexit, or fmod_ret.
This patchset includes several major changes.
- Define struct bpf_tramp_links to replace bpf_tramp_prog. struct bpf_tramp_links collects bpf_links of a trampoline
- Generate a trampoline to call bpf_progs of given bpf_links.
- Trampolines always set/reset bpf_run_ctx before/after calling/leaving a tracing program.
- Attach a cookie to a bpf_link of fentry/fexit/fmod_ret/lsm. The value will be available when running the associated bpf_prog.
Th major differences from v6:
- bpf_link_create() can create links of BPF_LSM_MAC attach type.
- Add a test for lsm.
- Add function proto of bpf_get_attach_cookie() for lsm.
- Check BPF_LSM_MAC in bpf_prog_has_trampoline().
- Adapt to the changes of LINK_CREATE made by Andrii.
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/bpf/[email protected]/
v3: https://lore.kernel.org/bpf/[email protected]/
v4: https://lore.kernel.org/bpf/[email protected]/
v5: https://lore.kernel.org/bpf/[email protected]/
v6: https://lore.kernel.org/bpf/[email protected]/
Kui-Feng Lee (5):
bpf, x86: Generate trampolines from bpf_tramp_links bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.
libbpf: Assign cookies to links in libbpf. selftest/bpf: The test cses of BPF cookie for
fentry/fexit/fmod_ret/lsm.
arch/x86/net/bpf_jit_comp.c | 82 +++++++++++--- include/linux/bpf.h | 54 +++++++--- include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 10 ++ kernel/bpf/bpf_lsm.c | 17 +++ kernel/bpf/bpf_struct_ops.c | 69 ++++++++---- kernel/bpf/syscall.c | 42 ++++---- kernel/bpf/trampoline.c | 100 +++++++++++------- kernel/trace/bpf_trace.c | 17 +++ net/bpf/bpf_dummy_struct_ops.c | 36 ++++++- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 10 ++ tools/lib/bpf/bpf.c | 8 ++ tools/lib/bpf/bpf.h | 3 + tools/lib/bpf/libbpf.c | 32 ++++++ tools/lib/bpf/libbpf.h | 12 +++ tools/lib/bpf/libbpf.map | 1 + .../selftests/bpf/prog_tests/bpf_cookie.c | 89 ++++++++++++++++ .../selftests/bpf/progs/test_bpf_cookie.c | 52 +++++++-- 19 files changed, 509 insertions(+), 127 deletions(-)
Signature · 1 line
2.30.2
PATCH v7 1/5 bpf, x86: Generate trampolines from bpf_tramp_links
Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.
arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.
Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.
--- arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 36 +++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++---------- kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++--------------- net/bpf/bpf_dummy_struct_ops.c | 36 ++++++++++++++--- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 1 + 10 files changed, 174 insertions(+), 103 deletions(-)
arch/x86/net/bpf_jit_comp.c +19 -17
@@ -1761,10 +1761,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, }static int invoke_bpf_prog (const struct btf_func_model *m,u8 **pprog, -struct bpf_prog *p,int stack_size,bool save_ret) +struct bpf_tramp_link *l,int stack_size, +bool save_ret) {u8 *prog = *pprog;u8 *jmp_insn; +struct bpf_prog *p = l->link.prog;/* arg1: mov rdi, progs[i] */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p);
@@ -1849,14 +1851,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) }static int invoke_bpf (const struct btf_func_model *m,u8 **pprog, -struct bpf_tramp_progs *tp,int stack_size, +struct bpf_tramp_links *tl,int stack_size,bool save_ret) {int i;u8 *prog = *pprog; -for (i =0 ; i < tp->nr_progs; i++) { -if (invoke_bpf_prog (m, &prog, tp->progs[i], stack_size, +for (i =0 ; i < tl->nr_links; i++) { +if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, save_ret))return -EINVAL ; }
@@ -1865,7 +1867,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, }static int invoke_bpf_mod_ret (const struct btf_func_model *m,u8 **pprog, -struct bpf_tramp_progs *tp,int stack_size, +struct bpf_tramp_links *tl,int stack_size,u8 **branches) {u8 *prog = *pprog;
@@ -1876,8 +1878,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,*/ emit_mov_imm32 (&prog, false,BPF_REG_0 ,0 );emit_stx (&prog,BPF_DW ,BPF_REG_FP ,BPF_REG_0 ,-8 ); -for (i =0 ; i < tp->nr_progs; i++) { -if (invoke_bpf_prog (m, &prog, tp->progs[i], stack_size, true)) +for (i =0 ; i < tl->nr_links; i++) { +if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, true))return -EINVAL ;/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1979,14 +1981,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags)*/ int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end,const struct btf_func_model *m,u32 flags, -struct bpf_tramp_progs *tprogs, +struct bpf_tramp_links *tlinks,void *orig_call) {int ret, i, nr_args = m->nr_args;int regs_off, ip_off, args_off, stack_size = nr_args *8 ; -struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY ]; -struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT ]; -struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN ]; +struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY ]; +struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT ]; +struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN ];u8 **branches =NULL ;u8 *prog;bool save_ret;
@@ -2077,13 +2079,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } } -if (fentry->nr_progs) +if (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, flags &BPF_TRAMP_F_RET_FENTRY_RET ))return -EINVAL ; -if (fmod_ret->nr_progs) { - branches =kcalloc (fmod_ret->nr_progs,sizeof (u8 *), +if (fmod_ret->nr_links) { + branches =kcalloc (fmod_ret->nr_links,sizeof (u8 *),GFP_KERNEL );if (!branches)return -ENOMEM ;
@@ -2110,7 +2112,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i prog +=X86_PATCH_SIZE ; } -if (fmod_ret->nr_progs) { +if (fmod_ret->nr_links) {/* From Intel 64 and IA-32 Architectures Optimization * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler * Coding Rule 11: All branch targets should be 16-byte
@@ -2120,12 +2122,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i/* Update the branches saved in invoke_bpf_mod_ret with the * aligned address of do_fexit. */ -for (i =0 ; i < fmod_ret->nr_progs; i++) +for (i =0 ; i < fmod_ret->nr_links; i++)emit_cond_near_jump (&branches[i], prog, branches[i],X86_JNE ); } -if (fexit->nr_progs) +if (fexit->nr_links)if (invoke_bpf (m, &prog, fexit, regs_off, false)) { ret = -EINVAL ; goto cleanup;
include/linux/bpf.h +24 -12
@@ -723,11 +723,11 @@ struct btf_func_model {/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 */ -#define BPF_MAX_TRAMP_PROGS 38 +#define BPF_MAX_TRAMP_LINKS 38 -struct bpf_tramp_progs { -struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS ]; -int nr_progs; +struct bpf_tramp_links { +struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS ]; +int nr_links; };/* Different use cases for BPF trampoline:
@@ -753,7 +753,7 @@ struct bpf_tramp_progs {struct bpf_tramp_image ;int arch_prepare_bpf_trampoline (struct bpf_tramp_image *tr,void *image,void *image_end,const struct btf_func_model *m,u32 flags, -struct bpf_tramp_progs *tprogs, +struct bpf_tramp_links *tlinks,void *orig_call);/* these two functions are called from generated trampoline */ u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
@@ -852,9 +852,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( {return bpf_func (ctx, insnsi); } +#ifdef CONFIG_BPF_JIT -int bpf_trampoline_link_prog (struct bpf_prog *prog,struct bpf_trampoline *tr); -int bpf_trampoline_unlink_prog (struct bpf_prog *prog,struct bpf_trampoline *tr); +int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr); +int bpf_trampoline_unlink_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr);struct bpf_trampoline *bpf_trampoline_get (u64 key,struct bpf_attach_target_info *tgt_info);void bpf_trampoline_put (struct bpf_trampoline *tr);
@@ -905,12 +906,12 @@ int bpf_jit_charge_modmem(u32 size);void bpf_jit_uncharge_modmem (u32 size);bool bpf_prog_has_trampoline (const struct bpf_prog *prog);#else -static inline int bpf_trampoline_link_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ; } -static inline int bpf_trampoline_unlink_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_unlink_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ;
@@ -1009,7 +1010,6 @@ struct bpf_prog_aux {bool tail_call_reachable;bool xdp_has_frags;bool use_bpf_prog_pack; -struct hlist_node tramp_hlist;/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto;/* function name for valid attach_btf_id */
@@ -1096,6 +1096,18 @@ struct bpf_link_ops { struct bpf_link_info *info); }; +struct bpf_tramp_link { +struct bpf_link link; +struct hlist_node tramp_hlist; +}; + +struct bpf_tracing_link { +struct bpf_tramp_link link; +enum bpf_attach_type attach_type; +struct bpf_trampoline *trampoline; +struct bpf_prog *tgt_prog; +}; +struct bpf_link_primer {struct bpf_link *link;struct file *file;
@@ -1133,8 +1145,8 @@ bool bpf_struct_ops_get(const void *kdata);void bpf_struct_ops_put (const void *kdata);int bpf_struct_ops_map_sys_lookup_elem (struct bpf_map *map,void *key,void *value); -int bpf_struct_ops_prepare_trampoline (struct bpf_tramp_progs *tprogs, -struct bpf_prog *prog, +int bpf_struct_ops_prepare_trampoline (struct bpf_tramp_links *tlinks, +struct bpf_tramp_link *link,const struct btf_func_model *model,void *image,void *image_end);static inline bool bpf_try_module_get (const void *data,struct module *owner)
include/linux/bpf_types.h +1 -0
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp) BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)#endif BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi) +BPF_LINK_TYPE (BPF_LINK_TYPE_STRUCT_OPS , struct_ops)
include/uapi/linux/bpf.h +1 -0
@@ -1013,6 +1013,7 @@ enum bpf_link_type {BPF_LINK_TYPE_XDP =6 ,BPF_LINK_TYPE_PERF_EVENT =7 ,BPF_LINK_TYPE_KPROBE_MULTI =8 , +BPF_LINK_TYPE_STRUCT_OPS =9 , MAX_BPF_LINK_TYPE, };
kernel/bpf/bpf_struct_ops.c +47 -22
153 lines not on this page · raw message
kernel/bpf/syscall.c +8 -15
@@ -2864,19 +2864,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) }EXPORT_SYMBOL (bpf_link_get_from_fd); -struct bpf_tracing_link { -struct bpf_link link; -enum bpf_attach_type attach_type; -struct bpf_trampoline *trampoline; -struct bpf_prog *tgt_prog; -}; -static void bpf_tracing_link_release (struct bpf_link *link) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link); -WARN_ON_ONCE (bpf_trampoline_unlink_prog (link->prog, +WARN_ON_ONCE (bpf_trampoline_unlink_prog (&tr_link->link, tr_link->trampoline));bpf_trampoline_put (tr_link->trampoline);
@@ -2889,7 +2882,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)static void bpf_tracing_link_dealloc (struct bpf_link *link) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link);kfree (tr_link); }
@@ -2898,7 +2891,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link); seq_printf(seq, "attach_type:\t%d\n",
@@ -2909,7 +2902,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, struct bpf_link_info *info) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link); info->tracing.attach_type = tr_link->attach_type; bpf_trampoline_unpack_key(tr_link->trampoline->key,
@@ -2990,7 +2983,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -ENOMEM ; goto out_put_prog; } -bpf_link_init (&link->link,BPF_LINK_TYPE_TRACING , +bpf_link_init (&link->link.link,BPF_LINK_TYPE_TRACING , &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type;
@@ -3060,11 +3053,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, tgt_prog = prog->aux->dst_prog; } - err =bpf_link_prime (&link->link, &link_primer); + err =bpf_link_prime (&link->link.link, &link_primer);if (err) goto out_unlock; - err =bpf_trampoline_link_prog (prog, tr); + err =bpf_trampoline_link_prog (&link->link, tr);if (err) {bpf_link_cleanup (&link_primer); link =NULL ;
kernel/bpf/trampoline.c +41 -32
171 lines not on this page · raw message
net/bpf/bpf_dummy_struct_ops.c +31 -5
@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) args->args[3], args->args[4]); } +static void bpf_struct_ops_link_release (struct bpf_link *link) +{ +} + +static void bpf_struct_ops_link_dealloc (struct bpf_link *link) +{ +kfree (link); +} + +static const struct bpf_link_ops bpf_struct_ops_link_lops = { + .release = bpf_struct_ops_link_release, + .dealloc = bpf_struct_ops_link_dealloc, +}; +int bpf_struct_ops_test_run (struct bpf_prog *prog,const union bpf_attr *kattr,union bpf_attr __user *uattr) {const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;const struct btf_type *func_proto;struct bpf_dummy_ops_test_args *args; -struct bpf_tramp_progs *tprogs; +struct bpf_tramp_links *tlinks; +struct bpf_tramp_link *link =NULL ;void *image =NULL ;unsigned int op_idx;int prog_ret;
@@ -92,8 +107,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,if (IS_ERR (args))return PTR_ERR (args); - tprogs =kcalloc (BPF_TRAMP_MAX ,sizeof (*tprogs),GFP_KERNEL ); -if (!tprogs) { + tlinks =kcalloc (BPF_TRAMP_MAX ,sizeof (*tlinks),GFP_KERNEL ); +if (!tlinks) { err = -ENOMEM ; goto out; }
@@ -105,8 +120,17 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, }set_vm_flush_reset_perms (image); + link =kzalloc (sizeof (*link),GFP_USER ); +if (!link) { + err = -ENOMEM ; + goto out; + } +/* prog doesn't take the ownership of the reference from caller */ +bpf_prog_inc (prog); +bpf_link_init (&link->link,BPF_LINK_TYPE_STRUCT_OPS , &bpf_struct_ops_link_lops, prog); + op_idx = prog->expected_attach_type; - err =bpf_struct_ops_prepare_trampoline (tprogs, prog, + err =bpf_struct_ops_prepare_trampoline (tlinks, link, &st_ops->func_models[op_idx], image, image +PAGE_SIZE );if (err <0 )
@@ -124,7 +148,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, out:kfree (args);bpf_jit_free_exec (image); -kfree (tprogs); +if (link) +bpf_link_put (&link->link); +kfree (tlinks);return err; }
tools/bpf/bpftool/link.c +1 -0
@@ -23,6 +23,7 @@ static const char * const link_type_name[] = { [BPF_LINK_TYPE_XDP] = "xdp", [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", + [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", };static struct hashmap *link_table;
tools/include/uapi/linux/bpf.h +1 -0
@@ -1013,6 +1013,7 @@ enum bpf_link_type {BPF_LINK_TYPE_XDP =6 ,BPF_LINK_TYPE_PERF_EVENT =7 ,BPF_LINK_TYPE_KPROBE_MULTI =8 , +BPF_LINK_TYPE_STRUCT_OPS =9 , MAX_BPF_LINK_TYPE, };
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v7 1/5] bpf, x86: Generate trampolines from bpf_tramp_links
Quoted 12 lines · Kui-Feng Lee
Replace struct
bpf_tramp_progswith structbpf_tramp_linksto collect structbpf_tramp_link(s)for a trampoline. structbpf_tramp_linkextendsbpf_linkto act as a linked list node.
arch_prepare_bpf_trampoline()accepts a structbpf_tramp_linksto collects allbpf_tramp_link(s)that a trampoline should call.Change BPF trampoline and
bpf_struct_opsto passbpf_tramp_linksinstead ofbpf_tramp_progs.Signed-off-by Kui-Feng Lee <[email protected]>
Quoted 13 lines
--- arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 36 +++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++---------- kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++--------------- net/bpf/bpf_dummy_struct_ops.c | 36 ++++++++++++++--- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 1 + 10 files changed, 174 insertions(+), 103 deletions(-)
Two things that can be done as a follow up, otherwise LGTM:
[...]
kernel/bpf/bpf_struct_ops.c static int check_zero_holes(const struct btf_type *t, void *data) · line 325
Quoted 9 lines
kernel/bpf/bpf_struct_ops.c +7
-2-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs, - struct bpf_prog *prog, +static void bpf_struct_ops_link_release (struct bpf_link *link) +{ +} + +static void bpf_struct_ops_link_dealloc (struct bpf_link *link) +{ +kfree (link);
This works by accident because struct bpf_link is at the top of struct bpf_tramp_link. But to do this properly you'd need container_of() to get struct bpf_tramp_link and then free that. I don't think it needs a respin just for this, but please send a follow-up fix.
Quoted 7 lines
kernel/bpf/bpf_struct_ops.c +7
-0+} + +static const struct bpf_link_ops bpf_struct_ops_link_lops = { + .release = bpf_struct_ops_link_release, + .dealloc = bpf_struct_ops_link_dealloc, +}; +
[...]
Quoted 12 lines
net/bpf/bpf_dummy_struct_ops.c +4
-0@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) args->args[3], args->args[4]); } +static void bpf_struct_ops_link_release (struct bpf_link *link) +{ +} +
net/bpf/bpf_dummy_struct_ops.c static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) · line 88
Quoted 10 lines
net/bpf/bpf_dummy_struct_ops.c +10
-0+static void bpf_struct_ops_link_dealloc (struct bpf_link *link) +{ +kfree (link); +} + +static const struct bpf_link_ops bpf_struct_ops_link_lops = { + .release = bpf_struct_ops_link_release, + .dealloc = bpf_struct_ops_link_dealloc, +}; +
You already defined this ops struct and release/dealloc implementation in kernel/bpf/bpf_struct_ops.c, we need to reuse it here. Just make the bpf_struct_ops.c's non-static and declare it in include/linux/bpf.h. Again, don't think we need a respin just for this, it's mostly code hygiene.
Quoted 12 lines
net/bpf/bpf_dummy_struct_ops.c +2
-1int bpf_struct_ops_test_run (struct bpf_prog *prog,const union bpf_attr *kattr,union bpf_attr __user *uattr) {const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;const struct btf_type *func_proto;struct bpf_dummy_ops_test_args *args; -struct bpf_tramp_progs *tprogs; +struct bpf_tramp_links *tlinks; +struct bpf_tramp_link *link =NULL ;void *image =NULL ;unsigned int op_idx;int prog_ret;
[...]
Re: [PATCH bpf-next v7 1/5] bpf, x86: Generate trampolines from bpf_tramp_links
Quoted 45 lines · Andrii Nakryiko
Quoted 26 lines · Kui-Feng Lee
Replace struct
bpf_tramp_progswith structbpf_tramp_linksto collect
structbpf_tramp_link(s)for a trampoline. structbpf_tramp_linkextendsbpf_linkto act as a linked list node.
arch_prepare_bpf_trampoline()accepts a structbpf_tramp_linksto collects allbpf_tramp_link(s)that a trampoline should call.Change BPF trampoline and
bpf_struct_opsto passbpf_tramp_linksinstead ofbpf_tramp_progs.Signed-off-by Kui-Feng Lee <[email protected]>---
arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 36 +++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++-------- --
kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++-----------
----
net/bpf/bpf_dummy_struct_ops.c | 36 ++++++++++++++--- tools/bpf/bpftool/link.c | 1 +tools/include/uapi/linux/bpf.h| 1 +Quoted 2 lines
10 files changed, 174 insertions(+), 103 deletions(-)Two things that can be done as a follow up, otherwise LGTM:
Acked-by Andrii Nakryiko <[email protected]>[...]
Quoted 9 lines
-int bpf_struct_ops_prepare_trampoline(struct
bpf_tramp_progs*tprogs,
- structbpf_prog*prog,kernel/bpf/bpf_struct_ops.c +6
-0+static void bpf_struct_ops_link_release (struct bpf_link *link) +{ +} + +static void bpf_struct_ops_link_dealloc (struct bpf_link *link) +{
Quoted 9 lines
Quoted 1 line
+ kfree(link);
This works by accident because struct
bpf_linkis at the top of structbpf_tramp_link. But to do this properly you'd needcontainer_of()to get structbpf_tramp_linkand then free that. I don't think it needs a
respin just for this, but please send a follow-up fix.
Fixed!
Quoted 33 lines
Quoted 7 lines
kernel/bpf/bpf_struct_ops.c +7
-0+} + +static const struct bpf_link_ops bpf_struct_ops_link_lops = { + .release = bpf_struct_ops_link_release, + .dealloc = bpf_struct_ops_link_dealloc, +}; +[...]
Quoted 23 lines
net/bpf/bpf_dummy_struct_ops.c +13
-0@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) args->args[3], args->args[4]); } +static void bpf_struct_ops_link_release (struct bpf_link *link) +{ +} + +static void bpf_struct_ops_link_dealloc (struct bpf_link *link) +{ + kfree(link); +} + +static const struct bpf_link_ops bpf_struct_ops_link_lops = { + .release = bpf_struct_ops_link_release, + .dealloc = bpf_struct_ops_link_dealloc, +};
Quoted 8 lines
net/bpf/bpf_dummy_struct_ops.c static int dummy_ops_call_op(void *image, · line 89
Quoted 1 line
net/bpf/bpf_dummy_struct_ops.c +1
-0+You already defined this ops struct and release/dealloc implementation
inkernel/bpf/bpf_struct_ops.c, we need to reuse it here. Just make the bpf_struct_ops.c's non-static and declare it ininclude/linux/bpf.h. Again, don't think we need a respin just for this, it's mostly code hygiene.
Fixed!
PATCH v7 2/5 bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
In reply to Kui-Feng LeeBPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.
--- arch/x86/net/bpf_jit_comp.c | 38 +++++++++++++++++++++++++++++++++++++ include/linux/bpf.h | 17 +++++++++++++---- kernel/bpf/syscall.c | 7 +++++-- kernel/bpf/trampoline.c | 20 +++++++++++++++---- 4 files changed, 72 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +38 -0
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, {u8 *prog = *pprog;u8 *jmp_insn; +int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; +/* mov rdi, 0 */ +emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); + +/* Prepare struct bpf_tramp_run_ctx. +* +* bpf_tramp_run_ctx is already preserved by +* arch_prepare_bpf_trampoline(). +* +* mov QWORD PTR [rsp + ctx_cookie_off], rdi +*/ +EMIT4 (0x48 ,0x89 ,0x7C ,0x24 );EMIT1 (ctx_cookie_off); +/* arg1: mov rdi, progs[i] */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p); +/* arg2: mov rsi, rsp (struct bpf_run_ctx *) */ +EMIT3 (0x48 ,0x89 ,0xE6 ); +if (emit_call (&prog, p->aux->sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter, prog))
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p);/* arg2: mov rsi, rbx <- start time in nsec */ emit_mov_reg (&prog, true,BPF_REG_2 ,BPF_REG_6 ); +/* arg3: mov rdx, rsp (struct bpf_run_ctx *) */ +EMIT3 (0x48 ,0x89 ,0xE2 );if (emit_call (&prog, p->aux->sleepable ? __bpf_prog_exit_sleepable : __bpf_prog_exit, prog))
@@ -2079,6 +2097,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } } +/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); +if (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, flags &BPF_TRAMP_F_RET_FENTRY_RET ))
@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i }if (flags &BPF_TRAMP_F_CALL_ORIG ) { +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); +restore_regs (m, &prog, nr_args, regs_off);/* call original function */
@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i im->ip_after_call = prog;memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ; + +/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); }if (fmod_ret->nr_links) {
@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i goto cleanup; } +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); +if (flags &BPF_TRAMP_F_RESTORE_REGS )restore_regs (m, &prog, nr_args, regs_off);
include/linux/bpf.h +13 -4
@@ -730,6 +730,8 @@ struct bpf_tramp_links {int nr_links; }; +struct bpf_tramp_run_ctx ; +/* Different use cases for BPF trampoline: * 1. replace nop at the function entry (kprobe equivalent) * flags = BPF_TRAMP_F_RESTORE_REGS
@@ -756,10 +758,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *istruct bpf_tramp_links *tlinks, void *orig_call);/* these two functions are called from generated trampoline */ -u64 notrace __bpf_prog_enter(struct bpf_prog *prog); -void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); -u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog); -void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start); +u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); +void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx); +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); +void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, +struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
@@ -1351,6 +1354,12 @@ struct bpf_trace_run_ctx {u64 bpf_cookie; }; +struct bpf_tramp_run_ctx { +struct bpf_run_ctx run_ctx; +u64 bpf_cookie; +struct bpf_run_ctx *saved_run_ctx; +}; +static inline struct bpf_run_ctx *bpf_set_run_ctx (struct bpf_run_ctx *new_ctx) {struct bpf_run_ctx *old_ctx =NULL ;
kernel/bpf/syscall.c +5 -2
@@ -5001,6 +5001,7 @@ static bool syscall_prog_is_valid_access(int off, int size, BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) {struct bpf_prog *__maybe_unused prog; +struct bpf_tramp_run_ctx __maybe_unused run_ctx;switch (cmd) {case BPF_MAP_CREATE :
@@ -5028,13 +5029,15 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) return -EINVAL; } -if (!__bpf_prog_enter_sleepable (prog)) { + run_ctx.bpf_cookie =0 ; + run_ctx.saved_run_ctx =NULL ; +if (!__bpf_prog_enter_sleepable (prog, &run_ctx)) {/* recursion detected */ bpf_prog_put (prog);return -EBUSY ; } attr->test.retval =bpf_prog_run (prog, (void *) (long ) attr->test.ctx_in); -__bpf_prog_exit_sleepable (prog,0 /* bpf_prog_run does runtime stats */ ); +__bpf_prog_exit_sleepable (prog,0 /* bpf_prog_run does runtime stats */ , &run_ctx);bpf_prog_put (prog);return 0 ;#endif
kernel/bpf/trampoline.c +16 -4
@@ -568,11 +568,14 @@ static void notrace inc_misses_counter(struct bpf_prog *prog)* [2..MAX_U64] - execute bpf prog and record execution time. * This is start time. */ -u64 notrace __bpf_prog_enter(struct bpf_prog *prog) +u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)__acquires (RCU ) {rcu_read_lock ();migrate_disable (); + + run_ctx->saved_run_ctx =bpf_set_run_ctx (&run_ctx->run_ctx); +if (unlikely (__this_cpu_inc_return (*(prog->active)) !=1 )) {inc_misses_counter (prog);return 0 ;
@@ -602,29 +605,38 @@ static void notrace update_prog_stats(struct bpf_prog *prog, } } -void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) +void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)__releases (RCU ) { +bpf_reset_run_ctx (run_ctx->saved_run_ctx); +update_prog_stats (prog, start);__this_cpu_dec (*(prog->active));migrate_enable ();rcu_read_unlock (); } -u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog) +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) {rcu_read_lock_trace ();migrate_disable ();might_fault (); +if (unlikely (__this_cpu_inc_return (*(prog->active)) !=1 )) {inc_misses_counter (prog);return 0 ; } + + run_ctx->saved_run_ctx =bpf_set_run_ctx (&run_ctx->run_ctx); +return bpf_prog_start_time (); } -void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) +void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, +struct bpf_tramp_run_ctx *run_ctx) { +bpf_reset_run_ctx (run_ctx->saved_run_ctx); +update_prog_stats (prog, start);__this_cpu_dec (*(prog->active));migrate_enable ();
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
Quoted 7 lines · Kui-Feng Lee
BPF trampolines will create a
bpf_tramp_run_ctx, abpf_run_ctx, on stacks and set/reset the currentbpf_run_ctxbefore/after calling abpf_prog.Signed-off-by Kui-Feng Lee <[email protected]>---
Please preserve received Acked-by/Reviewed-by/etc tags that you got on previous iterations, unless you feel like you did some major changes that might invalidate reviewer's "approval".
Still looks good to me:
Quoted 6 lines
arch/x86/net/bpf_jit_comp.c | 38 +++++++++++++++++++++++++++++++++++++ include/linux/bpf.h | 17 +++++++++++++---- kernel/bpf/syscall.c | 7 +++++-- kernel/bpf/trampoline.c | 20 +++++++++++++++---- 4 files changed, 72 insertions(+), 10 deletions(-)
[...]
Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
In reply to Kui-Feng LeeQuoted 34 lines · Kui-Feng Lee
arch/x86/net/bpf_jit_comp.c +15
-0+/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); +if (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, flags &BPF_TRAMP_F_RET_FENTRY_RET ))@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i }if (flags &BPF_TRAMP_F_CALL_ORIG ) { +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); +restore_regs (m, &prog, nr_args, regs_off);/* call original function */ @@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i im->ip_after_call = prog;memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ; + +/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); }if (fmod_ret->nr_links) {
arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2173
Quoted 8 lines
arch/x86/net/bpf_jit_comp.c +5
-0@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i goto cleanup; } +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); +
What is the point of all of these additional sub/add rsp ?
It seems unconditionally increasing stack_size by sizeof(struct bpf_tramp_run_ctx) will achieve the same and above 4 extra insns won't be needed.
Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
Quoted 45 lines · Alexei Starovoitov
Quoted 44 lines · Kui-Feng Lee
+ /* Prepare structbpf_tramp_run_ctx.
+ * sub rsp,sizeof(struct bpf_tramp_run_ctx)
+ */
+ EMIT4(0x48,0x83,0xEC,sizeof(struct bpf_tramp_run_ctx)); +
if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))(quoted diff) +0
-0@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i
}
if (flags &BPF_TRAMP_F_CALL_ORIG) {
+ /* pop structbpf_tramp_run_ctx
+ * add rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ EMIT4(0x48,0x83,0xC4,sizeof(struct bpf_tramp_run_ctx));
+
restore_regs(m, &prog,nr_args,regs_off);
/* call original function */(quoted diff) +0
-0@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *iim->ip_after_call= prog;memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog +=X86_PATCH_SIZE;
+
+ /* Prepare structbpf_tramp_run_ctx. + * sub rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ EMIT4(0x48,0x83,0xEC,sizeof(struct bpf_tramp_run_ctx));
}
if (fmod_ret->nr_links) {(quoted diff) +0
-0@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i goto cleanup;
}
+ /* pop structbpf_tramp_run_ctx
+ * add rsp,sizeof(struct bpf_tramp_run_ctx)
Quoted 8 lines
Quoted 3 lines
+ */
+ EMIT4(0x48,0x83,0xC4,sizeof(struct bpf_tramp_run_ctx)); +What is the point of all of these additional sub/add rsp ? It seems unconditionally increasing
stack_sizebysizeof(struct bpf_tramp_run_ctx)
will achieve the same and above 4 extra insns won't be needed.
I think you are right.
Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
Quoted 58 lines · Kui-Feng Lee
Quoted 48 lines · Alexei Starovoitov
Quoted 47 lines · Kui-Feng Lee
+ /* Prepare structbpf_tramp_run_ctx.
+ * sub rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ EMIT4(0x48,0x83,0xEC,sizeof(struct bpf_tramp_run_ctx));
+
if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))(quoted diff) +0
-0@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i
}
if (flags &BPF_TRAMP_F_CALL_ORIG) {
+ /* pop structbpf_tramp_run_ctx
+ * add rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ EMIT4(0x48,0x83,0xC4,sizeof(struct bpf_tramp_run_ctx));
+
restore_regs(m, &prog,nr_args,regs_off);
/* call original function */(quoted diff) +0
-0@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *iim->ip_after_call= prog;memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog +=X86_PATCH_SIZE;
+
+ /* Prepare structbpf_tramp_run_ctx. + * sub rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ EMIT4(0x48,0x83,0xEC,sizeof(struct bpf_tramp_run_ctx));
}
if (fmod_ret->nr_links) {(quoted diff) +0
-0@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i goto cleanup;
}
+ /* pop structbpf_tramp_run_ctx
+ * add rsp,sizeof(struct bpf_tramp_run_ctx)+ */Quoted 8 lines
Quoted 3 lines
+ EMIT4(0x48,
0x83,0xC4, sizeof(structbpf_tramp_run_ctx));
+What is the point of all of these additional sub/add rsp ? It seems unconditionally increasing
stack_sizebysizeof(struct bpf_tramp_run_ctx)
will achieve the same and above 4 extra insns won't be needed.
Quoted 2 lines
I think you are right.
The reason that I don't change stack_size is that we access arguments or saved registers basing on stack_size. Once the stack_size is changed, all these offsets should be changed too.
Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
Quoted 60 lines · Kui-Feng Lee
Quoted 58 lines · Kui-Feng Lee
Quoted 56 lines · Alexei Starovoitov
Quoted 48 lines · Kui-Feng Lee
arch/x86/net/bpf_jit_comp.c +4
-0+/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ + EMIT4(0x48, 0x83, 0xEC, sizeof(struct
bpf_tramp_run_ctx));
+
if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry,regs_off,
flags &BPF_TRAMP_F_RET_FENTRY_RET))(quoted diff) +0
-0@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i} if (flags & BPF_TRAMP_F_CALL_ORIG) {arch/x86/net/bpf_jit_comp.c +4
-0+/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ + EMIT4(0x48, 0x83, 0xC4, sizeof(struct
bpf_tramp_run_ctx));arch/x86/net/bpf_jit_comp.c +1
-0+restore_regs (m, &prog, nr_args, regs_off);/* call original function */ @@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *iim->ip_after_call = prog; memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE;arch/x86/net/bpf_jit_comp.c +5
-0+ +/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ + EMIT4(0x48, 0x83, 0xEC, sizeof(structbpf_tramp_run_ctx)); } if (fmod_ret->nr_links) {(quoted diff) +0
-0@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *igoto cleanup; }arch/x86/net/bpf_jit_comp.c +4
-0+/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ + EMIT4(0x48, 0x83, 0xC4, sizeof(structQuoted 2 lines
bpf_tramp_run_ctx));
+What is the point of all of these additional sub/add rsp ? It seems unconditionally increasing
stack_sizebysizeof(struct bpf_tramp_run_ctx)
will achieve the same and above 4 extra insns won't be needed.
Quoted 6 lines
Quoted 2 lines
I think you are right.
The reason that I don't change
stack_sizeis that we access arguments or saved registers basing onstack_size. Once thestack_sizeis changed, all these offsets should be changed too.
That should be trivial.
keep regs_off = stack_size;
and increase stack_size right after.
or some other math.
Maybe worth introducing another _off
in addition to int regs_off, ip_off, args_off. Definitely update 'Generated trampoline stack layout' comment and explain where bpf_tramp_run_ctx is in relation to regs_off. Maybe keeping regs_off (without new _off) is enough.
PATCH v7 3/5 bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.
In reply to Kui-Feng LeePass a cookie along with BPF_LINK_CREATE requests.
Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie. The cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.
The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.
--- arch/x86/net/bpf_jit_comp.c | 12 ++++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 9 +++++++++ kernel/bpf/bpf_lsm.c | 17 +++++++++++++++++ kernel/bpf/syscall.c | 12 ++++++++---- kernel/bpf/trampoline.c | 7 +++++-- kernel/trace/bpf_trace.c | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h | 9 +++++++++ 8 files changed, 76 insertions(+), 8 deletions(-)
arch/x86/net/bpf_jit_comp.c +10 -2
@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, bool save_ret) { + u64 cookie = 0;u8 *prog = *pprog;u8 *jmp_insn;int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; -/* mov rdi, 0 */ -emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); +if (l->link.type ==BPF_LINK_TYPE_TRACING ) { +struct bpf_tracing_link *tr_link = + container_of(l, struct bpf_tracing_link, link); + + cookie = tr_link->cookie; + } + +/* mov rdi, cookie */ +emit_mov_imm64 (&prog,BPF_REG_1 , (long ) cookie >>32 , (u32 ) (long ) cookie);/* Prepare struct bpf_tramp_run_ctx. *
include/linux/bpf.h +1 -0
@@ -1109,6 +1109,7 @@ struct bpf_tracing_link {enum bpf_attach_type attach_type;struct bpf_trampoline *trampoline;struct bpf_prog *tgt_prog; +u64 cookie; };struct bpf_link_primer {
include/uapi/linux/bpf.h +9 -0
@@ -1490,6 +1490,15 @@ union bpf_attr {__aligned_u64 addrs;__aligned_u64 cookies; } kprobe_multi; +struct { +/* this is overliad with the target_btf_id above. */ +__u32 target_btf_id; +/* black box user-provided value passed through +* to BPF program at the execution time and +* accessible through bpf_get_attach_cookie() BPF helper +*/ +__u64 cookie; + } tracing; }; } link_create;
kernel/bpf/bpf_lsm.c +17 -0
@@ -117,6 +117,21 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { .allowed = bpf_ima_inode_hash_allowed, }; +BPF_CALL_1(bpf_get_attach_cookie, void *, ctx) +{ +struct bpf_trace_run_ctx *run_ctx; + + run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); +return run_ctx->bpf_cookie; +} + +static const struct bpf_func_proto bpf_get_attach_cookie_proto = { + .func = bpf_get_attach_cookie, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_CTX , +}; +static const struct bpf_func_proto *bpf_lsm_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog) {
@@ -141,6 +156,8 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return prog->aux->sleepable ? &bpf_ima_inode_hash_proto :NULL ;case BPF_FUNC_ima_file_hash:return prog->aux->sleepable ? &bpf_ima_file_hash_proto :NULL ; +case BPF_FUNC_get_attach_cookie: +return bpf_prog_has_trampoline (prog) ? &bpf_get_attach_cookie_proto :NULL ;default :return tracing_prog_func_proto (func_id, prog); }
kernel/bpf/syscall.c +8 -4
@@ -2921,7 +2921,8 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {static int bpf_tracing_prog_attach (struct bpf_prog *prog,int tgt_prog_fd, -u32 btf_id) +u32 btf_id, +u64 bpf_cookie) {struct bpf_link_primer link_primer;struct bpf_prog *tgt_prog =NULL ;
@@ -2986,6 +2987,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,bpf_link_init (&link->link.link,BPF_LINK_TYPE_TRACING , &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type; + link->cookie = bpf_cookie;mutex_lock (&prog->aux->dst_mutex);
@@ -3271,7 +3273,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, tp_name = prog->aux->attach_func_name;break ; } -return bpf_tracing_prog_attach (prog,0 ,0 ); +return bpf_tracing_prog_attach (prog,0 ,0 ,0 );case BPF_PROG_TYPE_RAW_TRACEPOINT :case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE :if (strncpy_from_user (buf, user_tp_name,sizeof (buf) -1 ) <0 )
@@ -4524,7 +4526,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)case BPF_PROG_TYPE_EXT : ret =bpf_tracing_prog_attach (prog, attr->link_create.target_fd, - attr->link_create.target_btf_id); + attr->link_create.target_btf_id, + attr->link_create.tracing.cookie);break ;case BPF_PROG_TYPE_LSM :case BPF_PROG_TYPE_TRACING :
@@ -4539,7 +4542,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)else ret =bpf_tracing_prog_attach (prog, attr->link_create.target_fd, - attr->link_create.target_btf_id); + attr->link_create.target_btf_id, + attr->link_create.tracing.cookie);break ;case BPF_PROG_TYPE_FLOW_DISSECTOR :case BPF_PROG_TYPE_SK_LOOKUP :
kernel/bpf/trampoline.c +5 -2
@@ -30,9 +30,12 @@ static DEFINE_MUTEX(trampoline_mutex);bool bpf_prog_has_trampoline (const struct bpf_prog *prog) {enum bpf_attach_type eatype = prog->expected_attach_type; +enum bpf_prog_type ptype = prog->type; -return eatype ==BPF_TRACE_FENTRY || eatype ==BPF_TRACE_FEXIT || - eatype ==BPF_MODIFY_RETURN ; +return (ptype ==BPF_PROG_TYPE_TRACING && + (eatype ==BPF_TRACE_FENTRY || eatype ==BPF_TRACE_FEXIT || + eatype ==BPF_MODIFY_RETURN )) || + (ptype ==BPF_PROG_TYPE_LSM && eatype ==BPF_LSM_MAC ); }void *bpf_jit_alloc_exec_page (void )
kernel/trace/bpf_trace.c +17 -0
@@ -1091,6 +1091,21 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = { .arg1_type = ARG_PTR_TO_CTX, }; +BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx) +{ +struct bpf_trace_run_ctx *run_ctx; + + run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); +return run_ctx->bpf_cookie; +} + +static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = { + .func = bpf_get_attach_cookie_tracing, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_CTX , +}; + BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags) {#ifndef CONFIG_X86
@@ -1719,6 +1734,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return bpf_prog_has_trampoline (prog) ? &bpf_get_func_ret_proto :NULL ;case BPF_FUNC_get_func_arg_cnt:return bpf_prog_has_trampoline (prog) ? &bpf_get_func_arg_cnt_proto :NULL ; +case BPF_FUNC_get_attach_cookie: +return bpf_prog_has_trampoline (prog) ? &bpf_get_attach_cookie_proto_tracing :NULL ;default : fn =raw_tp_prog_func_proto (func_id, prog);if (!fn && prog->expected_attach_type ==BPF_TRACE_ITER )
tools/include/uapi/linux/bpf.h +9 -0
@@ -1490,6 +1490,15 @@ union bpf_attr {__aligned_u64 addrs;__aligned_u64 cookies; } kprobe_multi; +struct { +/* this is overliad with the target_btf_id above. */ +__u32 target_btf_id; +/* black box user-provided value passed through +* to BPF program at the execution time and +* accessible through bpf_get_attach_cookie() BPF helper +*/ +__u64 cookie; + } tracing; }; } link_create;
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v7 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.
Quoted 12 lines · Kui-Feng Lee
Pass a cookie along with
BPF_LINK_CREATErequests.Add a
bpf_cookiefield to structbpf_tracing_linkto attach a cookie. The cookie of abpf_tracing_linkis available by callingbpf_get_attach_cookiewhen running the BPF program of the attached link.The value of a cookie will be set at
bpf_tramp_run_ctxby the trampoline of the link.Signed-off-by Kui-Feng Lee <[email protected]>
Quoted 11 lines
--- arch/x86/net/bpf_jit_comp.c | 12 ++++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 9 +++++++++ kernel/bpf/bpf_lsm.c | 17 +++++++++++++++++ kernel/bpf/syscall.c | 12 ++++++++---- kernel/bpf/trampoline.c | 7 +++++-- kernel/trace/bpf_trace.c | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h | 9 +++++++++ 8 files changed, 76 insertions(+), 8 deletions(-)
LGTM with a suggestion for some follow up clean up.
Quoted 7 lines
arch/x86/net/bpf_jit_comp.c +0
-0@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, bool save_ret)
arch/x86/net/bpf_jit_comp.c static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · line 1773
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +2
-2{ +u64 cookie =0 ;u8 *prog = *pprog;u8 *jmp_insn;int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; -/* mov rdi, 0 */ -emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); +if (l->link.type ==BPF_LINK_TYPE_TRACING ) {
It would probably be nicer to put cookie field into struct bpf_tramp_link instead so that the JIT compiler doesn't have to do this special handling. It also makes sense that struct bpf_trampoline *trampoline is moved into struct bpf_tramp_link itself (given trampoline is always there for bpf_tramp_link).
Quoted 11 lines
arch/x86/net/bpf_jit_comp.c +8
-0+struct bpf_tracing_link *tr_link = + container_of(l, struct bpf_tracing_link, link); + + cookie = tr_link->cookie; + } + +/* mov rdi, cookie */ +emit_mov_imm64 (&prog,BPF_REG_1 , (long ) cookie >>32 , (u32 ) (long ) cookie);/* Prepare struct bpf_tramp_run_ctx. *
[...]
Re: [PATCH bpf-next v7 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.
Quoted 48 lines · Andrii Nakryiko
Quoted 22 lines · Kui-Feng Lee
Pass a cookie along with
BPF_LINK_CREATErequests.Add a
bpf_cookiefield to structbpf_tracing_linkto attach a cookie.
The cookie of abpf_tracing_linkis available by callingbpf_get_attach_cookiewhen running the BPF program of the attached link.The value of a cookie will be set at
bpf_tramp_run_ctxby the trampoline of the link.Signed-off-by Kui-Feng Lee <[email protected]>---
arch/x86/net/bpf_jit_comp.c | 12 ++++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 9 +++++++++ kernel/bpf/bpf_lsm.c | 17 +++++++++++++++++ kernel/bpf/syscall.c | 12 ++++++++---- kernel/bpf/trampoline.c | 7 +++++-- kernel/trace/bpf_trace.c | 17 +++++++++++++++++tools/include/uapi/linux/bpf.h| 9 +++++++++Quoted 2 lines
8 files changed, 76 insertions(+), 8 deletions(-)LGTM with a suggestion for some follow up clean up.
Acked-by Andrii Nakryiko <[email protected]>Quoted 18 lines
arch/x86/net/bpf_jit_comp.c +0
-0@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,struct
bpf_tramp_link*l, intstack_size,
boolsave_ret)
{
+ u64 cookie = 0;
u8 *prog = *pprog;
u8*jmp_insn;
intctx_cookie_off=offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
structbpf_prog*p =l->link.prog;
Quoted 9 lines
Quoted 3 lines
- /* mov rdi, 0 */
-emit_mov_imm64(&prog, BPF_REG_1, 0, 0); + if (l->link.type==BPF_LINK_TYPE_TRACING) {It would probably be nicer to put cookie field into struct
bpf_tramp_linkinstead so that the JIT compiler doesn't have to do this special handling. It also makes sense that structbpf_trampoline*trampoline is moved into structbpf_tramp_linkitself (given trampoline is always there forbpf_tramp_link).
It will increase the size of bpf_tramp_link a little bit, but they are not used by bpf_struct_ops.
Quoted 16 lines
Quoted 14 lines
+ struct
bpf_tracing_link*tr_link=
+container_of(l, struct bpf_tracing_link, link);arch/x86/net/bpf_jit_comp.c +6
-0+ + cookie = tr_link->cookie; + } + + /* mov rdi, cookie */ + emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32)(long) cookie);
/* Prepare struct
bpf_tramp_run_ctx.
*[...]
Re: [PATCH bpf-next v7 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.
Quoted 60 lines · Kui-Feng Lee
Quoted 48 lines · Andrii Nakryiko
Quoted 13 lines · Kui-Feng Lee
Pass a cookie along with
BPF_LINK_CREATErequests.Add a
bpf_cookiefield to structbpf_tracing_linkto attach a cookie.
The cookie of abpf_tracing_linkis available by callingbpf_get_attach_cookiewhen running the BPF program of the attached link.The value of a cookie will be set at
bpf_tramp_run_ctxby the trampoline of the link.Signed-off-by Kui-Feng Lee <[email protected]>Quoted 11 lines
--- arch/x86/net/bpf_jit_comp.c | 12 ++++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 9 +++++++++ kernel/bpf/bpf_lsm.c | 17 +++++++++++++++++ kernel/bpf/syscall.c | 12 ++++++++---- kernel/bpf/trampoline.c | 7 +++++-- kernel/trace/bpf_trace.c | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h | 9 +++++++++ 8 files changed, 76 insertions(+), 8 deletions(-)LGTM with a suggestion for some follow up clean up.
Acked-by Andrii Nakryiko <[email protected]>Quoted 18 lines
arch/x86/net/bpf_jit_comp.c +1
-0@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, bool save_ret) { +u64 cookie =0 ;u8 *prog = *pprog;u8 *jmp_insn;int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog;Quoted 9 lines
arch/x86/net/bpf_jit_comp.c static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · line 1773
Quoted 3 lines
arch/x86/net/bpf_jit_comp.c +1
-2-/* mov rdi, 0 */ -emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); +if (l->link.type ==BPF_LINK_TYPE_TRACING ) {It would probably be nicer to put cookie field into struct
bpf_tramp_linkinstead so that the JIT compiler doesn't have to do this special handling. It also makes sense that structbpf_trampoline*trampoline is moved into structbpf_tramp_linkitself (given trampoline is always there forbpf_tramp_link).
Quoted 3 lines
It will increase the size of
bpf_tramp_linka little bit, but they are not used bybpf_struct_ops.
It feels like the right tradeoff to keep architecture-specific trampoline code oblivious to these details. Some day structs_ops might support cookies as well. And either way 8 bytes for struct_ops link isn't a big deal.
Quoted 17 lines
Quoted 16 lines
Quoted 14 lines
arch/x86/net/bpf_jit_comp.c +2
-0+struct bpf_tracing_link *tr_link = + container_of(l, struct bpf_tracing_link,link);
arch/x86/net/bpf_jit_comp.c +6
-0+ + cookie = tr_link->cookie; + } + +/* mov rdi, cookie */ + emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32)(long) cookie);
/* Prepare struct bpf_tramp_run_ctx. *[...]
PATCH v7 4/5 libbpf: Assign cookies to links in libbpf.
In reply to Kui-Feng LeeAdd a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.
--- tools/lib/bpf/bpf.c | 8 ++++++++ tools/lib/bpf/bpf.h | 3 +++ tools/lib/bpf/libbpf.c | 32 ++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 12 ++++++++++++ tools/lib/bpf/libbpf.map | 1 + 5 files changed, 56 insertions(+)
tools/lib/bpf/bpf.c +8 -0
@@ -863,6 +863,14 @@ int bpf_link_create(int prog_fd, int target_fd,if (!OPTS_ZEROED (opts, kprobe_multi))return libbpf_err (-EINVAL );break ; +case BPF_TRACE_FENTRY : +case BPF_TRACE_FEXIT : +case BPF_MODIFY_RETURN : +case BPF_LSM_MAC : + attr.link_create.tracing.cookie =OPTS_GET (opts, tracing.cookie,0 ); +if (!OPTS_ZEROED (opts, tracing)) +return libbpf_err (-EINVAL ); +break ;default :if (!OPTS_ZEROED (opts, flags))return libbpf_err (-EINVAL );
tools/lib/bpf/bpf.h +3 -0
@@ -420,6 +420,9 @@ struct bpf_link_create_opts {const unsigned long *addrs;const __u64 *cookies; } kprobe_multi; +struct { +__u64 cookie; + } tracing; }; size_t :0; };
tools/lib/bpf/libbpf.c +32 -0
@@ -11440,6 +11440,38 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)return bpf_program__attach_btf_id (prog); } +struct bpf_link *bpf_program__attach_trace_opts (const struct bpf_program *prog, +const struct bpf_trace_opts *opts) +{ +char errmsg[STRERR_BUFSIZE ]; +struct bpf_link *link; +int prog_fd, pfd; +LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + prog_fd =bpf_program__fd (prog); +if (prog_fd <0 ) { +pr_warn ("prog '%s': can't attach before loaded\n" , prog->name); +return libbpf_err_ptr (-EINVAL ); + } + + link =calloc (1 ,sizeof (*link)); +if (!link) +return libbpf_err_ptr (-ENOMEM ); + link->detach = &bpf_link__detach_fd; + + link_opts.tracing.cookie =OPTS_GET (opts, cookie,0 ); + pfd =bpf_link_create (prog_fd,0 , prog->expected_attach_type, &link_opts); +if (pfd <0 ) { + pfd = -errno; +free (link); +pr_warn ("prog '%s': failed to attach: %s\n" , + prog->name,libbpf_strerror_r (pfd, errmsg,sizeof (errmsg))); +return libbpf_err_ptr (pfd); + } + link->fd = pfd; +return link; +} +struct bpf_link *bpf_program__attach_lsm (const struct bpf_program *prog) {return bpf_program__attach_btf_id (prog);
tools/lib/bpf/libbpf.h +12 -0
@@ -603,8 +603,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, LIBBPF_API struct bpf_link *bpf_program__attach_raw_tracepoint (const struct bpf_program *prog,const char *tp_name); + +struct bpf_trace_opts { +/* size of this struct, for forward/backward compatibility */ +size_t sz; +/* custom user-provided value fetchable through bpf_get_attach_cookie() */ +__u64 cookie; +}; +#define bpf_trace_opts__last_field cookie + LIBBPF_API struct bpf_link *bpf_program__attach_trace (const struct bpf_program *prog); +LIBBPF_API struct bpf_link * +bpf_program__attach_trace_opts (const struct bpf_program *prog,const struct bpf_trace_opts *opts); + LIBBPF_API struct bpf_link *bpf_program__attach_lsm (const struct bpf_program *prog); LIBBPF_API struct bpf_link *
tools/lib/bpf/libbpf.map +1 -0
@@ -444,6 +444,7 @@ LIBBPF_0.8.0 { global: bpf_object__destroy_subskeleton; bpf_object__open_subskeleton; + bpf_program__attach_trace_opts; bpf_program__attach_usdt; libbpf_register_prog_handler; libbpf_unregister_prog_handler;
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v7 4/5] libbpf: Assign cookies to links in libbpf.
Quoted 4 lines · Kui-Feng Lee
Add a cookie field to the attributes of
bpf_link_create(). Addbpf_program__attach_trace_opts()to attach a cookie to a link.
Quoted 9 lines
Signed-off-by Kui-Feng Lee <[email protected]>--- tools/lib/bpf/bpf.c | 8 ++++++++ tools/lib/bpf/bpf.h | 3 +++ tools/lib/bpf/libbpf.c | 32 ++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 12 ++++++++++++ tools/lib/bpf/libbpf.map | 1 + 5 files changed, 56 insertions(+)
I have a gripe with better code reuse, but that's internal change so we can do it in a follow up.
tools/lib/bpf/libbpf.c struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) · line 11444
Quoted 10 lines
tools/lib/bpf/libbpf.c +2
-0@@ -11440,6 +11440,38 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)return bpf_program__attach_btf_id (prog); } +struct bpf_link *bpf_program__attach_trace_opts (const struct bpf_program *prog, +const struct bpf_trace_opts *opts)
there is bpf_program__attach_btf_id() that does all of this except for the cookie. It would be nicer to extend bpf_program__attach_btf_id(), which won't break any API because it's an internal helper, add optional bpf_trace_opts to it and then just redirect bpf_program__attach_trace_opts() to bpf_program__attach_btf_id and update all the existing callers with just passing NULL for opts.
We can do that as a follow up, given your patch set seems to be pretty much ready to be landed.
Quoted 6 lines
tools/lib/bpf/libbpf.c +6
-0+{ +char errmsg[STRERR_BUFSIZE ]; +struct bpf_link *link; +int prog_fd, pfd; +LIBBPF_OPTS (bpf_link_create_opts, link_opts); +
[...]
Re: [PATCH bpf-next v7 4/5] libbpf: Assign cookies to links in libbpf.
Quoted 29 lines · Andrii Nakryiko
Quoted 4 lines · Kui-Feng Lee
Add a cookie field to the attributes of
bpf_link_create(). Addbpf_program__attach_trace_opts()to attach a cookie to a link.Quoted 9 lines
Signed-off-by Kui-Feng Lee <[email protected]>---
tools/lib/bpf/bpf.c | 8 ++++++++ tools/lib/bpf/bpf.h | 3 +++ tools/lib/bpf/libbpf.c | 32 ++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 12 ++++++++++++tools/lib/bpf/libbpf.map| 1 +5 files changed, 56 insertions(+)I have a gripe with better code reuse, but that's internal change so we can do it in a follow up.
Acked-by Andrii Nakryiko <[email protected]>Quoted 9 lines
tools/lib/bpf/libbpf.c +0
-0@@ -11440,6 +11440,38 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) return bpf_program__attach_btf_id(prog); }
Quoted 12 lines
tools/lib/bpf/libbpf.c struct bpf_link · line 11447
Quoted 4 lines
tools/lib/bpf/libbpf.c +2
-0+struct bpf_link *bpf_program__attach_trace_opts (const struct bpf_program *prog, + const structbpf_trace_opts *opts)there is
bpf_program__attach_btf_id()that does all of this except for
the cookie. It would be nicer to extendbpf_program__attach_btf_id(), which won't break any API because it's an internal helper, add optionalbpf_trace_optsto it and then just redirectbpf_program__attach_trace_opts()tobpf_program__attach_btf_idand update all the existing callers with just passing NULL for opts.
Fixed! I copied the code from bpf_program__attach_btf_id() for testing, and forgot to refactor it.
Quoted 13 lines
We can do that as a follow up, given your patch set seems to be pretty
much ready to be landed.Quoted 6 lines
tools/lib/bpf/libbpf.c +6
-0+{ + char errmsg[STRERR_BUFSIZE]; + struct bpf_link *link; + int prog_fd, pfd; + LIBBPF_OPTS(bpf_link_create_opts, link_opts); +[...]
PATCH v7 5/5 selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret/lsm.
In reply to Kui-Feng LeeMake sure BPF cookies are correct for fentry/fexit/fmod_ret/lsm.
--- .../selftests/bpf/prog_tests/bpf_cookie.c | 89 +++++++++++++++++++ .../selftests/bpf/progs/test_bpf_cookie.c | 52 +++++++++-- 2 files changed, 133 insertions(+), 8 deletions(-)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +89 -0
@@ -4,8 +4,11 @@#include <pthread.h> #include <sched.h> #include <sys/syscall.h> +#include <sys/mman.h> #include <unistd.h> #include <test_progs.h> +#include <network_helpers.h> +#include <bpf/btf.h> #include "test_bpf_cookie.skel.h" #include "kprobe_multi.skel.h"
@@ -410,6 +413,88 @@ static void pe_subtest(struct test_bpf_cookie *skel)bpf_link__destroy (link); } +static void tracing_subtest (struct test_bpf_cookie *skel) +{ +__u64 cookie; +int prog_fd; +int fentry_fd =-1 , fexit_fd =-1 , fmod_ret_fd =-1 ; +LIBBPF_OPTS (bpf_test_run_opts, opts); +LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + skel->bss->fentry_res =0 ; + skel->bss->fexit_res =0 ; + + cookie =0x10000000000000L ; + prog_fd =bpf_program__fd (skel->progs.fentry_test1); + link_opts.tracing.cookie = cookie; + fentry_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_FENTRY , &link_opts); +if (!ASSERT_GE (fentry_fd,0 ,"fentry.link_create" )) + goto cleanup; + + cookie =0x20000000000000L ; + prog_fd =bpf_program__fd (skel->progs.fexit_test1); + link_opts.tracing.cookie = cookie; + fexit_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_FEXIT , &link_opts); +if (!ASSERT_GE (fexit_fd,0 ,"fexit.link_create" )) + goto cleanup; + + cookie =0x30000000000000L ; + prog_fd =bpf_program__fd (skel->progs.fmod_ret_test); + link_opts.tracing.cookie = cookie; + fmod_ret_fd =bpf_link_create (prog_fd,0 ,BPF_MODIFY_RETURN , &link_opts); +if (!ASSERT_GE (fmod_ret_fd,0 ,"fmod_ret.link_create" )) + goto cleanup; + + prog_fd =bpf_program__fd (skel->progs.fentry_test1); +bpf_prog_test_run_opts (prog_fd, &opts); + + prog_fd =bpf_program__fd (skel->progs.fmod_ret_test); +bpf_prog_test_run_opts (prog_fd, &opts); + +ASSERT_EQ (skel->bss->fentry_res,0x10000000000000L ,"fentry_res" ); +ASSERT_EQ (skel->bss->fexit_res,0x20000000000000L ,"fexit_res" ); +ASSERT_EQ (skel->bss->fmod_ret_res,0x30000000000000L ,"fmod_ret_res" ); + +cleanup: +if (fentry_fd >=0 ) +close (fentry_fd); +if (fexit_fd >=0 ) +close (fexit_fd); +if (fmod_ret_fd >=0 ) +close (fmod_ret_fd); +} + +int stack_mprotect (void ); + +static void lsm_subtest (struct test_bpf_cookie *skel) +{ +__u64 cookie; +int prog_fd; +int lsm_fd =-1 ; +LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + skel->bss->lsm_res =0 ; + + cookie =0x90000000000090L ; + prog_fd =bpf_program__fd (skel->progs.test_int_hook); + link_opts.tracing.cookie = cookie; + lsm_fd =bpf_link_create (prog_fd,0 ,BPF_LSM_MAC , &link_opts); +if (!ASSERT_GE (lsm_fd,0 ,"lsm.link_create" )) + goto cleanup; + +stack_mprotect (); +if (!ASSERT_EQ (errno,EPERM ,"stack_mprotect" )) + goto cleanup; + +usleep (1 ); + +ASSERT_EQ (skel->bss->lsm_res,0x90000000000090L ,"fentry_res" ); + +cleanup: +if (lsm_fd >=0 ) +close (lsm_fd); +} +void test_bpf_cookie (void ) {struct test_bpf_cookie *skel;
@@ -432,6 +517,10 @@ void test_bpf_cookie(void)tp_subtest (skel);if (test__start_subtest ("perf_event" ))pe_subtest (skel); +if (test__start_subtest ("trampoline" )) +tracing_subtest (skel); +if (test__start_subtest ("lsm" )) +lsm_subtest (skel);test_bpf_cookie__destroy (skel); }
tools/testing/selftests/bpf/progs/test_bpf_cookie.c +44 -8
66 lines not on this page · raw message
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v7 5/5] selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret/lsm.
Quoted 9 lines · Kui-Feng Lee
Make sure BPF cookies are correct for fentry/fexit/fmod_ret/lsm.
Signed-off-by Kui-Feng Lee <[email protected]>--- .../selftests/bpf/prog_tests/bpf_cookie.c | 89 +++++++++++++++++++ .../selftests/bpf/progs/test_bpf_cookie.c | 52 +++++++++-- 2 files changed, 133 insertions(+), 8 deletions(-)
Looks great.
[...]