Attach a cookie to a tracing program.
Subsystems (12)
v5 cover Attach a cookie to a tracing program.
Allow users to attach a 64-bits cookie to a bpf_link of fentry, fexit, or fmod_ret.
This patchset includes several major changes.
- Define struct bpf_tramp_links to replace bpf_tramp_prog. struct bpf_tramp_links collects bpf_links of a trampoline
- Generate a trampoline to call bpf_progs of given bpf_links.
- Trampolines always set/reset bpf_run_ctx before/after calling/leaving a tracing program.
- Attach a cookie to a bpf_link of fentry/fexit/fmod_ret. The value will be available when running the associated bpf_prog.
The major differences from v4:
- Remove condition checking in __bpf_prog_{enter,exit)
The major differences from v3:
- rebase v3 to latest tip.
The major differences from v2:
- Move the allocations of run_ctx (struct bpf_tramp_run_ctx) out of invoke_bpf_prog().
- Move hlist_node out of bpf_link and introduce struct bpf_tramp_link to own hlist_node.
- Store cookies at struct bpf_tracing_link.
- Use SIB byte to reduce the number of instructions to set cookie values. (Use RSP directly)
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/bpf/[email protected]/
v3: https://lore.kernel.org/bpf/[email protected]/
v4: https://lore.kernel.org/bpf/[email protected]/
Kui-Feng Lee (5):
bpf, x86: Generate trampolines from bpf_tramp_links
bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack bpf, x86: Attach a cookie to fentry/fexit/fmod_ret. lib/bpf: Assign cookies to links in libbpf.
selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.
arch/x86/net/bpf_jit_comp.c | 98 +++++++++++++++---- include/linux/bpf.h | 56 ++++++++--- include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 9 ++ kernel/bpf/bpf_struct_ops.c | 69 ++++++++----- kernel/bpf/syscall.c | 57 +++++++---- kernel/bpf/trampoline.c | 93 +++++++++++------- kernel/trace/bpf_trace.c | 17 ++++ net/bpf/bpf_dummy_struct_ops.c | 35 ++++++- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 8 ++ tools/lib/bpf/bpf.c | 7 ++ tools/lib/bpf/bpf.h | 3 + tools/lib/bpf/libbpf.c | 33 +++++++ tools/lib/bpf/libbpf.h | 12 +++ tools/lib/bpf/libbpf.map | 1 + .../selftests/bpf/prog_tests/bpf_cookie.c | 52 ++++++++++ .../selftests/bpf/progs/test_bpf_cookie.c | 24 +++++ 18 files changed, 459 insertions(+), 117 deletions(-)
Signature · 1 line
2.30.2
PATCH v5 1/5 bpf, x86: Generate trampolines from bpf_tramp_links
Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.
arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.
Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.
--- arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 38 ++++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++---------- kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++--------------- net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++--- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 1 + 10 files changed, 175 insertions(+), 103 deletions(-)
arch/x86/net/bpf_jit_comp.c +19 -17
@@ -1761,10 +1761,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, }static int invoke_bpf_prog (const struct btf_func_model *m,u8 **pprog, -struct bpf_prog *p,int stack_size,bool save_ret) +struct bpf_tramp_link *l,int stack_size, +bool save_ret) {u8 *prog = *pprog;u8 *jmp_insn; +struct bpf_prog *p = l->link.prog;/* arg1: mov rdi, progs[i] */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p);
@@ -1849,14 +1851,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) }static int invoke_bpf (const struct btf_func_model *m,u8 **pprog, -struct bpf_tramp_progs *tp,int stack_size, +struct bpf_tramp_links *tl,int stack_size,bool save_ret) {int i;u8 *prog = *pprog; -for (i =0 ; i < tp->nr_progs; i++) { -if (invoke_bpf_prog (m, &prog, tp->progs[i], stack_size, +for (i =0 ; i < tl->nr_links; i++) { +if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, save_ret))return -EINVAL ; }
@@ -1865,7 +1867,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, }static int invoke_bpf_mod_ret (const struct btf_func_model *m,u8 **pprog, -struct bpf_tramp_progs *tp,int stack_size, +struct bpf_tramp_links *tl,int stack_size,u8 **branches) {u8 *prog = *pprog;
@@ -1876,8 +1878,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,*/ emit_mov_imm32 (&prog, false,BPF_REG_0 ,0 );emit_stx (&prog,BPF_DW ,BPF_REG_FP ,BPF_REG_0 ,-8 ); -for (i =0 ; i < tp->nr_progs; i++) { -if (invoke_bpf_prog (m, &prog, tp->progs[i], stack_size, true)) +for (i =0 ; i < tl->nr_links; i++) { +if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, true))return -EINVAL ;/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1979,14 +1981,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags)*/ int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end,const struct btf_func_model *m,u32 flags, -struct bpf_tramp_progs *tprogs, +struct bpf_tramp_links *tlinks,void *orig_call) {int ret, i, nr_args = m->nr_args;int regs_off, ip_off, args_off, stack_size = nr_args *8 ; -struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY ]; -struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT ]; -struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN ]; +struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY ]; +struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT ]; +struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN ];u8 **branches =NULL ;u8 *prog;bool save_ret;
@@ -2077,13 +2079,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } } -if (fentry->nr_progs) +if (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, flags &BPF_TRAMP_F_RET_FENTRY_RET ))return -EINVAL ; -if (fmod_ret->nr_progs) { - branches =kcalloc (fmod_ret->nr_progs,sizeof (u8 *), +if (fmod_ret->nr_links) { + branches =kcalloc (fmod_ret->nr_links,sizeof (u8 *),GFP_KERNEL );if (!branches)return -ENOMEM ;
@@ -2110,7 +2112,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i prog +=X86_PATCH_SIZE ; } -if (fmod_ret->nr_progs) { +if (fmod_ret->nr_links) {/* From Intel 64 and IA-32 Architectures Optimization * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler * Coding Rule 11: All branch targets should be 16-byte
@@ -2120,12 +2122,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i/* Update the branches saved in invoke_bpf_mod_ret with the * aligned address of do_fexit. */ -for (i =0 ; i < fmod_ret->nr_progs; i++) +for (i =0 ; i < fmod_ret->nr_links; i++)emit_cond_near_jump (&branches[i], prog, branches[i],X86_JNE ); } -if (fexit->nr_progs) +if (fexit->nr_links)if (invoke_bpf (m, &prog, fexit, regs_off, false)) { ret = -EINVAL ; goto cleanup;
include/linux/bpf.h +26 -12
@@ -674,11 +674,11 @@ struct btf_func_model {/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 */ -#define BPF_MAX_TRAMP_PROGS 38 +#define BPF_MAX_TRAMP_LINKS 38 -struct bpf_tramp_progs { -struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS ]; -int nr_progs; +struct bpf_tramp_links { +struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS ]; +int nr_links; };/* Different use cases for BPF trampoline:
@@ -704,7 +704,7 @@ struct bpf_tramp_progs {struct bpf_tramp_image ;int arch_prepare_bpf_trampoline (struct bpf_tramp_image *tr,void *image,void *image_end,const struct btf_func_model *m,u32 flags, -struct bpf_tramp_progs *tprogs, +struct bpf_tramp_links *tlinks,void *orig_call);/* these two functions are called from generated trampoline */ u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( {return bpf_func (ctx, insnsi); } + +struct bpf_link ; +#ifdef CONFIG_BPF_JIT -int bpf_trampoline_link_prog (struct bpf_prog *prog,struct bpf_trampoline *tr); -int bpf_trampoline_unlink_prog (struct bpf_prog *prog,struct bpf_trampoline *tr); +int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr); +int bpf_trampoline_unlink_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr);struct bpf_trampoline *bpf_trampoline_get (u64 key,struct bpf_attach_target_info *tgt_info);void bpf_trampoline_put (struct bpf_trampoline *tr);
@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);void bpf_jit_uncharge_modmem (u32 size);bool bpf_prog_has_trampoline (const struct bpf_prog *prog);#else -static inline int bpf_trampoline_link_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ; } -static inline int bpf_trampoline_unlink_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_unlink_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ;
@@ -960,7 +963,6 @@ struct bpf_prog_aux {bool tail_call_reachable;bool xdp_has_frags;bool use_bpf_prog_pack; -struct hlist_node tramp_hlist;/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto;/* function name for valid attach_btf_id */
@@ -1047,6 +1049,18 @@ struct bpf_link_ops { struct bpf_link_info *info); }; +struct bpf_tramp_link { +struct bpf_link link; +struct hlist_node tramp_hlist; +}; + +struct bpf_tracing_link { +struct bpf_tramp_link link; +enum bpf_attach_type attach_type; +struct bpf_trampoline *trampoline; +struct bpf_prog *tgt_prog; +}; +struct bpf_link_primer {struct bpf_link *link;struct file *file;
@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);void bpf_struct_ops_put (const void *kdata);int bpf_struct_ops_map_sys_lookup_elem (struct bpf_map *map,void *key,void *value); -int bpf_struct_ops_prepare_trampoline (struct bpf_tramp_progs *tprogs, -struct bpf_prog *prog, +int bpf_struct_ops_prepare_trampoline (struct bpf_tramp_links *tlinks, +struct bpf_tramp_link *link,const struct btf_func_model *model,void *image,void *image_end);static inline bool bpf_try_module_get (const void *data,struct module *owner)
include/linux/bpf_types.h +1 -0
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp) BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)#endif BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi) +BPF_LINK_TYPE (BPF_LINK_TYPE_STRUCT_OPS , struct_ops)
include/uapi/linux/bpf.h +1 -0
@@ -1013,6 +1013,7 @@ enum bpf_link_type {BPF_LINK_TYPE_XDP =6 ,BPF_LINK_TYPE_PERF_EVENT =7 ,BPF_LINK_TYPE_KPROBE_MULTI =8 , +BPF_LINK_TYPE_STRUCT_OPS =9 , MAX_BPF_LINK_TYPE, };
kernel/bpf/bpf_struct_ops.c +47 -22
153 lines not on this page · raw message
kernel/bpf/syscall.c +8 -15
@@ -2640,19 +2640,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) }EXPORT_SYMBOL (bpf_link_get_from_fd); -struct bpf_tracing_link { -struct bpf_link link; -enum bpf_attach_type attach_type; -struct bpf_trampoline *trampoline; -struct bpf_prog *tgt_prog; -}; -static void bpf_tracing_link_release (struct bpf_link *link) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link); -WARN_ON_ONCE (bpf_trampoline_unlink_prog (link->prog, +WARN_ON_ONCE (bpf_trampoline_unlink_prog (&tr_link->link, tr_link->trampoline));bpf_trampoline_put (tr_link->trampoline);
@@ -2665,7 +2658,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)static void bpf_tracing_link_dealloc (struct bpf_link *link) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link);kfree (tr_link); }
@@ -2674,7 +2667,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link); seq_printf(seq, "attach_type:\t%d\n",
@@ -2685,7 +2678,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, struct bpf_link_info *info) {struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link); + container_of(link, struct bpf_tracing_link, link.link); info->tracing.attach_type = tr_link->attach_type; bpf_trampoline_unpack_key(tr_link->trampoline->key,
@@ -2766,7 +2759,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -ENOMEM ; goto out_put_prog; } -bpf_link_init (&link->link,BPF_LINK_TYPE_TRACING , +bpf_link_init (&link->link.link,BPF_LINK_TYPE_TRACING , &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type;
@@ -2836,11 +2829,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, tgt_prog = prog->aux->dst_prog; } - err =bpf_link_prime (&link->link, &link_primer); + err =bpf_link_prime (&link->link.link, &link_primer);if (err) goto out_unlock; - err =bpf_trampoline_link_prog (prog, tr); + err =bpf_trampoline_link_prog (&link->link, tr);if (err) {bpf_link_cleanup (&link_primer); link =NULL ;
kernel/bpf/trampoline.c +41 -32
171 lines not on this page · raw message
net/bpf/bpf_dummy_struct_ops.c +30 -5
@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) args->args[3], args->args[4]); } +static void bpf_struct_ops_link_release (struct bpf_link *link) +{ +} + +static void bpf_struct_ops_link_dealloc (struct bpf_link *link) +{ +kfree (link); +} + +static const struct bpf_link_ops bpf_struct_ops_link_lops = { + .release = bpf_struct_ops_link_release, + .dealloc = bpf_struct_ops_link_dealloc, +}; +int bpf_struct_ops_test_run (struct bpf_prog *prog,const union bpf_attr *kattr,union bpf_attr __user *uattr) {const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;const struct btf_type *func_proto;struct bpf_dummy_ops_test_args *args; -struct bpf_tramp_progs *tprogs; +struct bpf_tramp_links *tlinks; +struct bpf_tramp_link *link =NULL ;void *image =NULL ;unsigned int op_idx;int prog_ret;
@@ -92,8 +107,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,if (IS_ERR (args))return PTR_ERR (args); - tprogs =kcalloc (BPF_TRAMP_MAX ,sizeof (*tprogs),GFP_KERNEL ); -if (!tprogs) { + tlinks =kcalloc (BPF_TRAMP_MAX ,sizeof (*tlinks),GFP_KERNEL ); +if (!tlinks) { err = -ENOMEM ; goto out; }
@@ -105,10 +120,18 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, }set_vm_flush_reset_perms (image); + link =kzalloc (sizeof (*link),GFP_USER ); +if (!link) { + err = -ENOMEM ; + goto out; + } +bpf_link_init (&link->link,BPF_LINK_TYPE_STRUCT_OPS , &bpf_struct_ops_link_lops, prog); + op_idx = prog->expected_attach_type; - err =bpf_struct_ops_prepare_trampoline (tprogs, prog, + err =bpf_struct_ops_prepare_trampoline (tlinks, link, &st_ops->func_models[op_idx], image, image +PAGE_SIZE ); +if (err <0 ) goto out;
@@ -124,7 +147,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, out:kfree (args);bpf_jit_free_exec (image); -kfree (tprogs); +if (link) +bpf_link_put (&link->link); +kfree (tlinks);return err; }
tools/bpf/bpftool/link.c +1 -0
@@ -23,6 +23,7 @@ static const char * const link_type_name[] = { [BPF_LINK_TYPE_XDP] = "xdp", [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", + [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", };static struct hashmap *link_table;
tools/include/uapi/linux/bpf.h +1 -0
@@ -1013,6 +1013,7 @@ enum bpf_link_type {BPF_LINK_TYPE_XDP =6 ,BPF_LINK_TYPE_PERF_EVENT =7 ,BPF_LINK_TYPE_KPROBE_MULTI =8 , +BPF_LINK_TYPE_STRUCT_OPS =9 , MAX_BPF_LINK_TYPE, };
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v5 1/5] bpf, x86: Generate trampolines from bpf_tramp_links
Quoted 5 lines · Kui-Feng Lee
Replace struct
bpf_tramp_progswith structbpf_tramp_linksto collect structbpf_tramp_link(s)for a trampoline. structbpf_tramp_linkextendsbpf_linkto act as a linked list node.
Quoted 8 lines
arch_prepare_bpf_trampoline()accepts a structbpf_tramp_linksto collects allbpf_tramp_link(s)that a trampoline should call.Change BPF trampoline and
bpf_struct_opsto passbpf_tramp_linksinstead ofbpf_tramp_progs.Signed-off-by Kui-Feng Lee <[email protected]>---
Looks good, see two comments below.
Quoted 12 lines
arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 38 ++++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++---------- kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++--------------- net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++--- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 1 + 10 files changed, 175 insertions(+), 103 deletions(-)
[...]
Quoted 11 lines
/* Different use cases for BPF trampoline:
include/linux/bpf.h +1
-1@@ -704,7 +704,7 @@ struct bpf_tramp_progs {struct bpf_tramp_image ;int arch_prepare_bpf_trampoline (struct bpf_tramp_image *tr,void *image,void *image_end,const struct btf_func_model *m,u32 flags, -struct bpf_tramp_progs *tprogs, +struct bpf_tramp_links *tlinks,void *orig_call);/* these two functions are called from generated trampoline */ u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
include/linux/bpf.h static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( · line 808
Quoted 6 lines
include/linux/bpf.h +3
-0@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( {return bpf_func (ctx, insnsi); } + +struct bpf_link ; +
is this forward declaration still needed? was it supposed to be a struct bpf_tramp_link instead? and also probably higher above, before bpf_tramp_links?
Quoted 36 lines
include/linux/bpf.h +5
-5#ifdef CONFIG_BPF_JIT -int bpf_trampoline_link_prog (struct bpf_prog *prog,struct bpf_trampoline *tr); -int bpf_trampoline_unlink_prog (struct bpf_prog *prog,struct bpf_trampoline *tr); +int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr); +int bpf_trampoline_unlink_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr);struct bpf_trampoline *bpf_trampoline_get (u64 key,struct bpf_attach_target_info *tgt_info);void bpf_trampoline_put (struct bpf_trampoline *tr);@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);void bpf_jit_uncharge_modmem (u32 size);bool bpf_prog_has_trampoline (const struct bpf_prog *prog);#else -static inline int bpf_trampoline_link_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ; } -static inline int bpf_trampoline_unlink_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_unlink_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ;@@ -960,7 +963,6 @@ struct bpf_prog_aux {bool tail_call_reachable;bool xdp_has_frags;bool use_bpf_prog_pack; -struct hlist_node tramp_hlist;/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto;/* function name for valid attach_btf_id */ @@ -1047,6 +1049,18 @@ struct bpf_link_ops { struct bpf_link_info *info); }; +struct bpf_tramp_link {
include/linux/bpf.h struct bpf_link_ops { · line 1062
Quoted 10 lines
include/linux/bpf.h +10
-0+struct bpf_link link; +struct hlist_node tramp_hlist; +}; + +struct bpf_tracing_link { +struct bpf_tramp_link link; +enum bpf_attach_type attach_type; +struct bpf_trampoline *trampoline; +struct bpf_prog *tgt_prog; +};
struct bpf_tracing_link can stay in syscall.c, no? don't see anyone needing it outside of syscall.c
Quoted 14 lines
include/linux/bpf.h +3
-2+struct bpf_link_primer {struct bpf_link *link;struct file *file;@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);void bpf_struct_ops_put (const void *kdata);int bpf_struct_ops_map_sys_lookup_elem (struct bpf_map *map,void *key,void *value); -int bpf_struct_ops_prepare_trampoline (struct bpf_tramp_progs *tprogs, -struct bpf_prog *prog, +int bpf_struct_ops_prepare_trampoline (struct bpf_tramp_links *tlinks, +struct bpf_tramp_link *link,const struct btf_func_model *model,void *image,void *image_end);
[...]
Re: [PATCH bpf-next v5 1/5] bpf, x86: Generate trampolines from bpf_tramp_links
Quoted 53 lines · Andrii Nakryiko
Quoted 6 lines · Kui-Feng Lee
Replace struct
bpf_tramp_progswith structbpf_tramp_linksto collect
structbpf_tramp_link(s)for a trampoline. structbpf_tramp_linkextendsbpf_linkto act as a linked list node.Quoted 8 lines
arch_prepare_bpf_trampoline()accepts a structbpf_tramp_linksto collects allbpf_tramp_link(s)that a trampoline should call.Change BPF trampoline and
bpf_struct_opsto passbpf_tramp_linksinstead ofbpf_tramp_progs.Signed-off-by Kui-Feng Lee <[email protected]>---
Looks good, see two comments below.
Acked-by Andrii Nakryiko <[email protected]>Quoted 14 lines
arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 38 ++++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++-------- --
kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++-----------
----
net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++--- tools/bpf/bpftool/link.c | 1 +tools/include/uapi/linux/bpf.h| 1 +10 files changed, 175 insertions(+), 103 deletions(-)[...]
Quoted 16 lines
/* Different use cases for BPF trampoline:
include/linux/bpf.h +0
-0@@ -704,7 +704,7 @@ struct bpf_tramp_progs { struct bpf_tramp_image;int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*tr, void *image, void*image_end,
const structbtf_func_model*m, u32 flags,
- structbpf_tramp_progs*tprogs, + structbpf_tramp_links*tlinks, void*orig_call);
/* these two functions are called from generated trampoline */ u64 notrace__bpf_prog_enter(struct bpf_prog *prog);(quoted diff) +0
-0@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned intbpf_dispatcher_nop_func(
{
returnbpf_func(ctx, insnsi);
Quoted 8 lines
include/linux/bpf.h static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( · line 808
Quoted 4 lines
}
include/linux/bpf.h +3
-0+ +struct bpf_link ; +is this forward declaration still needed? was it supposed to be a struct
bpf_tramp_linkinstead? and also probably higher above, beforebpf_tramp_links?
You are right, I should remvoe it.
Quoted 46 lines
Quoted 46 lines
#ifdef
CONFIG_BPF_JIT
-intbpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-intbpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+intbpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+intbpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
structbpf_trampoline*bpf_trampoline_get(u64 key, structbpf_attach_target_info*tgt_info);
voidbpf_trampoline_put(struct bpf_trampoline *tr);include/linux/bpf.h +1
-1@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size); void bpf_jit_uncharge_modmem(u32 size); bool bpf_prog_has_trampoline(const struct bpf_prog *prog); #else -static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, +static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,struct
bpf_trampoline*tr)
{
return-ENOTSUPP;
}
-static inline int bpf_trampoline_unlink_prog(structbpf_prog*prog,
+static inline int bpf_trampoline_unlink_prog(structbpf_tramp_link*link,
structbpf_trampoline*tr)
{
return-ENOTSUPP;include/linux/bpf.h +0
-1@@ -960,7 +963,6 @@ struct bpf_prog_aux { bool tail_call_reachable; bool xdp_has_frags; bool use_bpf_prog_pack; - struct hlist_node tramp_hlist; /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto; /* function name for valid attach_btf_id */@@ -1047,6 +1049,18 @@ struct bpf_link_ops { struct bpf_link_info *info); };
Quoted 15 lines
include/linux/bpf.h struct bpf_link_ops { · line 1062
Quoted 12 lines
include/linux/bpf.h +5
-0+struct bpf_tramp_link { + struct bpf_link link; + struct hlist_node tramp_hlist; +}; +https://docs.google.com/document/d/1FYYdi1Hw4pjulbkvI1VmG-kGqTJRCg7bh1vAF4bwjMc/edit?usp=sharing
include/linux/bpf.h +6
-0+struct bpf_tracing_link { + struct bpf_tramp_link link; + enum bpf_attach_type attach_type; + struct bpf_trampoline *trampoline; + struct bpf_prog *tgt_prog; +};struct
bpf_tracing_linkcan stay insyscall.c, no? don't see anyone needing it outside ofsyscall.c
It will be used by invoke_bpf_prog() of bpf_jit_comp.c in the 3rd patch to get the cookie value.
Quoted 21 lines
Quoted 19 lines
+
structbpf_link_primer{
structbpf_link*link;
struct file *file;include/linux/bpf.h +0
-0@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata); void bpf_struct_ops_put(const void *kdata);int bpf_struct_ops_map_sys_lookup_elem(struct
bpf_map*map, void *key,
void *value);
-int bpf_struct_ops_prepare_trampoline(structbpf_tramp_progs*tprogs,
- structbpf_prog*prog, +int bpf_struct_ops_prepare_trampoline(structbpf_tramp_links*tlinks,
+ structbpf_tramp_link*link, const structbtf_func_model*model,
void *image, void*image_end);[...]
Re: [PATCH bpf-next v5 1/5] bpf, x86: Generate trampolines from bpf_tramp_links
Quoted 128 lines · Kui-Feng Lee
Quoted 53 lines · Andrii Nakryiko
Quoted 6 lines · Kui-Feng Lee
Replace struct
bpf_tramp_progswith structbpf_tramp_linksto collect
structbpf_tramp_link(s)for a trampoline. structbpf_tramp_linkextendsbpf_linkto act as a linked list node.Quoted 8 lines
arch_prepare_bpf_trampoline()accepts a structbpf_tramp_linksto collects allbpf_tramp_link(s)that a trampoline should call.Change BPF trampoline and
bpf_struct_opsto passbpf_tramp_linksinstead ofbpf_tramp_progs.Signed-off-by Kui-Feng Lee <[email protected]>---
Looks good, see two comments below.
Acked-by Andrii Nakryiko <[email protected]>Quoted 14 lines
arch/x86/net/bpf_jit_comp.c | 36 +++++++++-------- include/linux/bpf.h | 38 ++++++++++++------ include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/bpf_struct_ops.c | 69 ++++++++++++++++++++++-------- -- kernel/bpf/syscall.c | 23 ++++------- kernel/bpf/trampoline.c | 73 +++++++++++++++++++----------- ---- net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++--- tools/bpf/bpftool/link.c | 1 + tools/include/uapi/linux/bpf.h | 1 + 10 files changed, 175 insertions(+), 103 deletions(-)[...]
Quoted 16 lines
/* Different use cases for BPF trampoline:
include/linux/bpf.h +1
-1@@ -704,7 +704,7 @@ struct bpf_tramp_progs {struct bpf_tramp_image ;int arch_prepare_bpf_trampoline (struct bpf_tramp_image *tr,void *image,void *image_end,const struct btf_func_model *m,u32 flags, -struct bpf_tramp_progs *tprogs, +struct bpf_tramp_links *tlinks,void *orig_call);/* these two functions are called from generated trampoline */ u64 notrace
__bpf_prog_enter(struct bpf_prog *prog);(quoted diff) +0
-0@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned intbpf_dispatcher_nop_func(
{ return bpf_func(ctx, insnsi);Quoted 8 lines
include/linux/bpf.h static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( · line 808
Quoted 4 lines
}
include/linux/bpf.h +3
-0+ +struct bpf_link ; +is this forward declaration still needed? was it supposed to be a struct
bpf_tramp_linkinstead? and also probably higher above, beforebpf_tramp_links?You are right, I should remvoe it.
Quoted 46 lines
Quoted 46 lines
#ifdef
CONFIG_BPF_JIT
-intbpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-intbpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+intbpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+intbpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
structbpf_trampoline*bpf_trampoline_get(u64 key,
structbpf_attach_target_info*tgt_info);
voidbpf_trampoline_put(struct bpf_trampoline *tr);include/linux/bpf.h +2
-2@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);void bpf_jit_uncharge_modmem (u32 size);bool bpf_prog_has_trampoline (const struct bpf_prog *prog);#else -static inline int bpf_trampoline_link_prog (struct bpf_prog *prog, +static inline int bpf_trampoline_link_prog (struct bpf_tramp_link *link,struct bpf_trampoline *tr) {return -ENOTSUPP ; } -static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog, +static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link*link,
structbpf_trampoline
*tr){ return -ENOTSUPP;include/linux/bpf.h +0
-1@@ -960,7 +963,6 @@ struct bpf_prog_aux {bool tail_call_reachable;bool xdp_has_frags;bool use_bpf_prog_pack; -struct hlist_node tramp_hlist;/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto;/* function name for valid attach_btf_id */ @@ -1047,6 +1049,18 @@ struct bpf_link_ops { struct bpf_link_info *info); };Quoted 15 lines
include/linux/bpf.h struct bpf_link_ops { · line 1062
Quoted 12 lines
include/linux/bpf.h +5
-0+struct bpf_tramp_link { +struct bpf_link link; +struct hlist_node tramp_hlist; +}; +https://docs.google.com/document/d/1FYYdi1Hw4pjulbkvI1VmG-kGqTJRCg7bh1vAF4bwjMc/edit?usp=sharing
include/linux/bpf.h +6
-0+struct bpf_tracing_link { +struct bpf_tramp_link link; +enum bpf_attach_type attach_type; +struct bpf_trampoline *trampoline; +struct bpf_prog *tgt_prog; +};struct
bpf_tracing_linkcan stay insyscall.c, no? don't see anyone needing it outside ofsyscall.c
Quoted 3 lines
It will be used by
invoke_bpf_prog()ofbpf_jit_comp.cin the 3rd patch to get the cookie value.
Yep, saw it when I got to it, forgot to mention it here. Ignore this suggestion.
Quoted 22 lines
Quoted 21 lines
Quoted 19 lines
include/linux/bpf.h +3
-2+struct bpf_link_primer {struct bpf_link *link;struct file *file;@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);void bpf_struct_ops_put (const void *kdata);int bpf_struct_ops_map_sys_lookup_elem (struct bpf_map *map,void *key,void *value); -int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs, - struct bpf_prog *prog, +int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, + struct bpf_tramp_link *link,const struct
btf_func_model
*model,void *image, void *image_end);[...]
PATCH v5 2/5 bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
In reply to Kui-Feng LeeBPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.
--- arch/x86/net/bpf_jit_comp.c | 55 +++++++++++++++++++++++++++++++++++++ include/linux/bpf.h | 17 +++++++++--- kernel/bpf/syscall.c | 7 +++-- kernel/bpf/trampoline.c | 20 +++++++++++--- 4 files changed, 89 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +55 -0
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, {u8 *prog = *pprog;u8 *jmp_insn; +int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; +/* mov rdi, 0 */ +emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); + +/* Prepare struct bpf_tramp_run_ctx. +* +* bpf_tramp_run_ctx is already preserved by +* arch_prepare_bpf_trampoline(). +* +* mov QWORD PTR [rsp + ctx_cookie_off], rdi +*/ +EMIT4 (0x48 ,0x89 ,0x7C ,0x24 );EMIT1 (ctx_cookie_off); +/* arg1: mov rdi, progs[i] */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p); +/* arg2: mov rsi, rsp (struct bpf_run_ctx *) */ +EMIT3 (0x48 ,0x89 ,0xE6 ); +if (emit_call (&prog, p->aux->sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter, prog))
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p);/* arg2: mov rsi, rbx <- start time in nsec */ emit_mov_reg (&prog, true,BPF_REG_2 ,BPF_REG_6 ); +/* arg3: mov rdx, rsp (struct bpf_run_ctx *) */ +EMIT3 (0x48 ,0x89 ,0xE2 );if (emit_call (&prog, p->aux->sleepable ? __bpf_prog_exit_sleepable : __bpf_prog_exit, prog))
@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } } +if (nr_args <3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)) +EMIT1 (0x52 );/* push rdx */ + +if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) { +/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); + } +if (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, flags &BPF_TRAMP_F_RET_FENTRY_RET ))
@@ -2098,6 +2126,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i }if (flags &BPF_TRAMP_F_CALL_ORIG ) { +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) +EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); + +if (nr_args <3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)) +EMIT1 (0x5A );/* pop rdx */ +restore_regs (m, &prog, nr_args, regs_off);/* call original function */
@@ -2110,6 +2147,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i im->ip_after_call = prog;memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ; + +if (nr_args <3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)) +EMIT1 (0x52 );/* push rdx */ + +/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); }if (fmod_ret->nr_links) {
@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i goto cleanup; } +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) +EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); + +if (nr_args <3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)) +EMIT1 (0x5A );/* pop rdx */ +if (flags &BPF_TRAMP_F_RESTORE_REGS )restore_regs (m, &prog, nr_args, regs_off);
include/linux/bpf.h +13 -4
@@ -681,6 +681,8 @@ struct bpf_tramp_links {int nr_links; }; +struct bpf_tramp_run_ctx ; +/* Different use cases for BPF trampoline: * 1. replace nop at the function entry (kprobe equivalent) * flags = BPF_TRAMP_F_RESTORE_REGS
@@ -707,10 +709,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *istruct bpf_tramp_links *tlinks, void *orig_call);/* these two functions are called from generated trampoline */ -u64 notrace __bpf_prog_enter(struct bpf_prog *prog); -void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); -u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog); -void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start); +u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); +void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx); +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); +void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, +struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
@@ -1304,6 +1307,12 @@ struct bpf_trace_run_ctx {u64 bpf_cookie; }; +struct bpf_tramp_run_ctx { +struct bpf_run_ctx run_ctx; +u64 bpf_cookie; +struct bpf_run_ctx *saved_run_ctx; +}; +static inline struct bpf_run_ctx *bpf_set_run_ctx (struct bpf_run_ctx *new_ctx) {struct bpf_run_ctx *old_ctx =NULL ;
kernel/bpf/syscall.c +5 -2
@@ -4775,6 +4775,7 @@ static bool syscall_prog_is_valid_access(int off, int size, BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) {struct bpf_prog *__maybe_unused prog; +struct bpf_tramp_run_ctx __maybe_unused run_ctx;switch (cmd) {case BPF_MAP_CREATE :
@@ -4802,13 +4803,15 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) return -EINVAL; } -if (!__bpf_prog_enter_sleepable (prog)) { + run_ctx.bpf_cookie =0 ; + run_ctx.saved_run_ctx =NULL ; +if (!__bpf_prog_enter_sleepable (prog, &run_ctx)) {/* recursion detected */ bpf_prog_put (prog);return -EBUSY ; } attr->test.retval =bpf_prog_run (prog, (void *) (long ) attr->test.ctx_in); -__bpf_prog_exit_sleepable (prog,0 /* bpf_prog_run does runtime stats */ ); +__bpf_prog_exit_sleepable (prog,0 /* bpf_prog_run does runtime stats */ , &run_ctx);bpf_prog_put (prog);return 0 ;#endif
kernel/bpf/trampoline.c +16 -4
@@ -568,11 +568,14 @@ static void notrace inc_misses_counter(struct bpf_prog *prog)* [2..MAX_U64] - execute bpf prog and record execution time. * This is start time. */ -u64 notrace __bpf_prog_enter(struct bpf_prog *prog) +u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)__acquires (RCU ) {rcu_read_lock ();migrate_disable (); + + run_ctx->saved_run_ctx =bpf_set_run_ctx (&run_ctx->run_ctx); +if (unlikely (__this_cpu_inc_return (*(prog->active)) !=1 )) {inc_misses_counter (prog);return 0 ;
@@ -602,29 +605,38 @@ static void notrace update_prog_stats(struct bpf_prog *prog, } } -void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) +void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)__releases (RCU ) { +bpf_reset_run_ctx (run_ctx->saved_run_ctx); +update_prog_stats (prog, start);__this_cpu_dec (*(prog->active));migrate_enable ();rcu_read_unlock (); } -u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog) +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) {rcu_read_lock_trace ();migrate_disable ();might_fault (); +if (unlikely (__this_cpu_inc_return (*(prog->active)) !=1 )) {inc_misses_counter (prog);return 0 ; } + + run_ctx->saved_run_ctx =bpf_set_run_ctx (&run_ctx->run_ctx); +return bpf_prog_start_time (); } -void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) +void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, +struct bpf_tramp_run_ctx *run_ctx) { +bpf_reset_run_ctx (run_ctx->saved_run_ctx); +update_prog_stats (prog, start);__this_cpu_dec (*(prog->active));migrate_enable ();
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v5 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
Quoted 54 lines · Kui-Feng Lee
BPF trampolines will create a
bpf_tramp_run_ctx, abpf_run_ctx, on stacks and set/reset the currentbpf_run_ctxbefore/after calling abpf_prog.Signed-off-by Kui-Feng Lee <[email protected]>--- arch/x86/net/bpf_jit_comp.c | 55 +++++++++++++++++++++++++++++++++++++ include/linux/bpf.h | 17 +++++++++--- kernel/bpf/syscall.c | 7 +++-- kernel/bpf/trampoline.c | 20 +++++++++++--- 4 files changed, 89 insertions(+), 10 deletions(-)arch/x86/net/bpf_jit_comp.c +18
-0@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, {u8 *prog = *pprog;u8 *jmp_insn; +int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; +/* mov rdi, 0 */ +emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); + +/* Prepare struct bpf_tramp_run_ctx. +* +* bpf_tramp_run_ctx is already preserved by +* arch_prepare_bpf_trampoline(). +* +* mov QWORD PTR [rsp + ctx_cookie_off], rdi +*/ +EMIT4 (0x48 ,0x89 ,0x7C ,0x24 );EMIT1 (ctx_cookie_off); +/* arg1: mov rdi, progs[i] */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p); +/* arg2: mov rsi, rsp (struct bpf_run_ctx *) */ +EMIT3 (0x48 ,0x89 ,0xE6 ); +if (emit_call (&prog, p->aux->sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter, prog))@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,emit_mov_imm64 (&prog,BPF_REG_1 , (long ) p >>32 , (u32 ) (long ) p);/* arg2: mov rsi, rbx <- start time in nsec */ emit_mov_reg (&prog, true,BPF_REG_2 ,BPF_REG_6 ); +/* arg3: mov rdx, rsp (struct bpf_run_ctx *) */ +EMIT3 (0x48 ,0x89 ,0xE2 );if (emit_call (&prog, p->aux->sleepable ? __bpf_prog_exit_sleepable : __bpf_prog_exit, prog))
arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2101
Quoted 5 lines
arch/x86/net/bpf_jit_comp.c +2
-0@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } } +if (nr_args <3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)) +EMIT1 (0x52 );/* push rdx */
this nr_args < 3 condition is new, maybe leave a comment on why we need this? Also instead of repeating this whole (fentry->nr_links || ... || ...) check, why not move if (nr_args < 3) inside the if right below?
arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2103
Quoted 2 lines
arch/x86/net/bpf_jit_comp.c +2
-0+ +if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) {
if (nr_args > 3) here?
Quoted 9 lines
arch/x86/net/bpf_jit_comp.c +6
-0+/* Prepare struct bpf_tramp_run_ctx. +* sub rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +EMIT4 (0x48 ,0x83 ,0xEC ,sizeof (struct bpf_tramp_run_ctx )); + } +if (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, flags &BPF_TRAMP_F_RET_FENTRY_RET ))
[...]
arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2185
Quoted 9 lines
if (fmod_ret->nr_links) {arch/x86/net/bpf_jit_comp.c +4
-0@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i goto cleanup; } +/* pop struct bpf_tramp_run_ctx +* add rsp, sizeof(struct bpf_tramp_run_ctx) +*/ +if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)
well, actually, can it ever be that this condition doesn't hold? That would mean we are generating empty trampoline for some reason, no? Do we do that? Checking bpf_trampoline_update() and bpf_struct_ops_prepare_trampoline() doesn't seem like we ever do this. So seems like all these checks can be dropped?
arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2136
Quoted 4 lines
arch/x86/net/bpf_jit_comp.c +4
-0+EMIT4 (0x48 ,0x83 ,0xC4 ,sizeof (struct bpf_tramp_run_ctx )); + +if (nr_args <3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)) +EMIT1 (0x5A );/* pop rdx */
same, move it inside if above?
Quoted 4 lines
arch/x86/net/bpf_jit_comp.c +1
-0+if (flags &BPF_TRAMP_F_RESTORE_REGS )restore_regs (m, &prog, nr_args, regs_off);
[...]
Re: [PATCH bpf-next v5 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
Quoted 106 lines · Andrii Nakryiko
Quoted 67 lines · Kui-Feng Lee
BPF trampolines will create a
bpf_tramp_run_ctx, abpf_run_ctx, on stacks and set/reset the currentbpf_run_ctxbefore/after calling abpf_prog.Signed-off-by Kui-Feng Lee <[email protected]>---
arch/x86/net/bpf_jit_comp.c | 55
+++++++++++++++++++++++++++++++++++++
include/linux/bpf.h | 17 +++++++++---
kernel/bpf/syscall.c | 7 +++--
kernel/bpf/trampoline.c | 20 +++++++++++---4 files changed, 89 insertions(+), 10 deletions(-)arch/x86/net/bpf_jit_comp.c +16
-0@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, { u8 *prog = *pprog; u8 *jmp_insn; + int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = l->link.prog; + /* mov rdi, 0 */ + emit_mov_imm64(&prog, BPF_REG_1, 0, 0); + + /* Prepare struct bpf_tramp_run_ctx. + * + * bpf_tramp_run_ctx is already preserved by + * arch_prepare_bpf_trampoline(). + * + * mov QWORD PTR [rsp + ctx_cookie_off], rdi + */ + EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off); + /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long ) p); + /* arg2: mov rsi, rsp (struct bpf_run_ctx *) */ + EMIT3(0x48, 0x89, 0xE6); +if (emit_call(&prog,
p->aux->sleepable?__bpf_prog_enter_sleepable:
__bpf_prog_enter, prog))arch/x86/net/bpf_jit_comp.c +2
-0@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long ) p); /* arg2: mov rsi, rbx <- start time in nsec */ emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); + /* arg3: mov rdx, rsp (struct bpf_run_ctx *) */ + EMIT3(0x48, 0x89, 0xE2); if (emit_call(&prog,
p->aux->sleepable?__bpf_prog_exit_sleepable:
__bpf_prog_exit, prog))arch/x86/net/bpf_jit_comp.c +0
-0@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i
}
}Quoted 3 lines
+ if (
nr_args< 3 && (fentry->nr_links||fexit->nr_links||fmod_ret->nr_links))
+ EMIT1(0x52); /* push rdx */this
nr_args< 3 condition is new, maybe leave a comment on why we need this? Also instead of repeating this whole (fentry->nr_links|| ... || ...) check, why not move if (nr_args< 3) inside the if right below?Quoted 3 lines
+
+ if (fentry->nr_links||fexit->nr_links|| fmod_ret-Quoted 1 line
nr_links) {if (
nr_args> 3) here?Quoted 10 lines
+ /* Prepare struct
bpf_tramp_run_ctx.
+ * sub rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+ }
+
if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))[...]
Quoted 10 lines
if (
fmod_ret->nr_links) {arch/x86/net/bpf_jit_comp.c +0
-0@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i
goto cleanup;
}+ /* pop struct
bpf_tramp_run_ctx
+ * add rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ if (fentry->nr_links||fexit->nr_links|| fmod_ret-
Quoted 8 lines
Quoted 1 line
Quoted 1 line
nr_links)well, actually, can it ever be that this condition doesn't hold? That would mean we are generating empty trampoline for some reason, no? Do we do that? Checking
bpf_trampoline_update()andbpf_struct_ops_prepare_trampoline()doesn't seem like we ever do this.
So seems like all these checks can be dropped?
You are right. I had added this check for doing the following line only for some cases, and didn't aware the check is no more useful after changing the way of doing it.
Quoted 15 lines
Quoted 6 lines
+
EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+ if (nr_args< 3 && (fentry->nr_links||fexit->nr_links||fmod_ret->nr_links))
+ EMIT1(0x5A); /* pop rdx */same, move it inside if above?
Quoted 4 lines
+
if (flags &BPF_TRAMP_F_RESTORE_REGS)restore_regs(m, &prog, nr_args, regs_off);[...]
Re: [PATCH bpf-next v5 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
In reply to Andrii NakryikoQuoted 68 lines · Andrii Nakryiko
Quoted 67 lines · Kui-Feng Lee
BPF trampolines will create a
bpf_tramp_run_ctx, abpf_run_ctx, on stacks and set/reset the currentbpf_run_ctxbefore/after calling abpf_prog.Signed-off-by Kui-Feng Lee <[email protected]>---
arch/x86/net/bpf_jit_comp.c | 55
+++++++++++++++++++++++++++++++++++++
include/linux/bpf.h | 17 +++++++++---
kernel/bpf/syscall.c | 7 +++--
kernel/bpf/trampoline.c | 20 +++++++++++---4 files changed, 89 insertions(+), 10 deletions(-)arch/x86/net/bpf_jit_comp.c +16
-0@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, { u8 *prog = *pprog; u8 *jmp_insn; + int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = l->link.prog; + /* mov rdi, 0 */ + emit_mov_imm64(&prog, BPF_REG_1, 0, 0); + + /* Prepare struct bpf_tramp_run_ctx. + * + * bpf_tramp_run_ctx is already preserved by + * arch_prepare_bpf_trampoline(). + * + * mov QWORD PTR [rsp + ctx_cookie_off], rdi + */ + EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off); + /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long ) p); + /* arg2: mov rsi, rsp (struct bpf_run_ctx *) */ + EMIT3(0x48, 0x89, 0xE6); +if (emit_call(&prog,
p->aux->sleepable?__bpf_prog_enter_sleepable:
__bpf_prog_enter, prog))arch/x86/net/bpf_jit_comp.c +2
-0@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long ) p); /* arg2: mov rsi, rbx <- start time in nsec */ emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); + /* arg3: mov rdx, rsp (struct bpf_run_ctx *) */ + EMIT3(0x48, 0x89, 0xE2); if (emit_call(&prog,
p->aux->sleepable?__bpf_prog_exit_sleepable:
__bpf_prog_exit, prog))arch/x86/net/bpf_jit_comp.c +0
-0@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i
}
}
Quoted 8 lines
Quoted 3 lines
+ if (
nr_args< 3 && (fentry->nr_links||fexit->nr_links||fmod_ret->nr_links))
+ EMIT1(0x52); /* push rdx */this
nr_args< 3 condition is new, maybe leave a comment on why we need this? Also instead of repeating this whole (fentry->nr_links|| ... || ...) check, why not move if (nr_args< 3) inside the if right below?
I thought rdx is a nonvolatile (callee-saved) register. Checking ABI again, I was wrong. I am removing this part.
Quoted 54 lines
Quoted 4 lines
+
+ if (fentry->nr_links||fexit->nr_links|| fmod_ret-Quoted 1 line
nr_links) {if (
nr_args> 3) here?Quoted 10 lines
+ /* Prepare struct
bpf_tramp_run_ctx.
+ * sub rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+ }
+
if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))[...]
Quoted 11 lines
if (
fmod_ret->nr_links) {arch/x86/net/bpf_jit_comp.c +0
-0@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void *image, void *i
goto cleanup;
}+ /* pop struct
bpf_tramp_run_ctx
+ * add rsp,sizeof(struct bpf_tramp_run_ctx)+ */
+ if (fentry->nr_links||fexit->nr_links|| fmod_ret-Quoted 1 line
nr_links)well, actually, can it ever be that this condition doesn't hold? That would mean we are generating empty trampoline for some reason, no? Do we do that? Checking
bpf_trampoline_update()andbpf_struct_ops_prepare_trampoline()doesn't seem like we ever do this.
So seems like all these checks can be dropped?Quoted 6 lines
+
EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+ if (nr_args< 3 && (fentry->nr_links||fexit->nr_links||fmod_ret->nr_links))
+ EMIT1(0x5A); /* pop rdx */same, move it inside if above?
Quoted 4 lines
+
if (flags &BPF_TRAMP_F_RESTORE_REGS)restore_regs(m, &prog, nr_args, regs_off);[...]
PATCH v5 3/5 bpf, x86: Attach a cookie to fentry/fexit/fmod_ret.
In reply to Kui-Feng LeeAdd a bpf_cookie field to struct bpf_tracing_link to attach a cookie to a link of a trace hook. A cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.
The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.
--- arch/x86/net/bpf_jit_comp.c | 11 +++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 8 ++++++++ kernel/bpf/syscall.c | 27 +++++++++++++++++++++++---- kernel/trace/bpf_trace.c | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h | 7 +++++++ 6 files changed, 65 insertions(+), 6 deletions(-)
arch/x86/net/bpf_jit_comp.c +9 -2
@@ -1764,13 +1764,20 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, bool save_ret) { + u64 cookie = 0;u8 *prog = *pprog;u8 *jmp_insn;int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; -/* mov rdi, 0 */ -emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); +if (l->link.type ==BPF_LINK_TYPE_TRACING ) { +struct bpf_tracing_link *tr_link = + container_of(l, struct bpf_tracing_link, link); + cookie = tr_link->cookie; + } + +/* mov rdi, cookie */ +emit_mov_imm64 (&prog,BPF_REG_1 , (long ) cookie >>32 , (u32 ) (long ) cookie);/* Prepare struct bpf_tramp_run_ctx. *
include/linux/bpf.h +1 -0
@@ -1062,6 +1062,7 @@ struct bpf_tracing_link {enum bpf_attach_type attach_type;struct bpf_trampoline *trampoline;struct bpf_prog *tgt_prog; +u64 cookie; };struct bpf_link_primer {
include/uapi/linux/bpf.h +8 -0
@@ -1436,6 +1436,7 @@ union bpf_attr {struct {/* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ __u64 name;__u32 prog_fd; +__u64 bpf_cookie; } raw_tracepoint;struct {/* anonymous struct for BPF_BTF_LOAD */
@@ -1490,6 +1491,13 @@ union bpf_attr {__aligned_u64 addrs;__aligned_u64 cookies; } kprobe_multi; +struct { +/* black box user-provided value passed through +* to BPF program at the execution time and +* accessible through bpf_get_attach_cookie() BPF helper +*/ +__u64 bpf_cookie; + } tracing; }; } link_create;
kernel/bpf/syscall.c +23 -4
@@ -2695,9 +2695,10 @@ static const struct bpf_link_ops bpf_tracing_link_lops = { .fill_link_info = bpf_tracing_link_fill_link_info, }; -static int bpf_tracing_prog_attach (struct bpf_prog *prog, -int tgt_prog_fd, -u32 btf_id) +static int bpf_tracing_prog_attach_cookie (struct bpf_prog *prog, +int tgt_prog_fd, +u32 btf_id, +u64 bpf_cookie) {struct bpf_link_primer link_primer;struct bpf_prog *tgt_prog =NULL ;
@@ -2762,6 +2763,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,bpf_link_init (&link->link.link,BPF_LINK_TYPE_TRACING , &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type; + link->cookie = bpf_cookie;mutex_lock (&prog->aux->dst_mutex);
@@ -2871,6 +2873,13 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,return err; } +static int bpf_tracing_prog_attach (struct bpf_prog *prog, +int tgt_prog_fd, +u32 btf_id) +{ +return bpf_tracing_prog_attach_cookie (prog, tgt_prog_fd, btf_id,0 ); +} +struct bpf_raw_tp_link {struct bpf_link link;struct bpf_raw_event_map *btp;
@@ -3023,7 +3032,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro }#endif /* CONFIG_PERF_EVENTS */ -#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd +#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.bpf_cookiestatic int bpf_raw_tracepoint_open (const union bpf_attr *attr) {
@@ -3187,6 +3196,10 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)return BPF_PROG_TYPE_SK_LOOKUP ;case BPF_XDP :return BPF_PROG_TYPE_XDP ; +case BPF_TRACE_FENTRY : +case BPF_TRACE_FEXIT : +case BPF_MODIFY_RETURN : +return BPF_PROG_TYPE_TRACING ;default :return BPF_PROG_TYPE_UNSPEC ; }
@@ -4251,6 +4264,12 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr,return bpf_tracing_prog_attach (prog, attr->link_create.target_fd, attr->link_create.target_btf_id); + else if (prog->type == BPF_PROG_TYPE_TRACING) + return bpf_tracing_prog_attach_cookie(prog, + 0, + 0, + attr->link_create.tracing.bpf_cookie); +return -EINVAL ; }
kernel/trace/bpf_trace.c +17 -0
@@ -1088,6 +1088,21 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = { .arg1_type = ARG_PTR_TO_CTX, }; +BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx) +{ +struct bpf_trace_run_ctx *run_ctx; + + run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); +return run_ctx->bpf_cookie; +} + +static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = { + .func = bpf_get_attach_cookie_tracing, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_CTX , +}; + BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags) {#ifndef CONFIG_X86
@@ -1716,6 +1731,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return bpf_prog_has_trampoline (prog) ? &bpf_get_func_ret_proto :NULL ;case BPF_FUNC_get_func_arg_cnt:return bpf_prog_has_trampoline (prog) ? &bpf_get_func_arg_cnt_proto :NULL ; +case BPF_FUNC_get_attach_cookie: +return bpf_prog_has_trampoline (prog) ? &bpf_get_attach_cookie_proto_tracing :NULL ;default : fn =raw_tp_prog_func_proto (func_id, prog);if (!fn && prog->expected_attach_type ==BPF_TRACE_ITER )
tools/include/uapi/linux/bpf.h +7 -0
@@ -1490,6 +1490,13 @@ union bpf_attr {__aligned_u64 addrs;__aligned_u64 cookies; } kprobe_multi; +struct { +/* black box user-provided value passed through +* to BPF program at the execution time and +* accessible through bpf_get_attach_cookie() BPF helper +*/ +__u64 bpf_cookie; + } tracing; }; } link_create;
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v5 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret.
Quoted 28 lines · Kui-Feng Lee
Add a
bpf_cookiefield to structbpf_tracing_linkto attach a cookie to a link of a trace hook. A cookie of abpf_tracing_linkis available by callingbpf_get_attach_cookiewhen running the BPF program of the attached link.The value of a cookie will be set at
bpf_tramp_run_ctxby the trampoline of the link.Signed-off-by Kui-Feng Lee <[email protected]>--- arch/x86/net/bpf_jit_comp.c | 11 +++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 8 ++++++++ kernel/bpf/syscall.c | 27 +++++++++++++++++++++++---- kernel/trace/bpf_trace.c | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h | 7 +++++++ 6 files changed, 65 insertions(+), 6 deletions(-)arch/x86/net/bpf_jit_comp.c +1
-0@@ -1764,13 +1764,20 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, bool save_ret) { + u64 cookie = 0;
arch/x86/net/bpf_jit_comp.c static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · line 1775
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +3
-2u8 *prog = *pprog;u8 *jmp_insn;int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx , bpf_cookie);struct bpf_prog *p = l->link.prog; -/* mov rdi, 0 */ -emit_mov_imm64 (&prog,BPF_REG_1 ,0 ,0 ); +if (l->link.type ==BPF_LINK_TYPE_TRACING ) { +struct bpf_tracing_link *tr_link = + container_of(l, struct bpf_tracing_link, link);
empty line between variable declaration block and statement
Quoted 25 lines
arch/x86/net/bpf_jit_comp.c +5
-0+ cookie = tr_link->cookie; + } + +/* mov rdi, cookie */ +emit_mov_imm64 (&prog,BPF_REG_1 , (long ) cookie >>32 , (u32 ) (long ) cookie);/* Prepare struct bpf_tramp_run_ctx. * include/linux/bpf.h +1
-0@@ -1062,6 +1062,7 @@ struct bpf_tracing_link {enum bpf_attach_type attach_type;struct bpf_trampoline *trampoline;struct bpf_prog *tgt_prog; +u64 cookie; };struct bpf_link_primer {include/uapi/linux/bpf.h +0
-0
include/uapi/linux/bpf.h union bpf_attr { · line 1439
Quoted 4 lines
include/uapi/linux/bpf.h +1
-0@@ -1436,6 +1436,7 @@ union bpf_attr {struct {/* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ __u64 name;__u32 prog_fd; +__u64 bpf_cookie;
I thought we are not adding bpf_cookie through RAW_TRACEPOINT_OPEN?
Quoted 22 lines
include/uapi/linux/bpf.h +7
-0} raw_tracepoint;struct {/* anonymous struct for BPF_BTF_LOAD */ @@ -1490,6 +1491,13 @@ union bpf_attr {__aligned_u64 addrs;__aligned_u64 cookies; } kprobe_multi; +struct { +/* black box user-provided value passed through +* to BPF program at the execution time and +* accessible through bpf_get_attach_cookie() BPF helper +*/ +__u64 bpf_cookie; + } tracing; }; } link_create;kernel/bpf/syscall.c +0
-0
kernel/bpf/syscall.c static const struct bpf_link_ops bpf_tracing_link_lops = { · line 2701
Quoted 10 lines
kernel/bpf/syscall.c +4
-3@@ -2695,9 +2695,10 @@ static const struct bpf_link_ops bpf_tracing_link_lops = { .fill_link_info = bpf_tracing_link_fill_link_info, }; -static int bpf_tracing_prog_attach (struct bpf_prog *prog, -int tgt_prog_fd, -u32 btf_id) +static int bpf_tracing_prog_attach_cookie (struct bpf_prog *prog, +int tgt_prog_fd, +u32 btf_id, +u64 bpf_cookie)
let's keep it as bpf_tracing_prog_attach and just update all the call sites
Quoted 12 lines
kernel/bpf/syscall.c +1
-0{struct bpf_link_primer link_primer;struct bpf_prog *tgt_prog =NULL ;@@ -2762,6 +2763,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,bpf_link_init (&link->link.link,BPF_LINK_TYPE_TRACING , &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type; + link->cookie = bpf_cookie;mutex_lock (&prog->aux->dst_mutex);
kernel/bpf/syscall.c static int bpf_tracing_prog_attach(struct bpf_prog *prog, · line 2882
Quoted 10 lines
kernel/bpf/syscall.c +7
-0@@ -2871,6 +2873,13 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,return err; } +static int bpf_tracing_prog_attach (struct bpf_prog *prog, +int tgt_prog_fd, +u32 btf_id) +{ +return bpf_tracing_prog_attach_cookie (prog, tgt_prog_fd, btf_id,0 ); +} +
why this wrapper, just add that 0 for all existing bpf_tracing_prog_attach() invocations
Quoted 13 lines
kernel/bpf/syscall.c +1
-1struct bpf_raw_tp_link {struct bpf_link link;struct bpf_raw_event_map *btp;@@ -3023,7 +3032,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro }#endif /* CONFIG_PERF_EVENTS */ -#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd +#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.bpf_cookiestatic int bpf_raw_tracepoint_open (const union bpf_attr *attr) {
kernel/bpf/syscall.c attach_type_to_prog_type(enum bpf_attach_type attach_type) · line 3202
Quoted 7 lines
kernel/bpf/syscall.c +4
-0@@ -3187,6 +3196,10 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)return BPF_PROG_TYPE_SK_LOOKUP ;case BPF_XDP :return BPF_PROG_TYPE_XDP ; +case BPF_TRACE_FENTRY : +case BPF_TRACE_FEXIT : +case BPF_MODIFY_RETURN : +return BPF_PROG_TYPE_TRACING ;
can you please split the part that adds ability to create fentry/fexit/etc links through LINK_CREATE into a separate commit, separate from wiring the bpf_cookie? It's two logically distinct changes.
Quoted 3 lines
kernel/bpf/syscall.c +0
-0default :return BPF_PROG_TYPE_UNSPEC ; }
[...]
Quoted 5 lines
tools/include/uapi/linux/bpf.h +0
-0
tools/include/uapi/linux/bpf.h union bpf_attr { · line 1498
Quoted 9 lines
tools/include/uapi/linux/bpf.h +6
-0@@ -1490,6 +1490,13 @@ union bpf_attr {__aligned_u64 addrs;__aligned_u64 cookies; } kprobe_multi; +struct { +/* black box user-provided value passed through +* to BPF program at the execution time and +* accessible through bpf_get_attach_cookie() BPF helper +*/ +__u64 bpf_cookie;
for kprobe_multi we used "cookies", so let's do "cookie" here? We'll have to keep perf_event.bpf_cookie, of course.
Quoted 7 lines
tools/include/uapi/linux/bpf.h +1
-1+ } tracing; }; } link_create; --2.30.2
Re: [PATCH bpf-next v5 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret.
Quoted 39 lines · Andrii Nakryiko
Quoted 38 lines · Kui-Feng Lee
Add a
bpf_cookiefield to structbpf_tracing_linkto attach a cookie
to a link of a trace hook. A cookie of abpf_tracing_linkis available by callingbpf_get_attach_cookiewhen running the BPF program of the attached link.The value of a cookie will be set at
bpf_tramp_run_ctxby the trampoline of the link.Signed-off-by Kui-Feng Lee <[email protected]>---
arch/x86/net/bpf_jit_comp.c | 11 +++++++++-- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 8 ++++++++ kernel/bpf/syscall.c | 27 +++++++++++++++++++++++---- kernel/trace/bpf_trace.c | 17 +++++++++++++++++tools/include/uapi/linux/bpf.h| 7 +++++++6 files changed, 65 insertions(+), 6 deletions(-)arch/x86/net/bpf_jit_comp.c +0
-0@@ -1764,13 +1764,20 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,struct
bpf_tramp_link*l, intstack_size,
boolsave_ret)
{
+ u64 cookie = 0;
u8 *prog = *pprog;
u8*jmp_insn;
intctx_cookie_off=offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
structbpf_prog*p =l->link.prog;
Quoted 8 lines
Quoted 6 lines
- /* mov rdi, 0 */
-emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+ if (l->link.type==BPF_LINK_TYPE_TRACING) {
+ structbpf_tracing_link*tr_link=
+container_of(l, struct bpf_tracing_link, link);empty line between variable declaration block and statement
Got it!
Quoted 21 lines
Quoted 21 lines
+ cookie =
tr_link->cookie;
+ }
+
+ /* mov rdi, cookie */
+emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);/* Prepare struct
bpf_tramp_run_ctx.
*include/linux/bpf.h +1
-0@@ -1062,6 +1062,7 @@ struct bpf_tracing_link { enum bpf_attach_type attach_type; struct bpf_trampoline *trampoline; struct bpf_prog *tgt_prog; + u64 cookie; };
Quoted 13 lines
Quoted 11 lines
include/linux/bpf.h +0
-0struct bpf_link_primer {include/uapi/linux/bpf.h +0
-0@@ -1436,6 +1436,7 @@ union bpf_attr {struct { /* anonymous struct used by
BPF_RAW_TRACEPOINT_OPENcommand */
__u64name;
__u32prog_fd;
+__u64bpf_cookie;I thought we are not adding
bpf_cookiethroughRAW_TRACEPOINT_OPEN?
Yes, this line should be removed.
Quoted 30 lines
Quoted 30 lines
}
raw_tracepoint;struct { /* anonymous struct for
BPF_BTF_LOAD*/include/uapi/linux/bpf.h +2
-0@@ -1490,6 +1491,13 @@ union bpf_attr { __aligned_u64 addrs; __aligned_u64 cookies; } kprobe_multi; + struct { + /* black box user-provided valuepassed through
+ * to BPF program at the execution time and
+ * accessible throughbpf_get_attach_cookie()BPF helper
+ */
+ __u64bpf_cookie; + } tracing;
};
}link_create;kernel/bpf/syscall.c +0
-0@@ -2695,9 +2695,10 @@ static const struct bpf_link_ops bpf_tracing_link_lops = { .fill_link_info = bpf_tracing_link_fill_link_info, };
Quoted 10 lines
kernel/bpf/syscall.c static const struct bpf_link_ops bpf_tracing_link_lops = { · line 2701
Quoted 7 lines
kernel/bpf/syscall.c +4
-3-static int bpf_tracing_prog_attach (struct bpf_prog *prog, - int tgt_prog_fd, - u32 btf_id) +static int bpf_tracing_prog_attach_cookie (struct bpf_prog *prog, + int tgt_prog_fd, + u32 btf_id, + u64 bpf_cookie)let's keep it as
bpf_tracing_prog_attachand just update all the call sites
Roger
Quoted 18 lines
Quoted 18 lines
kernel/bpf/syscall.c +0
-0{ struct bpf_link_primer link_primer;struct
bpf_prog*tgt_prog= NULL;kernel/bpf/syscall.c +1
-0@@ -2762,6 +2763,7 @@ static int bpf_tracing_prog_attach(structbpf_prog *prog, bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING, &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type; + link->cookie = bpf_cookie; mutex_lock(&prog->aux->dst_mutex);@@ -2871,6 +2873,13 @@ static int bpf_tracing_prog_attach(structbpf_prog *prog, return err; }
Quoted 12 lines
Quoted 9 lines
kernel/bpf/syscall.c +6
-0+static int bpf_tracing_prog_attach (struct bpf_prog *prog, + int tgt_prog_fd, + u32 btf_id) +{ + return bpf_tracing_prog_attach_cookie(prog, tgt_prog_fd, btf_id,0 ); +}https://docs.google.com/document/d/1FYYdi1Hw4pjulbkvI1VmG-kGqTJRCg7bh1vAF4bwjMc/edit?usp=sharing
+why this wrapper, just add that 0 for all existing
bpf_tracing_prog_attach()invocations
Got it!
Quoted 14 lines
Quoted 14 lines
struct
bpf_raw_tp_link{
structbpf_linklink;
structbpf_raw_event_map*btp;kernel/bpf/syscall.c +0
-0@@ -3023,7 +3032,7 @@ static int bpf_perf_link_attach(const union
bpf_attr*attr, structbpf_prog*pro
}
#endif /*CONFIG_PERF_EVENTS*/kernel/bpf/syscall.c +1
-1-#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd +#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.bpf_cookie
Quoted 16 lines
Quoted 11 lines
static int
bpf_raw_tracepoint_open(const union bpf_attr *attr){kernel/bpf/syscall.c +0
-0@@ -3187,6 +3196,10 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) return BPF_PROG_TYPE_SK_LOOKUP;case
BPF_XDP:
returnBPF_PROG_TYPE_XDP;
+ caseBPF_TRACE_FENTRY:
+ caseBPF_TRACE_FEXIT:
+ caseBPF_MODIFY_RETURN:
+ returnBPF_PROG_TYPE_TRACING;can you please split the part that adds ability to create fentry/fexit/etc links through
LINK_CREATEinto a separate commit, separate from wiring thebpf_cookie? It's two logically distinct changes.
No problem.
Quoted 20 lines
Quoted 4 lines
default:
returnBPF_PROG_TYPE_UNSPEC;
}[...]
Quoted 13 lines
tools/include/uapi/linux/bpf.h +2
-0@@ -1490,6 +1490,13 @@ union bpf_attr { __aligned_u64 addrs; __aligned_u64 cookies; } kprobe_multi; + struct { + /* black box user-provided valuepassed through
+ * to BPF program at the execution
Quoted 8 lines
Quoted 5 lines
time and
+ * accessible throughbpf_get_attach_cookie()BPF helper
+ */
+ __u64bpf_cookie;for
kprobe_multiwe used "cookies", so let's do "cookie" here? We'll have to keep perf_event.bpf_cookie, of course.
Got it!
Quoted 8 lines
Quoted 8 lines
+ } tracing;
};
}link_create;--
2.30.2
PATCH v5 4/5 lib/bpf: Assign cookies to links in libbpf.
In reply to Kui-Feng LeeAdd a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.
--- tools/lib/bpf/bpf.c | 7 +++++++ tools/lib/bpf/bpf.h | 3 +++ tools/lib/bpf/libbpf.c | 33 +++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 12 ++++++++++++ tools/lib/bpf/libbpf.map | 1 + 5 files changed, 56 insertions(+)
tools/lib/bpf/bpf.c +7 -0
@@ -863,6 +863,13 @@ int bpf_link_create(int prog_fd, int target_fd,if (!OPTS_ZEROED (opts, kprobe_multi))return libbpf_err (-EINVAL );break ; +case BPF_TRACE_FENTRY : +case BPF_TRACE_FEXIT : +case BPF_MODIFY_RETURN : + attr.link_create.tracing.bpf_cookie =OPTS_GET (opts, tracing.bpf_cookie,0 ); +if (!OPTS_ZEROED (opts, tracing)) +return libbpf_err (-EINVAL ); +break ;default :if (!OPTS_ZEROED (opts, flags))return libbpf_err (-EINVAL );
tools/lib/bpf/bpf.h +3 -0
@@ -410,6 +410,9 @@ struct bpf_link_create_opts {__u32 iter_info_len;__u32 target_btf_id;union { +struct { +__u64 bpf_cookie; + } tracing;struct {__u64 bpf_cookie; } perf_event;
tools/lib/bpf/libbpf.c +33 -0
@@ -11262,6 +11262,39 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)return bpf_program__attach_btf_id (prog); } +struct bpf_link *bpf_program__attach_trace_opts (const struct bpf_program *prog, +const struct bpf_trace_opts *opts) +{ +char errmsg[STRERR_BUFSIZE ]; +struct bpf_link *link; +int prog_fd, pfd; + +LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + prog_fd =bpf_program__fd (prog); +if (prog_fd <0 ) { +pr_warn ("prog '%s': can't attach before loaded\n" , prog->name); +return libbpf_err_ptr (-EINVAL ); + } + + link =calloc (1 ,sizeof (*link)); +if (!link) +return libbpf_err_ptr (-ENOMEM ); + link->detach = &bpf_link__detach_fd; + + link_opts.tracing.bpf_cookie =OPTS_GET (opts, bpf_cookie,0 ); + pfd =bpf_link_create (prog_fd,0 , prog->expected_attach_type, &link_opts); +if (pfd <0 ) { + pfd = -errno; +free (link); +pr_warn ("prog '%s': failed to attach: %s\n" , + prog->name,libbpf_strerror_r (pfd, errmsg,sizeof (errmsg))); +return libbpf_err_ptr (pfd); + } + link->fd = pfd; +return (struct bpf_link *)link; +} +struct bpf_link *bpf_program__attach_lsm (const struct bpf_program *prog) {return bpf_program__attach_btf_id (prog);
tools/lib/bpf/libbpf.h +12 -0
@@ -563,8 +563,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, LIBBPF_API struct bpf_link *bpf_program__attach_raw_tracepoint (const struct bpf_program *prog,const char *tp_name); + +struct bpf_trace_opts { +/* size of this struct, for forward/backward compatibility */ +size_t sz; +/* custom user-provided value fetchable through bpf_get_attach_cookie() */ +__u64 bpf_cookie; +}; +#define bpf_trace_opts__last_field bpf_cookie + LIBBPF_API struct bpf_link *bpf_program__attach_trace (const struct bpf_program *prog); +LIBBPF_API struct bpf_link * +bpf_program__attach_trace_opts (const struct bpf_program *prog,const struct bpf_trace_opts *opts); + LIBBPF_API struct bpf_link *bpf_program__attach_lsm (const struct bpf_program *prog); LIBBPF_API struct bpf_link *
tools/lib/bpf/libbpf.map +1 -0
@@ -424,6 +424,7 @@ LIBBPF_0.6.0 { LIBBPF_0.7.0 { global: bpf_btf_load; + bpf_program__attach_trace_opts; bpf_program__expected_attach_type; bpf_program__log_buf; bpf_program__log_level;
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v5 4/5] lib/bpf: Assign cookies to links in libbpf.
Quoted 6 lines · Kui-Feng Lee
Add a cookie field to the attributes of
bpf_link_create(). Addbpf_program__attach_trace_opts()to attach a cookie to a link.Signed-off-by Kui-Feng Lee <[email protected]>---
Please stick to "libbpf: " prefix for patch subject, we don't use "lib/bpf: "
Quoted 44 lines
tools/lib/bpf/bpf.c | 7 +++++++ tools/lib/bpf/bpf.h | 3 +++ tools/lib/bpf/libbpf.c | 33 +++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 12 ++++++++++++ tools/lib/bpf/libbpf.map | 1 + 5 files changed, 56 insertions(+)tools/lib/bpf/bpf.c +7
-0@@ -863,6 +863,13 @@ int bpf_link_create(int prog_fd, int target_fd,if (!OPTS_ZEROED (opts, kprobe_multi))return libbpf_err (-EINVAL );break ; +case BPF_TRACE_FENTRY : +case BPF_TRACE_FEXIT : +case BPF_MODIFY_RETURN : + attr.link_create.tracing.bpf_cookie =OPTS_GET (opts, tracing.bpf_cookie,0 ); +if (!OPTS_ZEROED (opts, tracing)) +return libbpf_err (-EINVAL ); +break ;default :if (!OPTS_ZEROED (opts, flags))return libbpf_err (-EINVAL );tools/lib/bpf/bpf.h +3
-0@@ -410,6 +410,9 @@ struct bpf_link_create_opts {__u32 iter_info_len;__u32 target_btf_id;union { +struct { +__u64 bpf_cookie; + } tracing;struct {__u64 bpf_cookie; } perf_event;tools/lib/bpf/libbpf.c +0
-0
tools/lib/bpf/libbpf.c struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) · line 11271
Quoted 10 lines
tools/lib/bpf/libbpf.c +7
-0@@ -11262,6 +11262,39 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)return bpf_program__attach_btf_id (prog); } +struct bpf_link *bpf_program__attach_trace_opts (const struct bpf_program *prog, +const struct bpf_trace_opts *opts) +{ +char errmsg[STRERR_BUFSIZE ]; +struct bpf_link *link; +int prog_fd, pfd; +
unnecessary empty line, LIBBPF_OPTS is a variable declaration, so keep it together with other variables
Quoted 5 lines
tools/lib/bpf/libbpf.c +5
-0+LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + prog_fd =bpf_program__fd (prog); +if (prog_fd <0 ) { +pr_warn ("prog '%s': can't attach before loaded\n" , prog->name);
tools/lib/bpf/libbpf.c struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) · line 11286
Quoted 10 lines
tools/lib/bpf/libbpf.c +10
-0+return libbpf_err_ptr (-EINVAL ); + } + + link =calloc (1 ,sizeof (*link)); +if (!link) +return libbpf_err_ptr (-ENOMEM ); + link->detach = &bpf_link__detach_fd; + + link_opts.tracing.bpf_cookie =OPTS_GET (opts, bpf_cookie,0 ); + pfd =bpf_link_create (prog_fd,0 , prog->expected_attach_type, &link_opts);
so this won't work on old kernels that don't support creating fentry/fexit links through LINK_CREATE. Let's for now check if bpf_cookie is zero, then redirect to bpf_program__attach_trace(). Ideally we should perform feature detection of ability to use LINK_CREATE to attach fentry and have a unified single bpf_program__attach_btf_id() implementation that will handle all that internally
Quoted 44 lines
tools/lib/bpf/libbpf.c +11
-0+if (pfd <0 ) { + pfd = -errno; +free (link); +pr_warn ("prog '%s': failed to attach: %s\n" , + prog->name,libbpf_strerror_r (pfd, errmsg,sizeof (errmsg))); +return libbpf_err_ptr (pfd); + } + link->fd = pfd; +return (struct bpf_link *)link; +} +struct bpf_link *bpf_program__attach_lsm (const struct bpf_program *prog) {return bpf_program__attach_btf_id (prog);tools/lib/bpf/libbpf.h +12
-0@@ -563,8 +563,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, LIBBPF_API struct bpf_link *bpf_program__attach_raw_tracepoint (const struct bpf_program *prog,const char *tp_name); + +struct bpf_trace_opts { +/* size of this struct, for forward/backward compatibility */ +size_t sz; +/* custom user-provided value fetchable through bpf_get_attach_cookie() */ +__u64 bpf_cookie; +}; +#define bpf_trace_opts__last_field bpf_cookie + LIBBPF_API struct bpf_link *bpf_program__attach_trace (const struct bpf_program *prog); +LIBBPF_API struct bpf_link * +bpf_program__attach_trace_opts (const struct bpf_program *prog,const struct bpf_trace_opts *opts); + LIBBPF_API struct bpf_link *bpf_program__attach_lsm (const struct bpf_program *prog); LIBBPF_API struct bpf_link *tools/lib/bpf/libbpf.map +0
-0
tools/lib/bpf/libbpf.map LIBBPF_0.6.0 { · line 427
Quoted 4 lines
tools/lib/bpf/libbpf.map +1
-0@@ -424,6 +424,7 @@ LIBBPF_0.6.0 { LIBBPF_0.7.0 { global: bpf_btf_load; + bpf_program__attach_trace_opts;
this should go into LIBBPF_0.8.0
Quoted 6 lines
tools/lib/bpf/libbpf.map +0
-1bpf_program__expected_attach_type; bpf_program__log_buf; bpf_program__log_level; --2.30.2
PATCH v5 5/5 selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.
In reply to Kui-Feng LeeMake sure BPF cookies are correct for fentry/fexit/fmod_ret.
--- .../selftests/bpf/prog_tests/bpf_cookie.c | 52 +++++++++++++++++++ .../selftests/bpf/progs/test_bpf_cookie.c | 24 +++++++++ 2 files changed, 76 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +52 -0
@@ -410,6 +410,56 @@ static void pe_subtest(struct test_bpf_cookie *skel)bpf_link__destroy (link); } +static void tracing_subtest (struct test_bpf_cookie *skel) +{ +__u64 cookie; +int prog_fd; +int fentry_fd =-1 , fexit_fd =-1 , fmod_ret_fd =-1 ; + + LIBBPF_OPTS(bpf_test_run_opts, opts, .repeat = 1); +LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + skel->bss->fentry_res =0 ; + skel->bss->fexit_res =0 ; + + cookie =0x100000 ; + prog_fd =bpf_program__fd (skel->progs.fentry_test1); + link_opts.tracing.bpf_cookie = cookie; + fentry_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_FENTRY , &link_opts); + + cookie =0x200000 ; + prog_fd =bpf_program__fd (skel->progs.fexit_test1); + link_opts.tracing.bpf_cookie = cookie; + fexit_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_FEXIT , &link_opts); +if (!ASSERT_GE (fexit_fd,0 ,"fexit.open" )) + goto cleanup; + + cookie =0x300000 ; + prog_fd =bpf_program__fd (skel->progs.fmod_ret_test); + link_opts.tracing.bpf_cookie = cookie; + fmod_ret_fd =bpf_link_create (prog_fd,0 ,BPF_MODIFY_RETURN , &link_opts); +if (!ASSERT_GE (fmod_ret_fd,0 ,"fmod_ret.opoen" )) + goto cleanup; + + prog_fd =bpf_program__fd (skel->progs.fentry_test1); +bpf_prog_test_run_opts (prog_fd, &opts); + + prog_fd =bpf_program__fd (skel->progs.fmod_ret_test); +bpf_prog_test_run_opts (prog_fd, &opts); + +ASSERT_EQ (skel->bss->fentry_res,0x100000 ,"fentry_res" ); +ASSERT_EQ (skel->bss->fexit_res,0x200000 ,"fexit_res" ); +ASSERT_EQ (skel->bss->fmod_ret_res,0x300000 ,"fmod_ret_res" ); + +cleanup: +if (fentry_fd >=0 ) +close (fentry_fd); +if (fexit_fd >=0 ) +close (fexit_fd); +if (fmod_ret_fd >=0 ) +close (fmod_ret_fd); +} +void test_bpf_cookie (void ) {struct test_bpf_cookie *skel;
@@ -432,6 +482,8 @@ void test_bpf_cookie(void)tp_subtest (skel);if (test__start_subtest ("perf_event" ))pe_subtest (skel); +if (test__start_subtest ("trampoline" )) +tracing_subtest (skel);test_bpf_cookie__destroy (skel); }
tools/testing/selftests/bpf/progs/test_bpf_cookie.c +24 -0
@@ -14,6 +14,9 @@ int uprobe_res;int uretprobe_res;int tp_res;int pe_res; +int fentry_res; +int fexit_res; +int fmod_ret_res;static void update (void *ctx,int *res) {
@@ -82,4 +85,25 @@ int handle_pe(struct pt_regs *ctx)return 0 ; } +SEC("fentry/bpf_fentry_test1") +int BPF_PROG (fentry_test1 ,int a) +{ +update (ctx, &fentry_res); +return 0 ; +} + +SEC("fexit/bpf_fentry_test1") +int BPF_PROG (fexit_test1 ,int a,int ret) +{ +update (ctx, &fexit_res); +return 0 ; +} + +SEC("fmod_ret/bpf_modify_return_test") +int BPF_PROG (fmod_ret_test ,int _a,int *_b,int _ret) +{ +update (ctx, &fmod_ret_res); +return 1234 ; +} + char _license[] SEC("license") = "GPL";
Signature · 1 line
2.30.2
Re: [PATCH bpf-next v5 5/5] selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.
Quoted 14 lines · Kui-Feng Lee
Make sure BPF cookies are correct for fentry/fexit/fmod_ret.
Signed-off-by Kui-Feng Lee <[email protected]>--- .../selftests/bpf/prog_tests/bpf_cookie.c | 52 +++++++++++++++++++ .../selftests/bpf/progs/test_bpf_cookie.c | 24 +++++++++ 2 files changed, 76 insertions(+)tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +0
-0
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 418
Quoted 9 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6
-0@@ -410,6 +410,56 @@ static void pe_subtest(struct test_bpf_cookie *skel)bpf_link__destroy (link); } +static void tracing_subtest (struct test_bpf_cookie *skel) +{ +__u64 cookie; +int prog_fd; +int fentry_fd =-1 , fexit_fd =-1 , fmod_ret_fd =-1 ; +
unnecessary empty line
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 419
Quoted 1 line
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +1
-0+ LIBBPF_OPTS(bpf_test_run_opts, opts, .repeat = 1);
.repeat = 1 is not necessary, I think, .repeat = 0 is equivalent to that
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 425
Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6
-0+LIBBPF_OPTS (bpf_link_create_opts, link_opts); + + skel->bss->fentry_res =0 ; + skel->bss->fexit_res =0 ; + + cookie =0x100000 ;
nit: make this value bigger to make sure higher 32 bits of u64 are preserved properly. Maybe 0x1000000010000000 (and similarly with 2 and
3)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 429
Quoted 4 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +4
-0+ prog_fd =bpf_program__fd (skel->progs.fentry_test1); + link_opts.tracing.bpf_cookie = cookie; + fentry_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_FENTRY , &link_opts); +
ASSERT_GE?
Quoted 7 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +7
-0+ cookie =0x200000 ; + prog_fd =bpf_program__fd (skel->progs.fexit_test1); + link_opts.tracing.bpf_cookie = cookie; + fexit_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_FEXIT , &link_opts); +if (!ASSERT_GE (fexit_fd,0 ,"fexit.open" )) + goto cleanup; +
[...]
Re: [PATCH bpf-next v5 5/5] selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.
Quoted 21 lines · Andrii Nakryiko
Quoted 20 lines · Kui-Feng Lee
Make sure BPF cookies are correct for fentry/fexit/fmod_ret.
Signed-off-by Kui-Feng Lee <[email protected]>---
.../selftests/bpf/prog_tests/bpf_cookie.c | 52
+++++++++++++++++++
.../selftests/bpf/progs/test_bpf_cookie.c | 24 +++++++++2 files changed, 76 insertions(+)tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +0
-0@@ -410,6 +410,56 @@ static void pe_subtest(struct test_bpf_cookie *skel) bpf_link__destroy(link); }
Quoted 8 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 418
Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6
-0+static void tracing_subtest (struct test_bpf_cookie *skel) +{ + __u64 cookie; + int prog_fd; + int fentry_fd = -1, fexit_fd = -1, fmod_ret_fd = -1; +unnecessary empty line
Got it!
Quoted 5 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 419
Quoted 2 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +1
-0+ LIBBPF_OPTS(bpf_test_run_opts, opts, .repeat = 1);.repeat = 1 is not necessary, I think, .repeat = 0 is equivalent to that
I will test it.
Quoted 12 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 425
Quoted 7 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6
-0+ LIBBPF_OPTS(bpf_link_create_opts, link_opts); + + skel->bss->fentry_res = 0; + skel->bss->fexit_res = 0; + + cookie = 0x100000;nit: make this value bigger to make sure higher 32 bits of u64 are preserved properly. Maybe
0x1000000010000000(and similarly with 2 and
3)
Ok!
Quoted 8 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 429
Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +4
-0+ prog_fd = bpf_program__fd(skel->progs.fentry_test1); + link_opts.tracing.bpf_cookie = cookie; + fentry_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY, &link_opts); +
ASSERT_GE?
sure!
Quoted 11 lines
Quoted 9 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +7
-0+ cookie = 0x200000; + prog_fd = bpf_program__fd(skel->progs.fexit_test1); + link_opts.tracing.bpf_cookie = cookie; + fexit_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FEXIT, &link_opts); + if (!ASSERT_GE(fexit_fd, 0, "fexit.open")) + goto cleanup; +[...]