BPF raw tracepoint support for BPF cookie
Subsystems (8)
Andrii Nakryiko
maintainerv2 cover BPF raw tracepoint support for BPF cookie
Add ability to specify and retrieve BPF cookie for raw tracepoint programs. Both BTF-aware (SEC("tp_btf")) and non-BTF-aware (SEC("raw_tp")) are supported, as they are exactly the same at runtime.
This issue recently came up in production use cases, where custom tried to switch from slower classic tracepoints to raw tracepoints and ran into this limitation. Luckily, it's not that hard to support this for raw_tp programs.
v1->v2:
- fixed type definition for stubs of bpf_probe_{register,unregister};
- added __u32 :u32 and aligned raw_tp fields (Jiri);
- added Stanislav's ack.
Andrii Nakryiko (5):
bpf: flatten bpf_probe_register call chain
bpf: pass whole link instead of prog when triggering raw tracepoint
bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
libbpf: add support for BPF cookie for raw_tp/tp_btf programs selftests/bpf: add raw_tp/tp_btf BPF cookie subtests
include/linux/bpf.h | 6 + include/linux/trace_events.h | 36 +++--- include/trace/bpf_probe.h | 3 +- include/uapi/linux/bpf.h | 6 +- kernel/bpf/syscall.c | 22 ++-- kernel/trace/bpf_trace.c | 36 ++++-- tools/include/uapi/linux/bpf.h | 1 + tools/lib/bpf/bpf.c | 16 ++- tools/lib/bpf/bpf.h | 9 ++ tools/lib/bpf/libbpf.c | 20 ++- tools/lib/bpf/libbpf.h | 11 ++ tools/lib/bpf/libbpf.map | 2 + .../selftests/bpf/prog_tests/bpf_cookie.c | 114 +++++++++++++++++- .../selftests/bpf/progs/test_bpf_cookie.c | 16 +++ 14 files changed, 248 insertions(+), 50 deletions(-)
Signature · 1 line
2.43.0
Andrii Nakryiko
maintainerPATCH v2 1/5 bpf: flatten bpf_probe_register call chain
bpf_probe_register() and __bpf_probe_register() have identical signatures and bpf_probe_register() just redirect to __bpf_probe_register(). So get rid of this extra function call step to simplify following the source code.
It has no difference at runtime due to inlining, of course.
--- kernel/trace/bpf_trace.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-)
kernel/trace/bpf_trace.c +1 -6
@@ -2429,7 +2429,7 @@ BPF_TRACE_DEFN_x(10);BPF_TRACE_DEFN_x (11 );BPF_TRACE_DEFN_x (12 ); -static int __bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_prog *prog) +int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_prog *prog) {struct tracepoint *tp = btp->tp;
@@ -2447,11 +2447,6 @@ static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog * prog); } -int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_prog *prog) -{ -return __bpf_probe_register (btp, prog); -} -int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_prog *prog) {return tracepoint_probe_unregister (btp->tp, (void *)btp->bpf_func, prog);
Signature · 1 line
2.43.0
Andrii Nakryiko
maintainerPATCH v2 2/5 bpf: pass whole link instead of prog when triggering raw tracepoint
In reply to Andrii NakryikoInstead of passing prog as an argument to bpf_trace_runX() helpers, that are called from tracepoint triggering calls, store BPF link itself (struct bpf_raw_tp_link for raw tracepoints). This will allow to pass extra information like BPF cookie into raw tracepoint registration.
Instead of replacing struct bpf_prog *prog = __data; with corresponding struct bpf_raw_tp_link *link = __data; assignment in __bpf_trace_##call I just passed __data through into underlying bpf_trace_runX() call. This works well because we implicitly cast void *, and it also avoids naming clashes with arguments coming from tracepoint's "proto" list. We could have run into the same problem with "prog", we just happened to not have a tracepoint that has "prog" input argument. We are less lucky with "link", as there are tracepoints using "link" argument name already. So instead of trying to avoid naming conflicts, let's just remove intermediate local variable. It doesn't hurt readibility, it's either way a bit of a maze of calls and macros, that requires careful reading.
--- include/linux/bpf.h | 5 +++++ include/linux/trace_events.h | 36 ++++++++++++++++++++---------------- include/trace/bpf_probe.h | 3 +-- kernel/bpf/syscall.c | 9 ++------- kernel/trace/bpf_trace.c | 18 ++++++++++-------- 5 files changed, 38 insertions(+), 33 deletions(-)
include/linux/bpf.h +5 -0
@@ -1608,6 +1608,11 @@ struct bpf_tracing_link {struct bpf_prog *tgt_prog; }; +struct bpf_raw_tp_link { +struct bpf_link link; +struct bpf_raw_event_map *btp; +}; +struct bpf_link_primer {struct bpf_link *link;struct file *file;
include/linux/trace_events.h +20 -16
@@ -759,8 +759,11 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);int perf_event_attach_bpf_prog (struct perf_event *event,struct bpf_prog *prog,u64 bpf_cookie);void perf_event_detach_bpf_prog (struct perf_event *event);int perf_event_query_prog_array (struct perf_event *event,void __user *info); -int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_prog *prog); -int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_prog *prog); + +struct bpf_raw_tp_link ; +int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_raw_tp_link *link); +int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_raw_tp_link *link); +struct bpf_raw_event_map *bpf_get_raw_tracepoint (const char *name);void bpf_put_raw_tracepoint (struct bpf_raw_event_map *btp); int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
@@ -788,11 +791,12 @@ perf_event_query_prog_array(struct perf_event *event, void __user *info) {return -EOPNOTSUPP ; } -static inline int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_prog *p) +struct bpf_raw_tp_link ; +static inline int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_raw_tp_link *link) {return -EOPNOTSUPP ; } -static inline int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_prog *p) +static inline int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_raw_tp_link *link) {return -EOPNOTSUPP ; }
@@ -903,31 +907,31 @@ void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);int perf_event_set_bpf_prog (struct perf_event *event,struct bpf_prog *prog,u64 bpf_cookie);void perf_event_free_bpf_prog (struct perf_event *event); -void bpf_trace_run1 (struct bpf_prog *prog,u64 arg1); -void bpf_trace_run2 (struct bpf_prog *prog,u64 arg1,u64 arg2); -void bpf_trace_run3 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run1 (struct bpf_raw_tp_link *link,u64 arg1); +void bpf_trace_run2 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2); +void bpf_trace_run3 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3); -void bpf_trace_run4 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run4 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4); -void bpf_trace_run5 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run5 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5); -void bpf_trace_run6 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run6 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6); -void bpf_trace_run7 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run7 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6,u64 arg7); -void bpf_trace_run8 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run8 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6,u64 arg7,u64 arg8); -void bpf_trace_run9 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run9 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6,u64 arg7,u64 arg8,u64 arg9); -void bpf_trace_run10 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run10 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6,u64 arg7,u64 arg8,u64 arg9,u64 arg10); -void bpf_trace_run11 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run11 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6,u64 arg7,u64 arg8,u64 arg9,u64 arg10,u64 arg11); -void bpf_trace_run12 (struct bpf_prog *prog,u64 arg1,u64 arg2, +void bpf_trace_run12 (struct bpf_raw_tp_link *link,u64 arg1,u64 arg2,u64 arg3,u64 arg4,u64 arg5,u64 arg6,u64 arg7,u64 arg8,u64 arg9,u64 arg10,u64 arg11,u64 arg12); void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
include/trace/bpf_probe.h +1 -2
@@ -46,8 +46,7 @@ static notrace void \ __bpf_trace_##call(void *__data, proto) \ { \ - struct bpf_prog *prog = __data; \ - CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \ + CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(__data, CAST_TO_U64(args)); \ }#undef DECLARE_EVENT_CLASS
kernel/bpf/syscall.c +2 -7
@@ -3469,17 +3469,12 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,return err; } -struct bpf_raw_tp_link { -struct bpf_link link; -struct bpf_raw_event_map *btp; -}; -static void bpf_raw_tp_link_release (struct bpf_link *link) {struct bpf_raw_tp_link *raw_tp = container_of(link, struct bpf_raw_tp_link, link); -bpf_probe_unregister (raw_tp->btp, raw_tp->link.prog); +bpf_probe_unregister (raw_tp->btp, raw_tp);bpf_put_raw_tracepoint (raw_tp->btp); }
@@ -3833,7 +3828,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, goto out_put_btp; } - err =bpf_probe_register (link->btp, prog); + err =bpf_probe_register (link->btp, link);if (err) {bpf_link_cleanup (&link_primer); goto out_put_btp;
kernel/trace/bpf_trace.c +10 -8
@@ -2370,8 +2370,10 @@ void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) }static __always_inline -void __bpf_trace_run(struct bpf_prog *prog, u64 *args) +void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) { +struct bpf_prog *prog = link->link.prog; +cant_sleep ();if (unlikely (this_cpu_inc_return (*(prog->active)) !=1 )) {bpf_prog_inc_misses_counter (prog);
@@ -2408,12 +2410,12 @@ void __bpf_trace_run(struct bpf_prog *prog, u64 *args)#define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11#define BPF_TRACE_DEFN_x (x) \ - void bpf_trace_run##x(struct bpf_prog *prog, \ + void bpf_trace_run##x(struct bpf_raw_tp_link *link, \ REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ { \ u64 args[x]; \ REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ - __bpf_trace_run(prog, args); \ + __bpf_trace_run(link, args); \ } \ EXPORT_SYMBOL_GPL(bpf_trace_run##x)BPF_TRACE_DEFN_x (1 );
@@ -2429,9 +2431,10 @@ BPF_TRACE_DEFN_x(10);BPF_TRACE_DEFN_x (11 );BPF_TRACE_DEFN_x (12 ); -int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_prog *prog) +int bpf_probe_register (struct bpf_raw_event_map *btp,struct bpf_raw_tp_link *link) {struct tracepoint *tp = btp->tp; +struct bpf_prog *prog = link->link.prog;/* * check that program doesn't access arguments beyond what's
@@ -2443,13 +2446,12 @@ int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)if (prog->aux->max_tp_access > btp->writable_size)return -EINVAL ; -return tracepoint_probe_register_may_exist (tp, (void *)btp->bpf_func, - prog); +return tracepoint_probe_register_may_exist (tp, (void *)btp->bpf_func, link); } -int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_prog *prog) +int bpf_probe_unregister (struct bpf_raw_event_map *btp,struct bpf_raw_tp_link *link) { -return tracepoint_probe_unregister (btp->tp, (void *)btp->bpf_func, prog); +return tracepoint_probe_unregister (btp->tp, (void *)btp->bpf_func, link); } int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
Signature · 1 line
2.43.0
Andrii Nakryiko
maintainerPATCH v2 3/5 bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
In reply to Andrii NakryikoWire up BPF cookie for raw tracepoint programs (both BTF and non-BTF aware variants). This brings them up to part w.r.t. BPF cookie usage with classic tracepoint and fentry/fexit programs.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 6 ++++-- kernel/bpf/syscall.c | 13 +++++++++---- kernel/trace/bpf_trace.c | 13 +++++++++++++ tools/include/uapi/linux/bpf.h | 1 + 5 files changed, 28 insertions(+), 6 deletions(-)
include/linux/bpf.h +1 -0
@@ -1611,6 +1611,7 @@ struct bpf_tracing_link {struct bpf_raw_tp_link {struct bpf_link link;struct bpf_raw_event_map *btp; +u64 cookie; };struct bpf_link_primer {
include/uapi/linux/bpf.h +4 -2
@@ -1662,8 +1662,10 @@ union bpf_attr { } query;struct {/* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ -__u64 name; -__u32 prog_fd; +__u64 name; +__u32 prog_fd; + __u32 :32; +__aligned_u64 cookie; } raw_tracepoint;struct {/* anonymous struct for BPF_BTF_LOAD */
kernel/bpf/syscall.c +9 -4
@@ -3774,7 +3774,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro#endif /* CONFIG_PERF_EVENTS */ static int bpf_raw_tp_link_attach (struct bpf_prog *prog, -const char __user *user_tp_name) +const char __user *user_tp_name,u64 cookie) {struct bpf_link_primer link_primer;struct bpf_raw_tp_link *link;
@@ -3821,6 +3821,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,bpf_link_init (&link->link,BPF_LINK_TYPE_RAW_TRACEPOINT , &bpf_raw_tp_link_lops, prog); link->btp = btp; + link->cookie = cookie; err =bpf_link_prime (&link->link, &link_primer);if (err) {
@@ -3841,11 +3842,13 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,return err; } -#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd +#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookiestatic int bpf_raw_tracepoint_open (const union bpf_attr *attr) {struct bpf_prog *prog; +void __user *tp_name; +__u64 cookie;int fd;if (CHECK_ATTR (BPF_RAW_TRACEPOINT_OPEN ))
@@ -3855,7 +3858,9 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)if (IS_ERR (prog))return PTR_ERR (prog); - fd =bpf_raw_tp_link_attach (prog,u64_to_user_ptr (attr->raw_tracepoint.name)); + tp_name =u64_to_user_ptr (attr->raw_tracepoint.name); + cookie = attr->raw_tracepoint.cookie; + fd =bpf_raw_tp_link_attach (prog, tp_name, cookie);if (fd <0 )bpf_prog_put (prog);return fd;
@@ -5193,7 +5198,7 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) goto out; }if (prog->expected_attach_type ==BPF_TRACE_RAW_TP ) - ret =bpf_raw_tp_link_attach (prog,NULL ); + ret =bpf_raw_tp_link_attach (prog,NULL , attr->link_create.tracing.cookie);else if (prog->expected_attach_type ==BPF_TRACE_ITER ) ret =bpf_iter_link_attach (attr, uattr, prog);else if (prog->expected_attach_type ==BPF_LSM_CGROUP )
kernel/trace/bpf_trace.c +13 -0
@@ -2008,6 +2008,8 @@ raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_get_stackid_proto_raw_tp;case BPF_FUNC_get_stack:return &bpf_get_stack_proto_raw_tp; +case BPF_FUNC_get_attach_cookie: +return &bpf_get_attach_cookie_proto_tracing;default :return bpf_tracing_func_proto (func_id, prog); }
@@ -2070,6 +2072,9 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)case BPF_FUNC_get_func_arg_cnt:return bpf_prog_has_trampoline (prog) ? &bpf_get_func_arg_cnt_proto :NULL ;case BPF_FUNC_get_attach_cookie: +if (prog->type ==BPF_PROG_TYPE_TRACING && + prog->expected_attach_type ==BPF_TRACE_RAW_TP ) +return &bpf_get_attach_cookie_proto_tracing;return bpf_prog_has_trampoline (prog) ? &bpf_get_attach_cookie_proto_tracing :NULL ;default : fn =raw_tp_prog_func_proto (func_id, prog);
@@ -2373,15 +2378,23 @@ static __always_inlinevoid __bpf_trace_run (struct bpf_raw_tp_link *link,u64 *args) {struct bpf_prog *prog = link->link.prog; +struct bpf_run_ctx *old_run_ctx; +struct bpf_trace_run_ctx run_ctx;cant_sleep ();if (unlikely (this_cpu_inc_return (*(prog->active)) !=1 )) {bpf_prog_inc_misses_counter (prog); goto out; } + + run_ctx.bpf_cookie = link->cookie; + old_run_ctx =bpf_set_run_ctx (&run_ctx.run_ctx); +rcu_read_lock (); (void )bpf_prog_run (prog, args);rcu_read_unlock (); + +bpf_reset_run_ctx (old_run_ctx); out:this_cpu_dec (*(prog->active)); }
tools/include/uapi/linux/bpf.h +1 -0
@@ -1664,6 +1664,7 @@ union bpf_attr {struct {/* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ __u64 name;__u32 prog_fd; +__aligned_u64 cookie; } raw_tracepoint;struct {/* anonymous struct for BPF_BTF_LOAD */
Signature · 1 line
2.43.0
Eduard Zingerman
maintainerRe: [PATCH v2 bpf-next 3/5] bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
On Mon, 2024-03-18 at 11:40 -0700, Andrii Nakryiko wrote:
[...]
kernel/trace/bpf_trace.c static __always_inline · line 2382
Quoted 6 lines
kernel/trace/bpf_trace.c +2
-0@@ -2373,15 +2378,23 @@ static __always_inlinevoid __bpf_trace_run (struct bpf_raw_tp_link *link,u64 *args) {struct bpf_prog *prog = link->link.prog; +struct bpf_run_ctx *old_run_ctx; +struct bpf_trace_run_ctx run_ctx;
The struct bpf_trace_run_ctx has two fields: bpf_cookie, is_uprobe (there is also run_ctx but it's size is zero).
The is_uprobe field is not set by the code below.
Is it necessary to zero-init run_ctx variable?
Re: [PATCH v2 bpf-next 3/5] bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
Quoted 4 lines · Eduard Zingerman
On Mon, 2024-03-18 at 11:40 -0700, Andrii Nakryiko wrote:
[...]
Quoted 12 lines
kernel/trace/bpf_trace.c static __always_inline · line 2382
Quoted 6 lines
kernel/trace/bpf_trace.c +2
-0@@ -2373,15 +2378,23 @@ static __always_inlinevoid __bpf_trace_run (struct bpf_raw_tp_link *link,u64 *args) {struct bpf_prog *prog = link->link.prog; +struct bpf_run_ctx *old_run_ctx; +struct bpf_trace_run_ctx run_ctx;The struct
bpf_trace_run_ctxhas two fields:bpf_cookie,is_uprobe(there is alsorun_ctxbut it's size is zero).
Theis_uprobefield is not set by the code below.
Is it necessary to zero-initrun_ctxvariable?
I think it's ok because the is_uprobe is used by kprobes/uprobes helpers while here it's used for tp programs, so it won't be used
OTOH it might be cleaner to add special run ctx struct for tp programs, (like bpf_tp_run_ctx) to avoid confusion.. but then we'd need new helper version for bpf_get_attach_cookie.. perhaps just adding some explaining comment will be fine
jirka
Re: [PATCH v2 bpf-next 3/5] bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
Quoted 6 lines · Jiri Olsa
Quoted 4 lines · Eduard Zingerman
On Mon, 2024-03-18 at 11:40 -0700, Andrii Nakryiko wrote:
[...]
Quoted 11 lines
Quoted 11 lines
kernel/trace/bpf_trace.c static __always_inline · line 2382
Quoted 6 lines
kernel/trace/bpf_trace.c +2
-0@@ -2373,15 +2378,23 @@ static __always_inlinevoid __bpf_trace_run (struct bpf_raw_tp_link *link,u64 *args) {struct bpf_prog *prog = link->link.prog; +struct bpf_run_ctx *old_run_ctx; +struct bpf_trace_run_ctx run_ctx;The struct
bpf_trace_run_ctxhas two fields:bpf_cookie,is_uprobe(there is alsorun_ctxbut it's size is zero).
Theis_uprobefield is not set by the code below.
Is it necessary to zero-initrun_ctxvariable?
no, not really, we need to initialize fields that are going to be used, and is_uprobe *shouldn't be used*, but see below
Quoted 5 lines
Quoted 1 line
I think it's ok because the
is_uprobeis used by kprobes/uprobes helpers while here it's used for tp programs, so it won't be used
yep, that's my understanding and why I didn't add is_uprobe = false, no need to pay for that (however the cost would be trivial, so if necessary it's not a problem to add it)
Quoted 4 lines
OTOH it might be cleaner to add special run ctx struct for tp programs, (like
bpf_tp_run_ctx) to avoid confusion.. but then we'd need new helper version forbpf_get_attach_cookie.. perhaps just adding some explaining comment will be fine
so when I was implementing this, I didn't want to touch yet more code, but I felt it wrong that bpf_trace_run_ctx is shared between kprobe/uprobe and other BPF_PROG_TYPE_TRACE programs (fentry/fexit, tp_btf) and also perf_event/tracepoint programs. I'd say kprobe/uprobe should get its own, given they have these extra things like uprobe vs kprobe flag.
But I'd like to leave it to some future clean ups, and minimize the amount of changes in this patch set, as I intend to backport it to a rather old kernel we need this functionality in.
Quoted 2 lines
jirka
Re: [PATCH v2 bpf-next 3/5] bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
Quoted 44 lines · Andrii Nakryiko
Quoted 6 lines · Jiri Olsa
Quoted 4 lines · Eduard Zingerman
On Mon, 2024-03-18 at 11:40 -0700, Andrii Nakryiko wrote:
[...]
Quoted 11 lines
Quoted 11 lines
kernel/trace/bpf_trace.c static __always_inline · line 2382
Quoted 6 lines
kernel/trace/bpf_trace.c +2
-0@@ -2373,15 +2378,23 @@ static __always_inlinevoid __bpf_trace_run (struct bpf_raw_tp_link *link,u64 *args) {struct bpf_prog *prog = link->link.prog; +struct bpf_run_ctx *old_run_ctx; +struct bpf_trace_run_ctx run_ctx;The struct
bpf_trace_run_ctxhas two fields:bpf_cookie,is_uprobe(there is alsorun_ctxbut it's size is zero).
Theis_uprobefield is not set by the code below.
Is it necessary to zero-initrun_ctxvariable?no, not really, we need to initialize fields that are going to be used, and
is_uprobe*shouldn't be used*, but see belowQuoted 5 lines
Quoted 1 line
I think it's ok because the
is_uprobeis used by kprobes/uprobes helpers while here it's used for tp programs, so it won't be usedyep, that's my understanding and why I didn't add
is_uprobe= false, no need to pay for that (however the cost would be trivial, so if necessary it's not a problem to add it)Quoted 4 lines
OTOH it might be cleaner to add special run ctx struct for tp programs, (like
bpf_tp_run_ctx) to avoid confusion.. but then we'd need new helper version forbpf_get_attach_cookie.. perhaps just adding some explaining comment will be fineso when I was implementing this, I didn't want to touch yet more code, but I felt it wrong that
bpf_trace_run_ctxis shared between kprobe/uprobe and otherBPF_PROG_TYPE_TRACEprograms (fentry/fexit,tp_btf) and also perf_event/tracepoint programs. I'd say kprobe/uprobe should get its own, given they have these extra things like uprobe vs kprobe flag.
Quoted 3 lines
But I'd like to leave it to some future clean ups, and minimize the amount of changes in this patch set, as I intend to backport it to a rather old kernel we need this functionality in.
ok, sounds good
jirka
Re: [PATCH v2 bpf-next 3/5] bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs
In reply to Andrii NakryikoOn Tue, 2024-03-19 at 09:09 -0700, Andrii Nakryiko wrote:
[...]
Quoted 7 lines
Quoted 4 lines
Quoted 4 lines
The struct
bpf_trace_run_ctxhas two fields:bpf_cookie,is_uprobe(there is alsorun_ctxbut it's size is zero).
Theis_uprobefield is not set by the code below.
Is it necessary to zero-initrun_ctxvariable?no, not really, we need to initialize fields that are going to be used, and
is_uprobe*shouldn't be used*, but see below
Ok, thank you and Jiri for explaining.
Andrii Nakryiko
maintainerPATCH v2 4/5 libbpf: add support for BPF cookie for raw_tp/tp_btf programs
In reply to Andrii NakryikoWire up BPF cookie passing or raw_tp and tp_btf programs, both in low-level and high-level APIs.
--- tools/lib/bpf/bpf.c | 16 ++++++++++++++-- tools/lib/bpf/bpf.h | 9 +++++++++ tools/lib/bpf/libbpf.c | 20 +++++++++++++++++--- tools/lib/bpf/libbpf.h | 11 +++++++++++ tools/lib/bpf/libbpf.map | 2 ++ 5 files changed, 53 insertions(+), 5 deletions(-)
tools/lib/bpf/bpf.c +14 -2
@@ -785,6 +785,7 @@ int bpf_link_create(int prog_fd, int target_fd,if (!OPTS_ZEROED (opts, uprobe_multi))return libbpf_err (-EINVAL );break ; +case BPF_TRACE_RAW_TP :case BPF_TRACE_FENTRY :case BPF_TRACE_FEXIT :case BPF_MODIFY_RETURN :
@@ -1173,20 +1174,31 @@ int bpf_link_get_info_by_fd(int link_fd, struct bpf_link_info *info, __u32 *inforeturn bpf_obj_get_info_by_fd (link_fd, info, info_len); } -int bpf_raw_tracepoint_open (const char *name,int prog_fd) +int bpf_raw_tp_open (int prog_fd,struct bpf_raw_tp_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint);union bpf_attr attr;int fd; +if (!OPTS_VALID (opts, bpf_raw_tp_opts)) +return libbpf_err (-EINVAL ); +memset (&attr,0 , attr_sz); - attr.raw_tracepoint.name =ptr_to_u64 (name); attr.raw_tracepoint.prog_fd = prog_fd; + attr.raw_tracepoint.name =ptr_to_u64 (OPTS_GET (opts, tp_name,NULL )); + attr.raw_tracepoint.cookie =OPTS_GET (opts, cookie,0 ); fd =sys_bpf_fd (BPF_RAW_TRACEPOINT_OPEN , &attr, attr_sz);return libbpf_err_errno (fd); } +int bpf_raw_tracepoint_open (const char *name,int prog_fd) +{ + LIBBPF_OPTS(bpf_raw_tp_opts, opts, .tp_name = name); + +return bpf_raw_tp_open (prog_fd, &opts); +} +int bpf_btf_load (const void *btf_data,size_t btf_size,struct bpf_btf_load_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd);
tools/lib/bpf/bpf.h +9 -0
@@ -617,6 +617,15 @@ LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); +struct bpf_raw_tp_opts { +size_t sz;/* size of this struct for forward/backward compatibility */ +const char *tp_name; +__u64 cookie; + size_t :0; +}; +#define bpf_raw_tp_opts__last_field cookie + +LIBBPF_API int bpf_raw_tp_open(int prog_fd, struct bpf_raw_tp_opts *opts); LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd); LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type,
tools/lib/bpf/libbpf.c +17 -3
@@ -12280,13 +12280,19 @@ static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_linreturn libbpf_get_error (*link); } -struct bpf_link *bpf_program__attach_raw_tracepoint (const struct bpf_program *prog, -const char *tp_name) +struct bpf_link * +bpf_program__attach_raw_tracepoint_opts (const struct bpf_program *prog, +const char *tp_name, +struct bpf_raw_tracepoint_opts *opts) { +LIBBPF_OPTS (bpf_raw_tp_opts, raw_opts);char errmsg[STRERR_BUFSIZE ];struct bpf_link *link;int prog_fd, pfd; +if (!OPTS_VALID (opts, bpf_raw_tracepoint_opts)) +return libbpf_err_ptr (-EINVAL ); + prog_fd =bpf_program__fd (prog);if (prog_fd <0 ) {pr_warn ("prog '%s': can't attach before loaded\n" , prog->name);
@@ -12298,7 +12304,9 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prreturn libbpf_err_ptr (-ENOMEM ); link->detach = &bpf_link__detach_fd; - pfd =bpf_raw_tracepoint_open (tp_name, prog_fd); + raw_opts.tp_name = tp_name; + raw_opts.cookie =OPTS_GET (opts, cookie,0 ); + pfd =bpf_raw_tp_open (prog_fd, &raw_opts);if (pfd <0 ) { pfd = -errno;free (link);
@@ -12310,6 +12318,12 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prreturn link; } +struct bpf_link *bpf_program__attach_raw_tracepoint (const struct bpf_program *prog, +const char *tp_name) +{ +return bpf_program__attach_raw_tracepoint_opts (prog, tp_name,NULL ); +} +static int attach_raw_tp (const struct bpf_program *prog,long cookie,struct bpf_link **link) {static const char *const prefixes[] = {
tools/lib/bpf/libbpf.h +11 -0
@@ -760,9 +760,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,const char *tp_name, const struct bpf_tracepoint_opts *opts); +struct bpf_raw_tracepoint_opts { +size_t sz;/* size of this struct for forward/backward compatibility */ +__u64 cookie; + size_t :0; +}; +#define bpf_raw_tracepoint_opts__last_field cookie + LIBBPF_API struct bpf_link *bpf_program__attach_raw_tracepoint (const struct bpf_program *prog,const char *tp_name); +LIBBPF_API struct bpf_link * +bpf_program__attach_raw_tracepoint_opts (const struct bpf_program *prog, +const char *tp_name, +struct bpf_raw_tracepoint_opts *opts);struct bpf_trace_opts {/* size of this struct, for forward/backward compatibility */
tools/lib/bpf/libbpf.map +2 -0
@@ -410,6 +410,8 @@ LIBBPF_1.3.0 { LIBBPF_1.4.0 { global: + bpf_program__attach_raw_tracepoint_opts; + bpf_raw_tp_open; bpf_token_create; btf__new_split; btf_ext__raw_data;
Signature · 1 line
2.43.0
Re: [PATCH v2 bpf-next 4/5] libbpf: add support for BPF cookie for raw_tp/tp_btf programs
Quoted 4 lines · Andrii Nakryiko
tools/lib/bpf/bpf.c +1
-1-int bpf_raw_tracepoint_open (const char *name,int prog_fd) +int bpf_raw_tp_open (int prog_fd,struct bpf_raw_tp_opts *opts)
...
tools/lib/bpf/libbpf.map LIBBPF_1.3.0 { · line 414
Quoted 4 lines
LIBBPF_1.4.0 {
global:tools/lib/bpf/libbpf.map +2
-0+ bpf_program__attach_raw_tracepoint_opts; + bpf_raw_tp_open;
So far all api-s that accept opts have the "_opts" suffix. Why deviate from that?
Re: [PATCH v2 bpf-next 4/5] libbpf: add support for BPF cookie for raw_tp/tp_btf programs
Quoted 9 lines · Alexei Starovoitov
Quoted 4 lines · Andrii Nakryiko
tools/lib/bpf/bpf.c +1
-1-int bpf_raw_tracepoint_open (const char *name,int prog_fd) +int bpf_raw_tp_open (int prog_fd,struct bpf_raw_tp_opts *opts)...
Quoted 7 lines
tools/lib/bpf/libbpf.map LIBBPF_1.3.0 { · line 414
Quoted 4 lines
LIBBPF_1.4.0 {
global:tools/lib/bpf/libbpf.map +2
-0+ bpf_program__attach_raw_tracepoint_opts; + bpf_raw_tp_open;So far all api-s that accept opts have the "
_opts" suffix. Why deviate from that?
_opts suffix in API functions was never really a universal rule. We were adding xxx_opts() variant if we already had xxx() that didn't have opts but already existed. But for new APIs where opts were added from the very beginning we usually didn't do _opts() naming, e.g., bpf_token_create() accepts opts struct, but doesn't reflect opts in the name.
In this case, given I decided to go with shorter "bpf_raw_tp_open()" naming (as I found "bpf_raw_tracepoint_open_opts" way too verbose), I decided to not add _opts suffix.
It's a different question on whether to do bpf_raw_tp_open() vs keeping the original long name bpf_raw_tracepoint_open + adding _opts. Let me address that in Eduard's email.
Re: [PATCH v2 bpf-next 4/5] libbpf: add support for BPF cookie for raw_tp/tp_btf programs
In reply to Andrii NakryikoQuoted 6 lines · Andrii Nakryiko
Wire up BPF cookie passing or
raw_tpandtp_btfprograms, both in low-level and high-level APIs.Signed-off-by Andrii Nakryiko <[email protected]>---
But I agree with Alexei regarding 'bpf_raw_tp_open' naming, 'bpf_raw_tracepoint_open_opts' is probably better.
Re: [PATCH v2 bpf-next 4/5] libbpf: add support for BPF cookie for raw_tp/tp_btf programs
Quoted 5 lines · Eduard Zingerman
Quoted 3 lines · Andrii Nakryiko
Wire up BPF cookie passing or
raw_tpandtp_btfprograms, both in low-level and high-level APIs.
Quoted 8 lines
Quoted 3 lines
Signed-off-by Andrii Nakryiko <[email protected]>---
Acked-by Eduard Zingerman <[email protected]>But I agree with Alexei regarding '
bpf_raw_tp_open' naming, 'bpf_raw_tracepoint_open_opts' is probably better.
So I considered bpf_raw_tracepoint_open_opts(), but it felt so verbose that I decided to shorten it to bpf_raw_tp_open(), given we do have SEC("raw_tp") and that's very recognizable contraction.
Having said that, I'm not opposed to going with bpf_raw_tracepoint_open_opts(), as I don't think many users will ever need to call it directly, so verboseness doesn't matter all that much.
Let me know if you still prefer the bpf_raw_tracepoint_open_opts() variant.
Eduard Zingerman
maintainerRe: [PATCH v2 bpf-next 4/5] libbpf: add support for BPF cookie for raw_tp/tp_btf programs
On Tue, 2024-03-19 at 09:16 -0700, Andrii Nakryiko wrote:
[...]
Quoted 9 lines
So I considered
bpf_raw_tracepoint_open_opts(), but it felt so verbose that I decided to shorten it tobpf_raw_tp_open(), given we do haveSEC("raw_tp")and that's very recognizable contraction.Having said that, I'm not opposed to going with
bpf_raw_tracepoint_open_opts(), as I don't think many users will ever need to call it directly, so verboseness doesn't matter all that much.Let me know if you still prefer the
bpf_raw_tracepoint_open_opts()variant.
I'd prefer the longer variant if you don't mind.
I'm a relative beginner to libbpf internals and seeing bpf_raw_tp_open instead bpf_raw_tracepoint_open_opts kinda broke my intuitive expectations based on other APIs, so we can use it as a test :)
Re: [PATCH v2 bpf-next 4/5] libbpf: add support for BPF cookie for raw_tp/tp_btf programs
Quoted 12 lines · Eduard Zingerman
On Tue, 2024-03-19 at 09:16 -0700, Andrii Nakryiko wrote:
[...]Quoted 8 lines
So I considered
bpf_raw_tracepoint_open_opts(), but it felt so verbose that I decided to shorten it tobpf_raw_tp_open(), given we do haveSEC("raw_tp")and that's very recognizable contraction.Having said that, I'm not opposed to going with
bpf_raw_tracepoint_open_opts(), as I don't think many users will ever need to call it directly, so verboseness doesn't matter all that much.
Quoted 6 lines
Quoted 1 line
Let me know if you still prefer the
bpf_raw_tracepoint_open_opts()variant.I'd prefer the longer variant if you don't mind.
I'm a relative beginner to libbpf internals and seeingbpf_raw_tp_openinsteadbpf_raw_tracepoint_open_optskinda broke my intuitive expectations based on other APIs, so we can use it as a test :)
Ok, sounds good. You can see I didn't deviate in high-level API (bpf_program__attach_raw_tracepoint_opts()), even though this verboseness is breaking my heart, because that's way more user-visible. But I guess it's not worth it, I'll post v3 with bpf_raw_tracepoint_open_opts() then.
Andrii Nakryiko
maintainerPATCH v2 5/5 selftests/bpf: add raw_tp/tp_btf BPF cookie subtests
In reply to Andrii NakryikoAdd test validating BPF cookie can be passed during raw_tp/tp_btf attachment and can be retried at runtime with bpf_get_attach_cookie() helper.
--- .../selftests/bpf/prog_tests/bpf_cookie.c | 114 +++++++++++++++++- .../selftests/bpf/progs/test_bpf_cookie.c | 16 +++ 2 files changed, 129 insertions(+), 1 deletion(-)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +113 -1
@@ -573,6 +573,115 @@ static void lsm_subtest(struct test_bpf_cookie *skel)close (lsm_fd); } +static void tp_btf_subtest (struct test_bpf_cookie *skel) +{ +__u64 cookie; +int prog_fd, link_fd =-1 ; +struct bpf_link *link =NULL ; +LIBBPF_OPTS (bpf_link_create_opts, link_opts); +LIBBPF_OPTS (bpf_raw_tp_opts, raw_tp_opts); +LIBBPF_OPTS (bpf_trace_opts, trace_opts); + +/* There are three different ways to attach tp_btf (BTF-aware raw +* tracepoint) programs. Let's test all of them. +*/ + prog_fd =bpf_program__fd (skel->progs.handle_tp_btf); + +/* low-level BPF_RAW_TRACEPOINT_OPEN command wrapper */ + skel->bss->tp_btf_res =0 ; + + raw_tp_opts.cookie = cookie =0x11000000000000L ; + link_fd =bpf_raw_tp_open (prog_fd, &raw_tp_opts); +if (!ASSERT_GE (link_fd,0 ,"bpf_raw_tp_open" )) + goto cleanup; + +usleep (1 );/* trigger */ +close (link_fd);/* detach */ + link_fd =-1 ; + +ASSERT_EQ (skel->bss->tp_btf_res, cookie,"raw_tp_open_res" ); + +/* low-level generic bpf_link_create() API */ + skel->bss->tp_btf_res =0 ; + + link_opts.tracing.cookie = cookie =0x22000000000000L ; + link_fd =bpf_link_create (prog_fd,0 ,BPF_TRACE_RAW_TP , &link_opts); +if (!ASSERT_GE (link_fd,0 ,"bpf_link_create" )) + goto cleanup; + +usleep (1 );/* trigger */ +close (link_fd);/* detach */ + link_fd =-1 ; + +ASSERT_EQ (skel->bss->tp_btf_res, cookie,"link_create_res" ); + +/* high-level bpf_link-based bpf_program__attach_trace_opts() API */ + skel->bss->tp_btf_res =0 ; + + trace_opts.cookie = cookie =0x33000000000000L ; + link =bpf_program__attach_trace_opts (skel->progs.handle_tp_btf, &trace_opts); +if (!ASSERT_OK_PTR (link,"attach_trace_opts" )) + goto cleanup; + +usleep (1 );/* trigger */ +bpf_link__destroy (link);/* detach */ + link =NULL ; + +ASSERT_EQ (skel->bss->tp_btf_res, cookie,"attach_trace_opts_res" ); + +cleanup: +if (link_fd >=0 ) +close (link_fd); +bpf_link__destroy (link); +} + +static void raw_tp_subtest (struct test_bpf_cookie *skel) +{ +__u64 cookie; +int prog_fd, link_fd =-1 ; +struct bpf_link *link =NULL ; +LIBBPF_OPTS (bpf_raw_tp_opts, raw_tp_opts); +LIBBPF_OPTS (bpf_raw_tracepoint_opts, opts); + +/* There are two different ways to attach raw_tp programs */ + prog_fd =bpf_program__fd (skel->progs.handle_raw_tp); + +/* low-level BPF_RAW_TRACEPOINT_OPEN command wrapper */ + skel->bss->raw_tp_res =0 ; + + raw_tp_opts.tp_name ="sys_enter" ; + raw_tp_opts.cookie = cookie =0x55000000000000L ; + link_fd =bpf_raw_tp_open (prog_fd, &raw_tp_opts); +if (!ASSERT_GE (link_fd,0 ,"bpf_raw_tp_open" )) + goto cleanup; + +usleep (1 );/* trigger */ +close (link_fd);/* detach */ + link_fd =-1 ; + +ASSERT_EQ (skel->bss->raw_tp_res, cookie,"raw_tp_open_res" ); + +/* high-level bpf_link-based bpf_program__attach_raw_tracepoint_opts() API */ + skel->bss->raw_tp_res =0 ; + + opts.cookie = cookie =0x66000000000000L ; + link =bpf_program__attach_raw_tracepoint_opts (skel->progs.handle_raw_tp, +"sys_enter" , &opts); +if (!ASSERT_OK_PTR (link,"attach_raw_tp_opts" )) + goto cleanup; + +usleep (1 );/* trigger */ +bpf_link__destroy (link);/* detach */ + link =NULL ; + +ASSERT_EQ (skel->bss->raw_tp_res, cookie,"attach_raw_tp_opts_res" ); + +cleanup: +if (link_fd >=0 ) +close (link_fd); +bpf_link__destroy (link); +} +void test_bpf_cookie (void ) {struct test_bpf_cookie *skel;
@@ -601,6 +710,9 @@ void test_bpf_cookie(void)tracing_subtest (skel);if (test__start_subtest ("lsm" ))lsm_subtest (skel); - +if (test__start_subtest ("tp_btf" )) +tp_btf_subtest (skel); +if (test__start_subtest ("raw_tp" )) +raw_tp_subtest (skel);test_bpf_cookie__destroy (skel); }
tools/testing/selftests/bpf/progs/test_bpf_cookie.c +16 -0
@@ -15,6 +15,8 @@ __u64 uprobe_res;__u64 uretprobe_res;__u64 tp_res;__u64 pe_res; +__u64 raw_tp_res; +__u64 tp_btf_res;__u64 fentry_res;__u64 fexit_res;__u64 fmod_ret_res;
@@ -87,6 +89,20 @@ int handle_pe(struct pt_regs *ctx)return 0 ; } +SEC("raw_tp/sys_enter") +int handle_raw_tp (void *ctx) +{ +update (ctx, &raw_tp_res); +return 0 ; +} + +SEC("tp_btf/sys_enter") +int handle_tp_btf (void *ctx) +{ +update (ctx, &tp_btf_res); +return 0 ; +} + SEC("fentry/bpf_fentry_test1")int BPF_PROG (fentry_test1 ,int a) {
Signature · 1 line
2.43.0