bpf: preallocate a perf_sample_data per event fd
Subsystems (7)
PATCH v2 1/1 bpf: preallocate a perf_sample_data per event fd
It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:
bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000 ? trace_call_bpf+0x82/0x100 ? sch_direct_xmit+0xe2/0x230 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_perf_func+0x19b/0x240 ? __qdisc_run+0x86/0x520 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_ftrace_handler+0x90/0xf0 ? ftrace_ops_assist_func+0x6e/0xe0 ? ip6_input_finish+0xbf/0x460 ? 0xffffffffa01e80bf ? nbd_dbg_flags_show+0xc0/0xc0 [nbd] ? blkdev_issue_zeroout+0x200/0x200 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? flush_smp_call_function_queue+0x6c/0xe0 ? smp_call_function_single_interrupt+0x32/0xc0 ? call_function_single_interrupt+0xf/0x20 ? call_function_single_interrupt+0xa/0x20 ? swiotlb_map_page+0x140/0x140 ? refcount_sub_and_test+0x1a/0x50 ? tcp_wfree+0x20/0xf0 ? skb_release_head_state+0x62/0xc0 ? skb_release_all+0xe/0x30 ? napi_consume_skb+0xb5/0x100 ? mlx5e_poll_tx_cq+0x1df/0x4e0 ? mlx5e_poll_tx_cq+0x38c/0x4e0 ? mlx5e_napi_poll+0x58/0xc30 ? mlx5e_napi_poll+0x232/0xc30 ? net_rx_action+0x128/0x340 ? __do_softirq+0xd4/0x2ad ? irq_exit+0xa5/0xb0 ? do_IRQ+0x7d/0xc0 ? common_interrupt+0xf/0xf </IRQ> ? __rb_free_aux+0xf0/0xf0 ? perf_output_sample+0x28/0x7b0 ? perf_prepare_sample+0x54/0x4a0 ? perf_event_output+0x43/0x60 ? bpf_perf_event_output_raw_tp+0x15f/0x180 ? blk_mq_start_request+0x1/0x120 ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000 ? bpf_trace_run3+0x2c/0x80 ? nbd_send_cmd+0x4c2/0x690 [nbd]
This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.
---v1->v2:
keep a pointer to the struct perf_sample_data rather than directly embedding it in the structure, avoiding the circular include and removing the need for in_use. Suggested by Song.
include/linux/bpf.h | 1 + kernel/bpf/arraymap.c | 3 ++- kernel/trace/bpf_trace.c | 29 ++++++++++++++++------------- 3 files changed, 19 insertions(+), 14 deletions(-)
include/linux/bpf.h +1 -0
@@ -472,6 +472,7 @@ struct bpf_event_entry {struct file *perf_file;struct file *map_file;struct rcu_head rcu; +struct perf_sample_data *sd; };bool bpf_prog_array_compatible (struct bpf_array *array,const struct bpf_prog *fp);
kernel/bpf/arraymap.c +2 -1
@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, {struct bpf_event_entry *ee; - ee =kzalloc (sizeof (*ee),GFP_ATOMIC ); + ee =kzalloc (sizeof (*ee) +sizeof (struct perf_sample_data ),GFP_ATOMIC );if (ee) { ee->event = perf_file->private_data; ee->perf_file = perf_file; ee->map_file = map_file; + ee->sd = (void *)ee +sizeof (*ee); }return ee;
kernel/trace/bpf_trace.c +16 -13
@@ -410,17 +410,17 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg4_type = ARG_CONST_SIZE, }; -static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd); - static __always_inline u64__bpf_perf_event_output (struct pt_regs *regs,struct bpf_map *map, -u64 flags,struct perf_sample_data *sd) +u64 flags,struct perf_raw_record *raw) { struct bpf_array *array = container_of(map, struct bpf_array, map);unsigned int cpu =smp_processor_id ();u64 index = flags &BPF_F_INDEX_MASK ;struct bpf_event_entry *ee;struct perf_event *event; +struct perf_sample_data *sd; +u64 ret;if (index ==BPF_F_CURRENT_CPU ) index = cpu;
@@ -439,13 +439,22 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,if (unlikely (event->oncpu != cpu))return -EOPNOTSUPP ; -return perf_event_output (event, sd, regs); + sd =xchg (&ee->sd,NULL ); +if (!sd) +return -EBUSY ; + +perf_sample_data_init (sd,0 ,0 ); + sd->raw = raw; + + ret =perf_event_output (event, sd, regs); + +xchg (&ee->sd, sd); +return ret; } BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, u64, flags, void *, data, u64, size) { -struct perf_sample_data *sd =this_cpu_ptr (&bpf_trace_sd);struct perf_raw_record raw = { .frag = { .size = size,
@@ -456,10 +465,8 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,if (unlikely (flags & ~(BPF_F_INDEX_MASK )))return -EINVAL ; -perf_sample_data_init (sd,0 ,0 ); - sd->raw = &raw; -return __bpf_perf_event_output (regs, map, flags, sd); +return __bpf_perf_event_output (regs, map, flags, &raw); }static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -474,12 +481,10 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { }; static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs); -static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);u64 bpf_event_output (struct bpf_map *map,u64 flags,void *meta,u64 meta_size,void *ctx,u64 ctx_size,bpf_ctx_copy_t ctx_copy) { -struct perf_sample_data *sd =this_cpu_ptr (&bpf_misc_sd);struct pt_regs *regs =this_cpu_ptr (&bpf_pt_regs);struct perf_raw_frag frag = { .copy = ctx_copy,
@@ -497,10 +502,8 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, };perf_fetch_caller_regs (regs); -perf_sample_data_init (sd,0 ,0 ); - sd->raw = &raw; -return __bpf_perf_event_output (regs, map, flags, sd); +return __bpf_perf_event_output (regs, map, flags, &raw); }BPF_CALL_0 (bpf_get_current_task)
Signature · 1 line
2.17.1
Quoted 63 lines · Matt Mullins
It is possible that a BPF program can be called while another BPF program is executing
bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000 ? trace_call_bpf+0x82/0x100 ? sch_direct_xmit+0xe2/0x230 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_perf_func+0x19b/0x240 ? __qdisc_run+0x86/0x520 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_ftrace_handler+0x90/0xf0 ? ftrace_ops_assist_func+0x6e/0xe0 ? ip6_input_finish+0xbf/0x460 ? 0xffffffffa01e80bf ? nbd_dbg_flags_show+0xc0/0xc0 [nbd] ? blkdev_issue_zeroout+0x200/0x200 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? flush_smp_call_function_queue+0x6c/0xe0 ? smp_call_function_single_interrupt+0x32/0xc0 ? call_function_single_interrupt+0xf/0x20 ? call_function_single_interrupt+0xa/0x20 ? swiotlb_map_page+0x140/0x140 ? refcount_sub_and_test+0x1a/0x50 ? tcp_wfree+0x20/0xf0 ? skb_release_head_state+0x62/0xc0 ? skb_release_all+0xe/0x30 ? napi_consume_skb+0xb5/0x100 ? mlx5e_poll_tx_cq+0x1df/0x4e0 ? mlx5e_poll_tx_cq+0x38c/0x4e0 ? mlx5e_napi_poll+0x58/0xc30 ? mlx5e_napi_poll+0x232/0xc30 ? net_rx_action+0x128/0x340 ? __do_softirq+0xd4/0x2ad ? irq_exit+0xa5/0xb0 ? do_IRQ+0x7d/0xc0 ? common_interrupt+0xf/0xf </IRQ> ? __rb_free_aux+0xf0/0xf0 ? perf_output_sample+0x28/0x7b0 ? perf_prepare_sample+0x54/0x4a0 ? perf_event_output+0x43/0x60 ? bpf_perf_event_output_raw_tp+0x15f/0x180 ? blk_mq_start_request+0x1/0x120 ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000 ? bpf_trace_run3+0x2c/0x80 ? nbd_send_cmd+0x4c2/0x690 [nbd]This also cannot be alleviated by further splitting the per-cpu
perf_sample_datastructs (as in commit283ca526a9bd("bpf: fix corruption on concurrentperf_event_outputcalls")), as araw_tpcould be attached to the block:block_rq_complete tracepoint and execute during anotherraw_tp. Instead, keep a pre-allocatedperf_sample_datastructure perperf_event_arrayelement and fail abpf_perf_event_outputif that element is concurrently being used.
Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")Signed-off-by Matt Mullins <[email protected]>
This looks great. Thanks for the fix.
Quoted 131 lines
---
v1->v2:
keep a pointer to the structperf_sample_datarather than directly embedding it in the structure, avoiding the circular include and removing the need forin_use. Suggested by Song.include/linux/bpf.h | 1 + kernel/bpf/arraymap.c | 3 ++- kernel/trace/bpf_trace.c | 29 ++++++++++++++++------------- 3 files changed, 19 insertions(+), 14 deletions(-)include/linux/bpf.h +1
-0@@ -472,6 +472,7 @@ struct bpf_event_entry {struct file *perf_file;struct file *map_file;struct rcu_head rcu; +struct perf_sample_data *sd; };bool bpf_prog_array_compatible (struct bpf_array *array,const struct bpf_prog *fp);kernel/bpf/arraymap.c +2
-1@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, {struct bpf_event_entry *ee; - ee =kzalloc (sizeof (*ee),GFP_ATOMIC ); + ee =kzalloc (sizeof (*ee) +sizeof (struct perf_sample_data ),GFP_ATOMIC );if (ee) { ee->event = perf_file->private_data; ee->perf_file = perf_file; ee->map_file = map_file; + ee->sd = (void *)ee +sizeof (*ee); }return ee;kernel/trace/bpf_trace.c +0
-2@@ -410,17 +410,17 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg4_type = ARG_CONST_SIZE, }; -static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd); -static
__always_inlineu64 __bpf_perf_event_output(structpt_regs*regs, structbpf_map*map,kernel/trace/bpf_trace.c +1
-1- u64 flags, struct perf_sample_data *sd) + u64 flags, struct perf_raw_record *raw){ struct bpf_array *array = container_of(map, struct bpf_array, map); unsigned int cpu = smp_processor_id(); u64 index = flags & BPF_F_INDEX_MASK; struct bpf_event_entry *ee; struct perf_event *event;kernel/trace/bpf_trace.c +2
-0+struct perf_sample_data *sd; +u64 ret;if (index == BPF_F_CURRENT_CPU) index = cpu;kernel/trace/bpf_trace.c +13
-10@@ -439,13 +439,22 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,if (unlikely (event->oncpu != cpu))return -EOPNOTSUPP ; -return perf_event_output (event, sd, regs); + sd =xchg (&ee->sd,NULL ); +if (!sd) +return -EBUSY ; + +perf_sample_data_init (sd,0 ,0 ); + sd->raw = raw; + + ret =perf_event_output (event, sd, regs); + +xchg (&ee->sd, sd); +return ret; } BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, u64, flags, void *, data, u64, size) { -struct perf_sample_data *sd =this_cpu_ptr (&bpf_trace_sd);struct perf_raw_record raw = { .frag = { .size = size,@@ -456,10 +465,8 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,if (unlikely (flags & ~(BPF_F_INDEX_MASK )))return -EINVAL ; -perf_sample_data_init (sd,0 ,0 ); - sd->raw = &raw; -return __bpf_perf_event_output (regs, map, flags, sd); +return __bpf_perf_event_output (regs, map, flags, &raw); }static const struct bpf_func_proto bpf_perf_event_output_proto = {@@ -474,12 +481,10 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { }; static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs); -static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);u64 bpf_event_output (struct bpf_map *map,u64 flags,void *meta,u64 meta_size,void *ctx,u64 ctx_size,bpf_ctx_copy_t ctx_copy) { -struct perf_sample_data *sd =this_cpu_ptr (&bpf_misc_sd);struct pt_regs *regs =this_cpu_ptr (&bpf_pt_regs);struct perf_raw_frag frag = { .copy = ctx_copy,@@ -497,10 +502,8 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, };perf_fetch_caller_regs (regs); -perf_sample_data_init (sd,0 ,0 ); - sd->raw = &raw; -return __bpf_perf_event_output (regs, map, flags, sd); +return __bpf_perf_event_output (regs, map, flags, &raw); }BPF_CALL_0 (bpf_get_current_task)Signature · 1 line
2.17.1
Alexei Starovoitov
AppliedQuoted 102 lines · Song Liu
Quoted 64 lines · Matt Mullins
It is possible that a BPF program can be called while another BPF program is executing
bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000 ? trace_call_bpf+0x82/0x100 ? sch_direct_xmit+0xe2/0x230 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_perf_func+0x19b/0x240 ? __qdisc_run+0x86/0x520 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_ftrace_handler+0x90/0xf0 ? ftrace_ops_assist_func+0x6e/0xe0 ? ip6_input_finish+0xbf/0x460 ? 0xffffffffa01e80bf ? nbd_dbg_flags_show+0xc0/0xc0 [nbd] ? blkdev_issue_zeroout+0x200/0x200 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? flush_smp_call_function_queue+0x6c/0xe0 ? smp_call_function_single_interrupt+0x32/0xc0 ? call_function_single_interrupt+0xf/0x20 ? call_function_single_interrupt+0xa/0x20 ? swiotlb_map_page+0x140/0x140 ? refcount_sub_and_test+0x1a/0x50 ? tcp_wfree+0x20/0xf0 ? skb_release_head_state+0x62/0xc0 ? skb_release_all+0xe/0x30 ? napi_consume_skb+0xb5/0x100 ? mlx5e_poll_tx_cq+0x1df/0x4e0 ? mlx5e_poll_tx_cq+0x38c/0x4e0 ? mlx5e_napi_poll+0x58/0xc30 ? mlx5e_napi_poll+0x232/0xc30 ? net_rx_action+0x128/0x340 ? __do_softirq+0xd4/0x2ad ? irq_exit+0xa5/0xb0 ? do_IRQ+0x7d/0xc0 ? common_interrupt+0xf/0xf </IRQ> ? __rb_free_aux+0xf0/0xf0 ? perf_output_sample+0x28/0x7b0 ? perf_prepare_sample+0x54/0x4a0 ? perf_event_output+0x43/0x60 ? bpf_perf_event_output_raw_tp+0x15f/0x180 ? blk_mq_start_request+0x1/0x120 ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000 ? bpf_trace_run3+0x2c/0x80 ? nbd_send_cmd+0x4c2/0x690 [nbd]This also cannot be alleviated by further splitting the per-cpu
perf_sample_datastructs (as in commit283ca526a9bd("bpf: fix corruption on concurrentperf_event_outputcalls")), as araw_tpcould be attached to the block:block_rq_complete tracepoint and execute during anotherraw_tp. Instead, keep a pre-allocatedperf_sample_datastructure perperf_event_arrayelement and fail abpf_perf_event_outputif that element is concurrently being used.Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")Signed-off-by Matt Mullins <[email protected]>This looks great. Thanks for the fix.
Acked-by Song Liu <[email protected]>Quoted 31 lines
---
v1->v2:
keep a pointer to the structperf_sample_datarather than directly embedding it in the structure, avoiding the circular include and removing the need forin_use. Suggested by Song.include/linux/bpf.h | 1 + kernel/bpf/arraymap.c | 3 ++- kernel/trace/bpf_trace.c | 29 ++++++++++++++++------------- 3 files changed, 19 insertions(+), 14 deletions(-)include/linux/bpf.h +1
-0@@ -472,6 +472,7 @@ struct bpf_event_entry {struct file *perf_file;struct file *map_file;struct rcu_head rcu; +struct perf_sample_data *sd; };bool bpf_prog_array_compatible (struct bpf_array *array,const struct bpf_prog *fp);kernel/bpf/arraymap.c +0
-0@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, {struct bpf_event_entry *ee;
Quoted 7 lines
kernel/bpf/arraymap.c static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, · line 662
Quoted 7 lines
kernel/bpf/arraymap.c +2
-1- ee =kzalloc (sizeof (*ee),GFP_ATOMIC ); + ee =kzalloc (sizeof (*ee) +sizeof (struct perf_sample_data ),GFP_ATOMIC );if (ee) { ee->event = perf_file->private_data; ee->perf_file = perf_file; ee->map_file = map_file; + ee->sd = (void *)ee +sizeof (*ee);
This bit looks quite weird, but I don't have better ideas to avoid circular .h pain.
Applied to bpf tree. Thanks
Daniel Borkmann
maintainerQuoted 59 lines · Matt Mullins
It is possible that a BPF program can be called while another BPF program is executing
bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000 ? trace_call_bpf+0x82/0x100 ? sch_direct_xmit+0xe2/0x230 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_perf_func+0x19b/0x240 ? __qdisc_run+0x86/0x520 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_ftrace_handler+0x90/0xf0 ? ftrace_ops_assist_func+0x6e/0xe0 ? ip6_input_finish+0xbf/0x460 ? 0xffffffffa01e80bf ? nbd_dbg_flags_show+0xc0/0xc0 [nbd] ? blkdev_issue_zeroout+0x200/0x200 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? flush_smp_call_function_queue+0x6c/0xe0 ? smp_call_function_single_interrupt+0x32/0xc0 ? call_function_single_interrupt+0xf/0x20 ? call_function_single_interrupt+0xa/0x20 ? swiotlb_map_page+0x140/0x140 ? refcount_sub_and_test+0x1a/0x50 ? tcp_wfree+0x20/0xf0 ? skb_release_head_state+0x62/0xc0 ? skb_release_all+0xe/0x30 ? napi_consume_skb+0xb5/0x100 ? mlx5e_poll_tx_cq+0x1df/0x4e0 ? mlx5e_poll_tx_cq+0x38c/0x4e0 ? mlx5e_napi_poll+0x58/0xc30 ? mlx5e_napi_poll+0x232/0xc30 ? net_rx_action+0x128/0x340 ? __do_softirq+0xd4/0x2ad ? irq_exit+0xa5/0xb0 ? do_IRQ+0x7d/0xc0 ? common_interrupt+0xf/0xf </IRQ> ? __rb_free_aux+0xf0/0xf0 ? perf_output_sample+0x28/0x7b0 ? perf_prepare_sample+0x54/0x4a0 ? perf_event_output+0x43/0x60 ? bpf_perf_event_output_raw_tp+0x15f/0x180 ? blk_mq_start_request+0x1/0x120 ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000 ? bpf_trace_run3+0x2c/0x80 ? nbd_send_cmd+0x4c2/0x690 [nbd]This also cannot be alleviated by further splitting the per-cpu
perf_sample_datastructs (as in commit283ca526a9bd("bpf: fix corruption on concurrentperf_event_outputcalls")), as araw_tpcould be attached to the block:block_rq_complete tracepoint and execute during anotherraw_tp. Instead, keep a pre-allocatedperf_sample_datastructure perperf_event_arrayelement and fail abpf_perf_event_outputif that element is concurrently being used.
Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")Signed-off-by Matt Mullins <[email protected]>
You do not elaborate why is this needed for all the networking programs that use this functionality. The bpf_misc_sd should therefore be kept as-is. There cannot be nested occurrences there (xdp, tc ingress/egress). Please explain why non-tracing should be affected here...
Daniel Borkmann
maintainerQuoted 60 lines · Daniel Borkmann
Quoted 59 lines · Matt Mullins
It is possible that a BPF program can be called while another BPF program is executing
bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000 ? trace_call_bpf+0x82/0x100 ? sch_direct_xmit+0xe2/0x230 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_perf_func+0x19b/0x240 ? __qdisc_run+0x86/0x520 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_ftrace_handler+0x90/0xf0 ? ftrace_ops_assist_func+0x6e/0xe0 ? ip6_input_finish+0xbf/0x460 ? 0xffffffffa01e80bf ? nbd_dbg_flags_show+0xc0/0xc0 [nbd] ? blkdev_issue_zeroout+0x200/0x200 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? flush_smp_call_function_queue+0x6c/0xe0 ? smp_call_function_single_interrupt+0x32/0xc0 ? call_function_single_interrupt+0xf/0x20 ? call_function_single_interrupt+0xa/0x20 ? swiotlb_map_page+0x140/0x140 ? refcount_sub_and_test+0x1a/0x50 ? tcp_wfree+0x20/0xf0 ? skb_release_head_state+0x62/0xc0 ? skb_release_all+0xe/0x30 ? napi_consume_skb+0xb5/0x100 ? mlx5e_poll_tx_cq+0x1df/0x4e0 ? mlx5e_poll_tx_cq+0x38c/0x4e0 ? mlx5e_napi_poll+0x58/0xc30 ? mlx5e_napi_poll+0x232/0xc30 ? net_rx_action+0x128/0x340 ? __do_softirq+0xd4/0x2ad ? irq_exit+0xa5/0xb0 ? do_IRQ+0x7d/0xc0 ? common_interrupt+0xf/0xf </IRQ> ? __rb_free_aux+0xf0/0xf0 ? perf_output_sample+0x28/0x7b0 ? perf_prepare_sample+0x54/0x4a0 ? perf_event_output+0x43/0x60 ? bpf_perf_event_output_raw_tp+0x15f/0x180 ? blk_mq_start_request+0x1/0x120 ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000 ? bpf_trace_run3+0x2c/0x80 ? nbd_send_cmd+0x4c2/0x690 [nbd]This also cannot be alleviated by further splitting the per-cpu
perf_sample_datastructs (as in commit283ca526a9bd("bpf: fix corruption on concurrentperf_event_outputcalls")), as araw_tpcould be attached to the block:block_rq_complete tracepoint and execute during anotherraw_tp. Instead, keep a pre-allocatedperf_sample_datastructure perperf_event_arrayelement and fail abpf_perf_event_outputif that element is concurrently being used.
Quoted 7 lines
Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")Signed-off-by Matt Mullins <[email protected]>You do not elaborate why is this needed for all the networking programs that use this functionality. The
bpf_misc_sdshould therefore be kept as-is. There cannot be nested occurrences there (xdp, tc ingress/egress). Please explain why non-tracing should be affected here...
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
Quoted 61 lines · Daniel Borkmann
Quoted 60 lines · Daniel Borkmann
Quoted 59 lines · Matt Mullins
It is possible that a BPF program can be called while another BPF program is executing
bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000 ? trace_call_bpf+0x82/0x100 ? sch_direct_xmit+0xe2/0x230 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_perf_func+0x19b/0x240 ? __qdisc_run+0x86/0x520 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? kprobe_ftrace_handler+0x90/0xf0 ? ftrace_ops_assist_func+0x6e/0xe0 ? ip6_input_finish+0xbf/0x460 ? 0xffffffffa01e80bf ? nbd_dbg_flags_show+0xc0/0xc0 [nbd] ? blkdev_issue_zeroout+0x200/0x200 ? blk_mq_end_request+0x1/0x100 ? blk_mq_end_request+0x5/0x100 ? flush_smp_call_function_queue+0x6c/0xe0 ? smp_call_function_single_interrupt+0x32/0xc0 ? call_function_single_interrupt+0xf/0x20 ? call_function_single_interrupt+0xa/0x20 ? swiotlb_map_page+0x140/0x140 ? refcount_sub_and_test+0x1a/0x50 ? tcp_wfree+0x20/0xf0 ? skb_release_head_state+0x62/0xc0 ? skb_release_all+0xe/0x30 ? napi_consume_skb+0xb5/0x100 ? mlx5e_poll_tx_cq+0x1df/0x4e0 ? mlx5e_poll_tx_cq+0x38c/0x4e0 ? mlx5e_napi_poll+0x58/0xc30 ? mlx5e_napi_poll+0x232/0xc30 ? net_rx_action+0x128/0x340 ? __do_softirq+0xd4/0x2ad ? irq_exit+0xa5/0xb0 ? do_IRQ+0x7d/0xc0 ? common_interrupt+0xf/0xf </IRQ> ? __rb_free_aux+0xf0/0xf0 ? perf_output_sample+0x28/0x7b0 ? perf_prepare_sample+0x54/0x4a0 ? perf_event_output+0x43/0x60 ? bpf_perf_event_output_raw_tp+0x15f/0x180 ? blk_mq_start_request+0x1/0x120 ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000 ? bpf_trace_run3+0x2c/0x80 ? nbd_send_cmd+0x4c2/0x690 [nbd]This also cannot be alleviated by further splitting the per-cpu
perf_sample_datastructs (as in commit283ca526a9bd("bpf: fix corruption on concurrentperf_event_outputcalls")), as araw_tpcould be attached to the block:block_rq_complete tracepoint and execute during anotherraw_tp. Instead, keep a pre-allocatedperf_sample_datastructure perperf_event_arrayelement and fail abpf_perf_event_outputif that element is concurrently being used.
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")Signed-off-by Matt Mullins <[email protected]>You do not elaborate why is this needed for all the networking programs that use this functionality. The
bpf_misc_sdshould therefore be kept as-is. There cannot be nested occurrences there (xdp, tc ingress/egress). Please explain why non-tracing should be affected here...
If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.
What resource gives you worry about doing this for the networking codepath?
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.
Quoted 7 lines · Matt Mullins
If these are invariably non-nested, I can easily keep
bpf_misc_sdwhen I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.What resource gives you worry about doing this for the networking codepath?
my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf
Quoted 10 lines
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same
perf_eventis attempted to be used recursively (i.e. the same CPU on the samePERF_EVENT_ARRAYmap, as I haven't observed anything use index !=BPF_F_CURRENT_CPUin testing).
Quoted 4 lines
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3
perf_sample_dataper cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.
I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.
Daniel Borkmann
maintainerQuoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins
If these are invariably non-nested, I can easily keep
bpf_misc_sdwhen I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.What resource gives you worry about doing this for the networking codepath?
my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf
Quoted 10 lines
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same
perf_eventis attempted to be used recursively (i.e. the same CPU on the samePERF_EVENT_ARRAYmap, as I haven't observed anything use index !=BPF_F_CURRENT_CPUin testing).Quoted 4 lines
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3
perf_sample_dataper cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.
Quoted 5 lines
I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue whenevent_outputswill be dropped for valid progs. Only when user called the helper incorrectly withoutBPF_F_CURRENT_CPU. But that's an error anyway.
My main worry with this xchg() trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.
Quoted 36 lines · Daniel Borkmann
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins
If these are invariably non-nested, I can easily keep
bpf_misc_sdwhen I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.What resource gives you worry about doing this for the networking codepath?
my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf
Quoted 10 lines
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same
perf_eventis attempted to be used recursively (i.e. the same CPU on the samePERF_EVENT_ARRAYmap, as I haven't observed anything use index !=BPF_F_CURRENT_CPUin testing).Quoted 4 lines
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3
perf_sample_dataper cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.Quoted 5 lines
I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue whenevent_outputswill be dropped for valid progs. Only when user called the helper incorrectly withoutBPF_F_CURRENT_CPU. But that's an error anyway.
Quoted 7 lines
My main worry with this
xchg()trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.
I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog calls event_output and while rb logic is working
another prog needs to start executing and use the same event array slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing BPF_F_CURRENT_CPU.
Same kind of a bug.
Daniel Borkmann
maintainerQuoted 44 lines · Alexei Starovoitov
Quoted 35 lines · Daniel Borkmann
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins
If these are invariably non-nested, I can easily keep
bpf_misc_sdwhen I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.What resource gives you worry about doing this for the networking codepath?
my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf
Quoted 10 lines
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same
perf_eventis attempted to be used recursively (i.e. the same CPU on the samePERF_EVENT_ARRAYmap, as I haven't observed anything use index !=BPF_F_CURRENT_CPUin testing).Quoted 4 lines
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3
perf_sample_dataper cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.Quoted 5 lines
I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue whenevent_outputswill be dropped for valid progs. Only when user called the helper incorrectly withoutBPF_F_CURRENT_CPU. But that's an error anyway.Quoted 7 lines
My main worry with this
xchg()trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.
Quoted 4 lines
I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog callsevent_outputand while rb logic is working
another prog needs to start executing and use the same event array
Correct.
Quoted 3 lines
slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing
BPF_F_CURRENT_CPU.
I don't see the user bug part, why should that be a user bug? It's the same as if we would say that sharing a BPF hash map between networking programs attached to different hooks or networking and tracing would be a user bug which it is not. One concrete example would be cilium monitor where we currently expose skb trace and drop events a well as debug data through the same rb. This should be usable from any type that has perf_event_output helper enabled (e.g. XDP and tc/BPF) w/o requiring to walk yet another per cpu mmap rb from user space.
Quoted 51 lines · Daniel Borkmann
Quoted 44 lines · Alexei Starovoitov
Quoted 35 lines · Daniel Borkmann
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins
If these are invariably non-nested, I can easily keep
bpf_misc_sdwhen I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.What resource gives you worry about doing this for the networking codepath?
my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf
Quoted 10 lines
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same
perf_eventis attempted to be used recursively (i.e. the same CPU on the samePERF_EVENT_ARRAYmap, as I haven't observed anything use index !=BPF_F_CURRENT_CPUin testing).Quoted 4 lines
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3
perf_sample_dataper cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.Quoted 5 lines
I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue whenevent_outputswill be dropped for valid progs. Only when user called the helper incorrectly withoutBPF_F_CURRENT_CPU. But that's an error anyway.Quoted 7 lines
My main worry with this
xchg()trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.Quoted 4 lines
I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog callsevent_outputand while rb logic is working
another prog needs to start executing and use the same event array
Quoted 7 lines
Correct.
Quoted 3 lines
slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing
BPF_F_CURRENT_CPU.I don't see the user bug part, why should that be a user bug?
because I suspect that 'struct bpf_event_entry' is not reentrable (even w/o issues with 'struct perf_sample_data').
Even if we always use 'struct perf_sample_data' on stack, I suspect the same 'struct bpf_event_entry' cannot be reused in the nested way.
Quoted 8 lines
It's the same
as if we would say that sharing a BPF hash map between networking programs attached to different hooks or networking and tracing would be a user bug which it is not. One concrete example would be cilium monitor where we currently expose skb trace and drop events a well as debug data through the same rb. This should be usable from any type that hasperf_event_outputhelper enabled (e.g. XDP and tc/BPF) w/o requiring to walk yet another per cpu mmap rb from user space.
sure. those are valid use cases, but in all cases bpf_event_output() on particular 'struct bpf_event_entry' will end before another one will begin, no?
Quoted 37 lines · Daniel Borkmann
Quoted 36 lines · Alexei Starovoitov
Quoted 35 lines · Daniel Borkmann
Quoted 33 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins
If these are invariably non-nested, I can easily keep
bpf_misc_sdwhen I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.What resource gives you worry about doing this for the networking codepath?
my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf
Quoted 10 lines
Quoted 4 lines
Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)
This change would only drop messages if the same
perf_eventis attempted to be used recursively (i.e. the same CPU on the samePERF_EVENT_ARRAYmap, as I haven't observed anything use index !=BPF_F_CURRENT_CPUin testing).Quoted 4 lines
I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3
perf_sample_dataper cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue whenevent_outputswill be dropped for valid progs. Only when user called the helper incorrectly withoutBPF_F_CURRENT_CPU. But that's an error anyway.
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
My main worry with this
xchg()trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.
Tracing can already be lossy: trace_call_bpf() silently simply doesn't call the prog and instead returns zero if bpf_prog_active != 1.
Quoted 12 lines
Quoted 5 lines
I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog callsevent_outputand while rb logic is working
another prog needs to start executing and use the same event arrayCorrect.
Quoted 3 lines
slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing
BPF_F_CURRENT_CPU.
Quoted 8 lines
I don't see the user bug part, why should that be a user bug? It's the same as if we would say that sharing a BPF hash map between networking programs attached to different hooks or networking and tracing would be a user bug which it is not. One concrete example would be cilium monitor where we currently expose skb trace and drop events a well as debug data through the same rb. This should be usable from any type that has
perf_event_outputhelper enabled (e.g. XDP and tc/BPF) w/o requiring to walk yet another per cpu mmap rb from user space.
Neither of these solutions would affect the behavior of sharing the perf array between networking programs -- since they're never called in a nested fashion, then you'll never hit the "xchg() returned NULL" at all.
That said, I think I can logically limit nesting in tracing to 3 levels:
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to run, and
- if we're really unlucky, the same in nmi
at most one of which can be a kprobe or perf event.
There is also a comment
/*
* bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
* to avoid potential recursive reuse issue when/if tracepoints are added
* inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack */
that suggests that one day bpf_perf_event_output might grow a static tracepoint. However, if the program attached to such a hypothetical tracepoint were to call bpf_perf_event_output, that would infinitely recurse ... it seems fine to let that case return -EBUSY as well. It does make me wonder if I should do the same nesting for the pt_regs.
I've now got an experiment running with the counter approach, and the workload that I hit the original crash with seems to be fine with 2 layers' worth -- though if we decide that's the one I should move forward with, I'll probably bump it to a third to be safe for an NMI.
PATCH 1/1 bpf: fix nested bpf tracepoints with per-cpu data
BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as they do not increment bpf_prog_active while executing.
This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
(at most one of which may be a kprobe or perf event).
---
This is more lines of code, but possibly less intrusive than the per-array-element approach.
I don't necessarily like that I duplicated the nest_level logic in two places, but I don't see a way to unify them:
- kprobes' bpf_perf_event_output doesn't use bpf_raw_tp_regs, and does use the perf_sample_data,
- raw tracepoints' bpf_get_stackid uses bpf_raw_tp_regs, but not the perf_sample_data, and
- raw tracepoints' bpf_perf_event_output uses both...
kernel/trace/bpf_trace.c | 95 +++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 15 deletions(-)
kernel/trace/bpf_trace.c +80 -15
@@ -410,8 +410,6 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg4_type = ARG_CONST_SIZE, }; -static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd); - static __always_inline u64__bpf_perf_event_output (struct pt_regs *regs,struct bpf_map *map,u64 flags,struct perf_sample_data *sd)
@@ -442,24 +440,47 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,return perf_event_output (event, sd, regs); } +/* +* Support executing tracepoints in normal, irq, and nmi context that each call +* bpf_perf_event_output +*/ +struct bpf_trace_sample_data { +struct perf_sample_data sds[3 ]; +}; + +static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); +static DEFINE_PER_CPU(int, bpf_trace_nest_level);BPF_CALL_5 (bpf_perf_event_output ,struct pt_regs *,regs ,struct bpf_map *,map ,u64 ,flags ,void *,data ,u64 ,size ) { -struct perf_sample_data *sd =this_cpu_ptr (&bpf_trace_sd); +struct bpf_trace_sample_data *sds =this_cpu_ptr (&bpf_trace_sds); +struct perf_sample_data *sd; +int nest_level =this_cpu_inc_return (bpf_trace_nest_level);struct perf_raw_record raw = { .frag = { .size = size, .data = data, }, }; +int err = -EBUSY ; +if (WARN_ON_ONCE (nest_level >ARRAY_SIZE (sds->sds))) + goto out; + + sd = &sds->sds[nest_level -1 ]; + + err = -EINVAL ;if (unlikely (flags & ~(BPF_F_INDEX_MASK ))) -return -EINVAL ; + goto out;perf_sample_data_init (sd,0 ,0 ); sd->raw = &raw; -return __bpf_perf_event_output (regs, map, flags, sd); + err =__bpf_perf_event_output (regs, map, flags, sd); + +out: +this_cpu_dec (bpf_trace_nest_level); +return err; }static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -822,16 +843,48 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)/* * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp * to avoid potential recursive reuse issue when/if tracepoints are added -* inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack +* inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. +* +* Since raw tracepoints run despite bpf_prog_active, support concurrent usage +* in normal, irq, and nmi context. */ -static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs); +struct bpf_raw_tp_regs { +struct pt_regs regs[3 ]; +}; +static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); +static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); +static struct pt_regs *get_bpf_raw_tp_regs (void ) +{ +struct bpf_raw_tp_regs *tp_regs =this_cpu_ptr (&bpf_raw_tp_regs); +int nest_level =this_cpu_inc_return (bpf_raw_tp_nest_level); + +if (WARN_ON_ONCE (nest_level >ARRAY_SIZE (tp_regs->regs))) { +this_cpu_dec (bpf_raw_tp_nest_level); +return ERR_PTR (-EBUSY ); + } + +return &tp_regs->regs[nest_level -1 ]; +} + +static void put_bpf_raw_tp_regs (void ) +{ +this_cpu_dec (bpf_raw_tp_nest_level); +} + BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, struct bpf_map *, map, u64, flags, void *, data, u64, size) { -struct pt_regs *regs =this_cpu_ptr (&bpf_raw_tp_regs); +struct pt_regs *regs =get_bpf_raw_tp_regs (); +int ret; + +if (IS_ERR (regs)) +return PTR_ERR (regs);perf_fetch_caller_regs (regs); -return ____bpf_perf_event_output (regs, map, flags, data, size); + ret =____bpf_perf_event_output (regs, map, flags, data, size); + +put_bpf_raw_tp_regs (); +return ret; }static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
@@ -848,12 +901,18 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, struct bpf_map *, map, u64, flags) { -struct pt_regs *regs =this_cpu_ptr (&bpf_raw_tp_regs); +struct pt_regs *regs =get_bpf_raw_tp_regs (); +int ret; + +if (IS_ERR (regs)) +return PTR_ERR (regs);perf_fetch_caller_regs (regs);/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ -return bpf_get_stackid ((unsigned long ) regs, (unsigned long ) map, - flags,0 ,0 ); + ret =bpf_get_stackid ((unsigned long ) regs, (unsigned long ) map, + flags,0 ,0 ); +put_bpf_raw_tp_regs (); +return ret; }static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
@@ -868,11 +927,17 @@ static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, void *, buf, u32, size, u64, flags) { -struct pt_regs *regs =this_cpu_ptr (&bpf_raw_tp_regs); +struct pt_regs *regs =get_bpf_raw_tp_regs (); +int ret; + +if (IS_ERR (regs)) +return PTR_ERR (regs);perf_fetch_caller_regs (regs); -return bpf_get_stack ((unsigned long ) regs, (unsigned long ) buf, - (unsigned long ) size, flags,0 ); + ret =bpf_get_stack ((unsigned long ) regs, (unsigned long ) buf, + (unsigned long ) size, flags,0 ); +put_bpf_raw_tp_regs (); +return ret; }static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
Signature · 1 line
2.17.1
Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data
Quoted 10 lines · Matt Mullins
BPF_PROG_TYPE_RAW_TRACEPOINTscan be executed nested on the same CPU, as they do not incrementbpf_prog_activewhile executing.This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
(at most one of which may be a kprobe or perf event).Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")
No comment on the code, but you're definitely missing a sign-off.
Quoted 11 lines
---
This is more lines of code, but possibly less intrusive than the per-array-element approach.I don't necessarily like that I duplicated the
nest_levellogic in two places, but I don't see a way to unify them:
- kprobes'bpf_perf_event_outputdoesn't usebpf_raw_tp_regs, and does use theperf_sample_data,
- raw tracepoints'bpf_get_stackidusesbpf_raw_tp_regs, but not theperf_sample_data, and
- raw tracepoints'bpf_perf_event_outputuses both...
Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data
Quoted 10 lines · Jakub Kicinski
Quoted 9 lines · Matt Mullins
BPF_PROG_TYPE_RAW_TRACEPOINTscan be executed nested on the same CPU, as they do not incrementbpf_prog_activewhile executing.This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
(at most one of which may be a kprobe or perf event).
Quoted 3 lines
Quoted 1 line
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")No comment on the code, but you're definitely missing a sign-off.
Oops, I totally am. I'll give it some more time for opinions to roll in, and I'll fix that before I resubmit :)
Quoted 14 lines
Quoted 12 lines
---
This is more lines of code, but possibly less intrusive than the per-array-element approach.I don't necessarily like that I duplicated the
nest_levellogic in two places, but I don't see a way to unify them:
- kprobes'bpf_perf_event_outputdoesn't usebpf_raw_tp_regs, and does use theperf_sample_data,
- raw tracepoints'bpf_get_stackidusesbpf_raw_tp_regs, but not theperf_sample_data, and
- raw tracepoints'bpf_perf_event_outputuses both...
Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data
In reply to Matt MullinsQuoted 8 lines · Matt Mullins
BPF_PROG_TYPE_RAW_TRACEPOINTscan be executed nested on the same CPU, as they do not incrementbpf_prog_activewhile executing.This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
Can NMIs be nested?
Quoted 42 lines
(at most one of which may be a kprobe or perf event).
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")---
This is more lines of code, but possibly less intrusive than the per-array-element approach.I don't necessarily like that I duplicated the
nest_levellogic in two places, but I don't see a way to unify them:
- kprobes'bpf_perf_event_outputdoesn't usebpf_raw_tp_regs, and does use theperf_sample_data,
- raw tracepoints'bpf_get_stackidusesbpf_raw_tp_regs, but not theperf_sample_data, and
- raw tracepoints'bpf_perf_event_outputuses both...kernel/trace/bpf_trace.c | 95 +++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 15 deletions(-)kernel/trace/bpf_trace.c +7
-2@@ -410,8 +410,6 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg4_type = ARG_CONST_SIZE, }; -static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd); - static __always_inline u64__bpf_perf_event_output (struct pt_regs *regs,struct bpf_map *map,u64 flags,struct perf_sample_data *sd)@@ -442,24 +440,47 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,return perf_event_output (event, sd, regs); } +/* +* Support executing tracepoints in normal, irq, and nmi context that each call +* bpf_perf_event_output +*/ +struct bpf_trace_sample_data { +struct perf_sample_data sds[3 ]; +};
kernel/trace/bpf_trace.c __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, · line 458
Quoted 10 lines
kernel/trace/bpf_trace.c +6
-1+ +static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); +static DEFINE_PER_CPU(int, bpf_trace_nest_level);BPF_CALL_5 (bpf_perf_event_output ,struct pt_regs *,regs ,struct bpf_map *,map ,u64 ,flags ,void *,data ,u64 ,size ) { -struct perf_sample_data *sd =this_cpu_ptr (&bpf_trace_sd); +struct bpf_trace_sample_data *sds =this_cpu_ptr (&bpf_trace_sds); +struct perf_sample_data *sd; +int nest_level =this_cpu_inc_return (bpf_trace_nest_level);
reverse Christmas tree?
kernel/trace/bpf_trace.c __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, · line 468
Quoted 10 lines
kernel/trace/bpf_trace.c +3
-0struct perf_raw_record raw = { .frag = { .size = size, .data = data, }, }; +int err = -EBUSY ; +if (WARN_ON_ONCE (nest_level >ARRAY_SIZE (sds->sds))) + goto out;
consider this a nit, but I find it much simpler to follow when err is set just before goto, so that it's clear what's going to be returned:
int err;
if (something_bad) {
err = -EBAD_ERR_CODE1;
goto out;
}kernel/trace/bpf_trace.c __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, · line 474
Quoted 7 lines
kernel/trace/bpf_trace.c +5
-1+ + sd = &sds->sds[nest_level -1 ]; + + err = -EINVAL ;if (unlikely (flags & ~(BPF_F_INDEX_MASK ))) -return -EINVAL ; + goto out;
Same here.
Quoted 112 lines
kernel/trace/bpf_trace.c +59
-12perf_sample_data_init (sd,0 ,0 ); sd->raw = &raw; -return __bpf_perf_event_output (regs, map, flags, sd); + err =__bpf_perf_event_output (regs, map, flags, sd); + +out: +this_cpu_dec (bpf_trace_nest_level); +return err; }static const struct bpf_func_proto bpf_perf_event_output_proto = {@@ -822,16 +843,48 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)/* * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp * to avoid potential recursive reuse issue when/if tracepoints are added -* inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack +* inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. +* +* Since raw tracepoints run despite bpf_prog_active, support concurrent usage +* in normal, irq, and nmi context. */ -static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs); +struct bpf_raw_tp_regs { +struct pt_regs regs[3 ]; +}; +static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); +static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); +static struct pt_regs *get_bpf_raw_tp_regs (void ) +{ +struct bpf_raw_tp_regs *tp_regs =this_cpu_ptr (&bpf_raw_tp_regs); +int nest_level =this_cpu_inc_return (bpf_raw_tp_nest_level); + +if (WARN_ON_ONCE (nest_level >ARRAY_SIZE (tp_regs->regs))) { +this_cpu_dec (bpf_raw_tp_nest_level); +return ERR_PTR (-EBUSY ); + } + +return &tp_regs->regs[nest_level -1 ]; +} + +static void put_bpf_raw_tp_regs (void ) +{ +this_cpu_dec (bpf_raw_tp_nest_level); +} + BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, struct bpf_map *, map, u64, flags, void *, data, u64, size) { -struct pt_regs *regs =this_cpu_ptr (&bpf_raw_tp_regs); +struct pt_regs *regs =get_bpf_raw_tp_regs (); +int ret; + +if (IS_ERR (regs)) +return PTR_ERR (regs);perf_fetch_caller_regs (regs); -return ____bpf_perf_event_output (regs, map, flags, data, size); + ret =____bpf_perf_event_output (regs, map, flags, data, size); + +put_bpf_raw_tp_regs (); +return ret; }static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {@@ -848,12 +901,18 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, struct bpf_map *, map, u64, flags) { -struct pt_regs *regs =this_cpu_ptr (&bpf_raw_tp_regs); +struct pt_regs *regs =get_bpf_raw_tp_regs (); +int ret; + +if (IS_ERR (regs)) +return PTR_ERR (regs);perf_fetch_caller_regs (regs);/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ -return bpf_get_stackid ((unsigned long ) regs, (unsigned long ) map, - flags,0 ,0 ); + ret =bpf_get_stackid ((unsigned long ) regs, (unsigned long ) map, + flags,0 ,0 ); +put_bpf_raw_tp_regs (); +return ret; }static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {@@ -868,11 +927,17 @@ static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, void *, buf, u32, size, u64, flags) { -struct pt_regs *regs =this_cpu_ptr (&bpf_raw_tp_regs); +struct pt_regs *regs =get_bpf_raw_tp_regs (); +int ret; + +if (IS_ERR (regs)) +return PTR_ERR (regs);perf_fetch_caller_regs (regs); -return bpf_get_stack ((unsigned long ) regs, (unsigned long ) buf, - (unsigned long ) size, flags,0 ); + ret =bpf_get_stack ((unsigned long ) regs, (unsigned long ) buf, + (unsigned long ) size, flags,0 ); +put_bpf_raw_tp_regs (); +return ret; }static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { --2.17.1
Steven Rostedt
maintainerRe: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data
Quoted 11 lines · On Thu, 6 Jun 2019 19:59:18 -0700 Andrii Nakryiko <[email protected]>…
Quoted 8 lines · Matt Mullins
BPF_PROG_TYPE_RAW_TRACEPOINTscan be executed nested on the same CPU, as they do not incrementbpf_prog_activewhile executing.This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi contextCan NMIs be nested?
No, otherwise several things in the kernel will break.
-- Steve