Skip to contentSkip to conversation

bpf: preallocate a perf_sample_data per event fd

PATCH Applied Matt Mullins 7 participants 17 messages May 31, 2019 – Jun 7, 2019

Subsystems (7)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 bpf: preallocate a perf_sample_data per event fd

It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:

	bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000
	? trace_call_bpf+0x82/0x100
	? sch_direct_xmit+0xe2/0x230
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_perf_func+0x19b/0x240
	? __qdisc_run+0x86/0x520
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_ftrace_handler+0x90/0xf0
	? ftrace_ops_assist_func+0x6e/0xe0
	? ip6_input_finish+0xbf/0x460
	? 0xffffffffa01e80bf
	? nbd_dbg_flags_show+0xc0/0xc0 [nbd]
	? blkdev_issue_zeroout+0x200/0x200
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? flush_smp_call_function_queue+0x6c/0xe0
	? smp_call_function_single_interrupt+0x32/0xc0
	? call_function_single_interrupt+0xf/0x20
	? call_function_single_interrupt+0xa/0x20
	? swiotlb_map_page+0x140/0x140
	? refcount_sub_and_test+0x1a/0x50
	? tcp_wfree+0x20/0xf0
	? skb_release_head_state+0x62/0xc0
	? skb_release_all+0xe/0x30
	? napi_consume_skb+0xb5/0x100
	? mlx5e_poll_tx_cq+0x1df/0x4e0
	? mlx5e_poll_tx_cq+0x38c/0x4e0
	? mlx5e_napi_poll+0x58/0xc30
	? mlx5e_napi_poll+0x232/0xc30
	? net_rx_action+0x128/0x340
	? __do_softirq+0xd4/0x2ad
	? irq_exit+0xa5/0xb0
	? do_IRQ+0x7d/0xc0
	? common_interrupt+0xf/0xf
	</IRQ>
	? __rb_free_aux+0xf0/0xf0
	? perf_output_sample+0x28/0x7b0
	? perf_prepare_sample+0x54/0x4a0
	? perf_event_output+0x43/0x60
	? bpf_perf_event_output_raw_tp+0x15f/0x180
	? blk_mq_start_request+0x1/0x120
	? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000
	? bpf_trace_run3+0x2c/0x80
	? nbd_send_cmd+0x4c2/0x690 [nbd]

This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.

Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

---
v1->v2:
keep a pointer to the struct perf_sample_data rather than directly embedding it in the structure, avoiding the circular include and removing the need for in_use. Suggested by Song.

 include/linux/bpf.h      |  1 +
 kernel/bpf/arraymap.c    |  3 ++-
 kernel/trace/bpf_trace.c | 29 ++++++++++++++++-------------
 3 files changed, 19 insertions(+), 14 deletions(-)
include/linux/bpf.h +1 -0
index 4fb3aa2dc975..47fd85cfbbaf 100644
@@ -472,6 +472,7 @@ struct bpf_event_entry {
 	struct file *perf_file;
 	struct file *map_file;
 	struct rcu_head rcu;
+	struct perf_sample_data *sd;
 };
 
 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
kernel/bpf/arraymap.c +2 -1
index 584636c9e2eb..c7f5d593e04f 100644
@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
 {
 	struct bpf_event_entry *ee;
 
-	ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
+	ee = kzalloc(sizeof(*ee) + sizeof(struct perf_sample_data), GFP_ATOMIC);
 	if (ee) {
 		ee->event = perf_file->private_data;
 		ee->perf_file = perf_file;
 		ee->map_file = map_file;
+		ee->sd = (void *)ee + sizeof(*ee);
 	}
 
 	return ee;
kernel/trace/bpf_trace.c +16 -13
index f92d6ad5e080..076f8e987355 100644
@@ -410,17 +410,17 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
 	.arg4_type	= ARG_CONST_SIZE,
 };
 
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
-
 static __always_inline u64
 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
-			u64 flags, struct perf_sample_data *sd)
+			u64 flags, struct perf_raw_record *raw)
 {
 	struct bpf_array *array = container_of(map, struct bpf_array, map);
 	unsigned int cpu = smp_processor_id();
 	u64 index = flags & BPF_F_INDEX_MASK;
 	struct bpf_event_entry *ee;
 	struct perf_event *event;
+	struct perf_sample_data *sd;
+	u64 ret;
 
 	if (index == BPF_F_CURRENT_CPU)
 		index = cpu;
@@ -439,13 +439,22 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 	if (unlikely(event->oncpu != cpu))
 		return -EOPNOTSUPP;
 
-	return perf_event_output(event, sd, regs);
+	sd = xchg(&ee->sd, NULL);
+	if (!sd)
+		return -EBUSY;
+
+	perf_sample_data_init(sd, 0, 0);
+	sd->raw = raw;
+
+	ret = perf_event_output(event, sd, regs);
+
+	xchg(&ee->sd, sd);
+	return ret;
 }
 
 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
 	   u64, flags, void *, data, u64, size)
 {
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
 	struct perf_raw_record raw = {
 		.frag = {
 			.size = size,
@@ -456,10 +465,8 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
 		return -EINVAL;
 
-	perf_sample_data_init(sd, 0, 0);
-	sd->raw = &raw;
 
-	return __bpf_perf_event_output(regs, map, flags, sd);
+	return __bpf_perf_event_output(regs, map, flags, &raw);
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -474,12 +481,10 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = {
 };
 
 static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);
 
 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
 {
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_misc_sd);
 	struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);
 	struct perf_raw_frag frag = {
 		.copy		= ctx_copy,
@@ -497,10 +502,8 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
 	};
 
 	perf_fetch_caller_regs(regs);
-	perf_sample_data_init(sd, 0, 0);
-	sd->raw = &raw;
 
-	return __bpf_perf_event_output(regs, map, flags, sd);
+	return __bpf_perf_event_output(regs, map, flags, &raw);
 }
 
 BPF_CALL_0(bpf_get_current_task)
Signature · 1 line
2.17.1
Acked-by on v2 1/1
#1 lore raw
Quoted 63 lines · Matt Mullins

It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:

	bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000
	? trace_call_bpf+0x82/0x100
	? sch_direct_xmit+0xe2/0x230
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_perf_func+0x19b/0x240
	? __qdisc_run+0x86/0x520
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_ftrace_handler+0x90/0xf0
	? ftrace_ops_assist_func+0x6e/0xe0
	? ip6_input_finish+0xbf/0x460
	? 0xffffffffa01e80bf
	? nbd_dbg_flags_show+0xc0/0xc0 [nbd]
	? blkdev_issue_zeroout+0x200/0x200
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? flush_smp_call_function_queue+0x6c/0xe0
	? smp_call_function_single_interrupt+0x32/0xc0
	? call_function_single_interrupt+0xf/0x20
	? call_function_single_interrupt+0xa/0x20
	? swiotlb_map_page+0x140/0x140
	? refcount_sub_and_test+0x1a/0x50
	? tcp_wfree+0x20/0xf0
	? skb_release_head_state+0x62/0xc0
	? skb_release_all+0xe/0x30
	? napi_consume_skb+0xb5/0x100
	? mlx5e_poll_tx_cq+0x1df/0x4e0
	? mlx5e_poll_tx_cq+0x38c/0x4e0
	? mlx5e_napi_poll+0x58/0xc30
	? mlx5e_napi_poll+0x232/0xc30
	? net_rx_action+0x128/0x340
	? __do_softirq+0xd4/0x2ad
	? irq_exit+0xa5/0xb0
	? do_IRQ+0x7d/0xc0
	? common_interrupt+0xf/0xf
	</IRQ>
	? __rb_free_aux+0xf0/0xf0
	? perf_output_sample+0x28/0x7b0
	? perf_prepare_sample+0x54/0x4a0
	? perf_event_output+0x43/0x60
	? bpf_perf_event_output_raw_tp+0x15f/0x180
	? blk_mq_start_request+0x1/0x120
	? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000
	? bpf_trace_run3+0x2c/0x80
	? nbd_send_cmd+0x4c2/0x690 [nbd]

This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.

Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

This looks great. Thanks for the fix.

Quoted 131 lines

---
v1->v2:
keep a pointer to the struct perf_sample_data rather than directly embedding it in the structure, avoiding the circular include and removing the need for in_use. Suggested by Song.

include/linux/bpf.h      |  1 +
kernel/bpf/arraymap.c    |  3 ++-
kernel/trace/bpf_trace.c | 29 ++++++++++++++++-------------
3 files changed, 19 insertions(+), 14 deletions(-)
include/linux/bpf.h +1 -0
index 4fb3aa2dc975..47fd85cfbbaf 100644
@@ -472,6 +472,7 @@ struct bpf_event_entry {
 	struct file *perf_file;
 	struct file *map_file;
 	struct rcu_head rcu;
+	struct perf_sample_data *sd;
 };
 
 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
kernel/bpf/arraymap.c +2 -1
index 584636c9e2eb..c7f5d593e04f 100644
@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
 {
 	struct bpf_event_entry *ee;
 
-	ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
+	ee = kzalloc(sizeof(*ee) + sizeof(struct perf_sample_data), GFP_ATOMIC);
 	if (ee) {
 		ee->event = perf_file->private_data;
 		ee->perf_file = perf_file;
 		ee->map_file = map_file;
+		ee->sd = (void *)ee + sizeof(*ee);
 	}
 
 	return ee;
kernel/trace/bpf_trace.c +0 -2
index f92d6ad5e080..076f8e987355 100644
@@ -410,17 +410,17 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
 	.arg4_type	= ARG_CONST_SIZE,
 };
 
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
-

static __always_inline u64 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,

kernel/trace/bpf_trace.c +1 -1
-			u64 flags, struct perf_sample_data *sd)
+			u64 flags, struct perf_raw_record *raw)
{
	struct bpf_array *array = container_of(map, struct bpf_array, map);
	unsigned int cpu = smp_processor_id();
	u64 index = flags & BPF_F_INDEX_MASK;
	struct bpf_event_entry *ee;
	struct perf_event *event;
kernel/trace/bpf_trace.c +2 -0
+	struct perf_sample_data *sd;
+	u64 ret;
	if (index == BPF_F_CURRENT_CPU)
		index = cpu;
kernel/trace/bpf_trace.c +13 -10
@@ -439,13 +439,22 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 	if (unlikely(event->oncpu != cpu))
 		return -EOPNOTSUPP;
 
-	return perf_event_output(event, sd, regs);
+	sd = xchg(&ee->sd, NULL);
+	if (!sd)
+		return -EBUSY;
+
+	perf_sample_data_init(sd, 0, 0);
+	sd->raw = raw;
+
+	ret = perf_event_output(event, sd, regs);
+
+	xchg(&ee->sd, sd);
+	return ret;
 }
 
 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
 	   u64, flags, void *, data, u64, size)
 {
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
 	struct perf_raw_record raw = {
 		.frag = {
 			.size = size,
@@ -456,10 +465,8 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
 		return -EINVAL;
 
-	perf_sample_data_init(sd, 0, 0);
-	sd->raw = &raw;
 
-	return __bpf_perf_event_output(regs, map, flags, sd);
+	return __bpf_perf_event_output(regs, map, flags, &raw);
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -474,12 +481,10 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = {
 };
 
 static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);
 
 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
 {
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_misc_sd);
 	struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);
 	struct perf_raw_frag frag = {
 		.copy		= ctx_copy,
@@ -497,10 +502,8 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
 	};
 
 	perf_fetch_caller_regs(regs);
-	perf_sample_data_init(sd, 0, 0);
-	sd->raw = &raw;
 
-	return __bpf_perf_event_output(regs, map, flags, sd);
+	return __bpf_perf_event_output(regs, map, flags, &raw);
 }
 
 BPF_CALL_0(bpf_get_current_task)
Signature · 1 line
2.17.1
#2 lore raw
Quoted 102 lines · Song Liu
Quoted 64 lines · Matt Mullins

It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:

      bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000
      ? trace_call_bpf+0x82/0x100
      ? sch_direct_xmit+0xe2/0x230
      ? blk_mq_end_request+0x1/0x100
      ? blk_mq_end_request+0x5/0x100
      ? kprobe_perf_func+0x19b/0x240
      ? __qdisc_run+0x86/0x520
      ? blk_mq_end_request+0x1/0x100
      ? blk_mq_end_request+0x5/0x100
      ? kprobe_ftrace_handler+0x90/0xf0
      ? ftrace_ops_assist_func+0x6e/0xe0
      ? ip6_input_finish+0xbf/0x460
      ? 0xffffffffa01e80bf
      ? nbd_dbg_flags_show+0xc0/0xc0 [nbd]
      ? blkdev_issue_zeroout+0x200/0x200
      ? blk_mq_end_request+0x1/0x100
      ? blk_mq_end_request+0x5/0x100
      ? flush_smp_call_function_queue+0x6c/0xe0
      ? smp_call_function_single_interrupt+0x32/0xc0
      ? call_function_single_interrupt+0xf/0x20
      ? call_function_single_interrupt+0xa/0x20
      ? swiotlb_map_page+0x140/0x140
      ? refcount_sub_and_test+0x1a/0x50
      ? tcp_wfree+0x20/0xf0
      ? skb_release_head_state+0x62/0xc0
      ? skb_release_all+0xe/0x30
      ? napi_consume_skb+0xb5/0x100
      ? mlx5e_poll_tx_cq+0x1df/0x4e0
      ? mlx5e_poll_tx_cq+0x38c/0x4e0
      ? mlx5e_napi_poll+0x58/0xc30
      ? mlx5e_napi_poll+0x232/0xc30
      ? net_rx_action+0x128/0x340
      ? __do_softirq+0xd4/0x2ad
      ? irq_exit+0xa5/0xb0
      ? do_IRQ+0x7d/0xc0
      ? common_interrupt+0xf/0xf
      </IRQ>
      ? __rb_free_aux+0xf0/0xf0
      ? perf_output_sample+0x28/0x7b0
      ? perf_prepare_sample+0x54/0x4a0
      ? perf_event_output+0x43/0x60
      ? bpf_perf_event_output_raw_tp+0x15f/0x180
      ? blk_mq_start_request+0x1/0x120
      ? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000
      ? bpf_trace_run3+0x2c/0x80
      ? nbd_send_cmd+0x4c2/0x690 [nbd]

This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.

Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

This looks great. Thanks for the fix.

Quoted 31 lines

---
v1->v2:
keep a pointer to the struct perf_sample_data rather than directly embedding it in the structure, avoiding the circular include and removing the need for in_use. Suggested by Song.

include/linux/bpf.h      |  1 +
kernel/bpf/arraymap.c    |  3 ++-
kernel/trace/bpf_trace.c | 29 ++++++++++++++++-------------
3 files changed, 19 insertions(+), 14 deletions(-)
include/linux/bpf.h +1 -0
index 4fb3aa2dc975..47fd85cfbbaf 100644
@@ -472,6 +472,7 @@ struct bpf_event_entry {
      struct file *perf_file;
      struct file *map_file;
      struct rcu_head rcu;
+     struct perf_sample_data *sd;
 };
 
 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
kernel/bpf/arraymap.c +0 -0
index 584636c9e2eb..c7f5d593e04f 100644
@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
 {
      struct bpf_event_entry *ee;
 
Quoted 7 lines

kernel/bpf/arraymap.c static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, · line 662

Quoted 7 lines
kernel/bpf/arraymap.c +2 -1
-     ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
+     ee = kzalloc(sizeof(*ee) + sizeof(struct perf_sample_data), GFP_ATOMIC);
      if (ee) {
              ee->event = perf_file->private_data;
              ee->perf_file = perf_file;
              ee->map_file = map_file;
+             ee->sd = (void *)ee + sizeof(*ee);

This bit looks quite weird, but I don't have better ideas to avoid circular .h pain.

Applied to bpf tree. Thanks

maintainer
#3 lore raw
In reply to Matt Mullins
Quoted 59 lines · Matt Mullins

It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:

	bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000
	? trace_call_bpf+0x82/0x100
	? sch_direct_xmit+0xe2/0x230
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_perf_func+0x19b/0x240
	? __qdisc_run+0x86/0x520
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_ftrace_handler+0x90/0xf0
	? ftrace_ops_assist_func+0x6e/0xe0
	? ip6_input_finish+0xbf/0x460
	? 0xffffffffa01e80bf
	? nbd_dbg_flags_show+0xc0/0xc0 [nbd]
	? blkdev_issue_zeroout+0x200/0x200
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? flush_smp_call_function_queue+0x6c/0xe0
	? smp_call_function_single_interrupt+0x32/0xc0
	? call_function_single_interrupt+0xf/0x20
	? call_function_single_interrupt+0xa/0x20
	? swiotlb_map_page+0x140/0x140
	? refcount_sub_and_test+0x1a/0x50
	? tcp_wfree+0x20/0xf0
	? skb_release_head_state+0x62/0xc0
	? skb_release_all+0xe/0x30
	? napi_consume_skb+0xb5/0x100
	? mlx5e_poll_tx_cq+0x1df/0x4e0
	? mlx5e_poll_tx_cq+0x38c/0x4e0
	? mlx5e_napi_poll+0x58/0xc30
	? mlx5e_napi_poll+0x232/0xc30
	? net_rx_action+0x128/0x340
	? __do_softirq+0xd4/0x2ad
	? irq_exit+0xa5/0xb0
	? do_IRQ+0x7d/0xc0
	? common_interrupt+0xf/0xf
	</IRQ>
	? __rb_free_aux+0xf0/0xf0
	? perf_output_sample+0x28/0x7b0
	? perf_prepare_sample+0x54/0x4a0
	? perf_event_output+0x43/0x60
	? bpf_perf_event_output_raw_tp+0x15f/0x180
	? blk_mq_start_request+0x1/0x120
	? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000
	? bpf_trace_run3+0x2c/0x80
	? nbd_send_cmd+0x4c2/0x690 [nbd]

This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.

Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

You do not elaborate why is this needed for all the networking programs that use this functionality. The bpf_misc_sd should therefore be kept as-is. There cannot be nested occurrences there (xdp, tc ingress/egress). Please explain why non-tracing should be affected here...

maintainer
#4 lore raw
Quoted 60 lines · Daniel Borkmann
Quoted 59 lines · Matt Mullins

It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:

	bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000
	? trace_call_bpf+0x82/0x100
	? sch_direct_xmit+0xe2/0x230
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_perf_func+0x19b/0x240
	? __qdisc_run+0x86/0x520
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_ftrace_handler+0x90/0xf0
	? ftrace_ops_assist_func+0x6e/0xe0
	? ip6_input_finish+0xbf/0x460
	? 0xffffffffa01e80bf
	? nbd_dbg_flags_show+0xc0/0xc0 [nbd]
	? blkdev_issue_zeroout+0x200/0x200
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? flush_smp_call_function_queue+0x6c/0xe0
	? smp_call_function_single_interrupt+0x32/0xc0
	? call_function_single_interrupt+0xf/0x20
	? call_function_single_interrupt+0xa/0x20
	? swiotlb_map_page+0x140/0x140
	? refcount_sub_and_test+0x1a/0x50
	? tcp_wfree+0x20/0xf0
	? skb_release_head_state+0x62/0xc0
	? skb_release_all+0xe/0x30
	? napi_consume_skb+0xb5/0x100
	? mlx5e_poll_tx_cq+0x1df/0x4e0
	? mlx5e_poll_tx_cq+0x38c/0x4e0
	? mlx5e_napi_poll+0x58/0xc30
	? mlx5e_napi_poll+0x232/0xc30
	? net_rx_action+0x128/0x340
	? __do_softirq+0xd4/0x2ad
	? irq_exit+0xa5/0xb0
	? do_IRQ+0x7d/0xc0
	? common_interrupt+0xf/0xf
	</IRQ>
	? __rb_free_aux+0xf0/0xf0
	? perf_output_sample+0x28/0x7b0
	? perf_prepare_sample+0x54/0x4a0
	? perf_event_output+0x43/0x60
	? bpf_perf_event_output_raw_tp+0x15f/0x180
	? blk_mq_start_request+0x1/0x120
	? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000
	? bpf_trace_run3+0x2c/0x80
	? nbd_send_cmd+0x4c2/0x690 [nbd]

This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.

Quoted 7 lines
Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

You do not elaborate why is this needed for all the networking programs that use this functionality. The bpf_misc_sd should therefore be kept as-is. There cannot be nested occurrences there (xdp, tc ingress/egress). Please explain why non-tracing should be affected here...

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

#5 lore raw
Quoted 61 lines · Daniel Borkmann
Quoted 60 lines · Daniel Borkmann
Quoted 59 lines · Matt Mullins

It is possible that a BPF program can be called while another BPF program is executing bpf_perf_event_output. This has been observed with I/O completion occurring as a result of an interrupt:

	bpf_prog_247fd1341cddaea4_trace_req_end+0x8d7/0x1000
	? trace_call_bpf+0x82/0x100
	? sch_direct_xmit+0xe2/0x230
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_perf_func+0x19b/0x240
	? __qdisc_run+0x86/0x520
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? kprobe_ftrace_handler+0x90/0xf0
	? ftrace_ops_assist_func+0x6e/0xe0
	? ip6_input_finish+0xbf/0x460
	? 0xffffffffa01e80bf
	? nbd_dbg_flags_show+0xc0/0xc0 [nbd]
	? blkdev_issue_zeroout+0x200/0x200
	? blk_mq_end_request+0x1/0x100
	? blk_mq_end_request+0x5/0x100
	? flush_smp_call_function_queue+0x6c/0xe0
	? smp_call_function_single_interrupt+0x32/0xc0
	? call_function_single_interrupt+0xf/0x20
	? call_function_single_interrupt+0xa/0x20
	? swiotlb_map_page+0x140/0x140
	? refcount_sub_and_test+0x1a/0x50
	? tcp_wfree+0x20/0xf0
	? skb_release_head_state+0x62/0xc0
	? skb_release_all+0xe/0x30
	? napi_consume_skb+0xb5/0x100
	? mlx5e_poll_tx_cq+0x1df/0x4e0
	? mlx5e_poll_tx_cq+0x38c/0x4e0
	? mlx5e_napi_poll+0x58/0xc30
	? mlx5e_napi_poll+0x232/0xc30
	? net_rx_action+0x128/0x340
	? __do_softirq+0xd4/0x2ad
	? irq_exit+0xa5/0xb0
	? do_IRQ+0x7d/0xc0
	? common_interrupt+0xf/0xf
	</IRQ>
	? __rb_free_aux+0xf0/0xf0
	? perf_output_sample+0x28/0x7b0
	? perf_prepare_sample+0x54/0x4a0
	? perf_event_output+0x43/0x60
	? bpf_perf_event_output_raw_tp+0x15f/0x180
	? blk_mq_start_request+0x1/0x120
	? bpf_prog_411a64a706fc6044_should_trace+0xad4/0x1000
	? bpf_trace_run3+0x2c/0x80
	? nbd_send_cmd+0x4c2/0x690 [nbd]

This also cannot be alleviated by further splitting the per-cpu perf_sample_data structs (as in commit 283ca526a9bd ("bpf: fix corruption on concurrent perf_event_output calls")), as a raw_tp could be attached to the block:block_rq_complete tracepoint and execute during another raw_tp. Instead, keep a pre-allocated perf_sample_data structure per perf_event_array element and fail a bpf_perf_event_output if that element is concurrently being used.

Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

You do not elaborate why is this needed for all the networking programs that use this functionality. The bpf_misc_sd should therefore be kept as-is. There cannot be nested occurrences there (xdp, tc ingress/egress). Please explain why non-tracing should be affected here...

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

#6 lore raw
Quoted 7 lines · Matt Mullins

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf

Quoted 10 lines
Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

Quoted 4 lines

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.

maintainer
#7 lore raw
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf

Quoted 10 lines
Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

Quoted 4 lines

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

Quoted 5 lines

I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.

My main worry with this xchg() trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.

#8 lore raw
Quoted 36 lines · Daniel Borkmann
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf

Quoted 10 lines
Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

Quoted 4 lines

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

Quoted 5 lines

I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.

Quoted 7 lines

My main worry with this xchg() trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.

I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog calls event_output and while rb logic is working
another prog needs to start executing and use the same event array slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing BPF_F_CURRENT_CPU.
Same kind of a bug.

maintainer
#9 lore raw
Quoted 44 lines · Alexei Starovoitov
Quoted 35 lines · Daniel Borkmann
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf

Quoted 10 lines
Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

Quoted 4 lines

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

Quoted 5 lines

I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.

Quoted 7 lines

My main worry with this xchg() trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.

Quoted 4 lines

I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog calls event_output and while rb logic is working
another prog needs to start executing and use the same event array

Correct.

Quoted 3 lines

slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing BPF_F_CURRENT_CPU.

I don't see the user bug part, why should that be a user bug? It's the same as if we would say that sharing a BPF hash map between networking programs attached to different hooks or networking and tracing would be a user bug which it is not. One concrete example would be cilium monitor where we currently expose skb trace and drop events a well as debug data through the same rb. This should be usable from any type that has perf_event_output helper enabled (e.g. XDP and tc/BPF) w/o requiring to walk yet another per cpu mmap rb from user space.

#10 lore raw
Quoted 51 lines · Daniel Borkmann
Quoted 44 lines · Alexei Starovoitov
Quoted 35 lines · Daniel Borkmann
Quoted 28 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf

Quoted 10 lines
Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

Quoted 4 lines

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

Quoted 5 lines

I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.

Quoted 7 lines

My main worry with this xchg() trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.

Quoted 4 lines

I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog calls event_output and while rb logic is working
another prog needs to start executing and use the same event array

Quoted 7 lines

Correct.

Quoted 3 lines

slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing BPF_F_CURRENT_CPU.

I don't see the user bug part, why should that be a user bug?

because I suspect that 'struct bpf_event_entry' is not reentrable (even w/o issues with 'struct perf_sample_data').

Even if we always use 'struct perf_sample_data' on stack, I suspect the same 'struct bpf_event_entry' cannot be reused in the nested way.

Quoted 8 lines

It's the same
as if we would say that sharing a BPF hash map between networking programs attached to different hooks or networking and tracing would be a user bug which it is not. One concrete example would be cilium monitor where we currently expose skb trace and drop events a well as debug data through the same rb. This should be usable from any type that has perf_event_output helper enabled (e.g. XDP and tc/BPF) w/o requiring to walk yet another per cpu mmap rb from user space.

sure. those are valid use cases, but in all cases bpf_event_output() on particular 'struct bpf_event_entry' will end before another one will begin, no?

#11 lore raw
In reply to Daniel Borkmann
Quoted 37 lines · Daniel Borkmann
Quoted 36 lines · Alexei Starovoitov
Quoted 35 lines · Daniel Borkmann
Quoted 33 lines · Alexei Starovoitov
Quoted 7 lines · Matt Mullins

If these are invariably non-nested, I can easily keep bpf_misc_sd when I resubmit. There was no technical reason other than keeping the two codepaths as similar as possible.

What resource gives you worry about doing this for the networking codepath?

my preference would be to keep tracing and networking the same. there is already minimal nesting in networking and probably we see more when reuseport progs will start running from xdp and clsbpf

Quoted 10 lines
Quoted 4 lines

Aside from that it's also really bad to miss events like this as exporting through rb is critical. Why can't you have a per-CPU counter that selects a sample data context based on nesting level in tracing? (I don't see a discussion of this in your commit message.)

This change would only drop messages if the same perf_event is attempted to be used recursively (i.e. the same CPU on the same PERF_EVENT_ARRAY map, as I haven't observed anything use index != BPF_F_CURRENT_CPU in testing).

Quoted 4 lines

I'll try to accomplish the same with a percpu nesting level and allocating 2 or 3 perf_sample_data per cpu. I think that'll solve the same problem -- a local patch keeping track of the nesting level is how I got the above stack trace, too.

I don't think counter approach works. The amount of nesting is unknown. imo the approach taken in this patch is good.
I don't see any issue when event_outputs will be dropped for valid progs. Only when user called the helper incorrectly without BPF_F_CURRENT_CPU. But that's an error anyway.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

My main worry with this xchg() trick is that we'll miss to export crucial data with the EBUSY bailing out especially given nesting could increase in future as you state, so users might have a hard time debugging this kind of issue if they share the same perf event map among these programs, and no option to get to this data otherwise. Supporting nesting up to a certain level would still be better than a lost event which is also not reported through the usual way aka perf rb.

Tracing can already be lossy: trace_call_bpf() silently simply doesn't call the prog and instead returns zero if bpf_prog_active != 1.

Quoted 12 lines
Quoted 5 lines

I simply don't see this 'miss to export data' in all but contrived conditions. Say two progs share the same perf event array.
One prog calls event_output and while rb logic is working
another prog needs to start executing and use the same event array

Correct.

Quoted 3 lines

slot. Today it's only possible for tracing prog combined with networking, but having two progs use the same event output array is pretty much a user bug. Just like not passing BPF_F_CURRENT_CPU.

Quoted 8 lines

I don't see the user bug part, why should that be a user bug? It's the same as if we would say that sharing a BPF hash map between networking programs attached to different hooks or networking and tracing would be a user bug which it is not. One concrete example would be cilium monitor where we currently expose skb trace and drop events a well as debug data through the same rb. This should be usable from any type that has perf_event_output helper enabled (e.g. XDP and tc/BPF) w/o requiring to walk yet another per cpu mmap rb from user space.

Neither of these solutions would affect the behavior of sharing the perf array between networking programs -- since they're never called in a nested fashion, then you'll never hit the "xchg() returned NULL" at all.

That said, I think I can logically limit nesting in tracing to 3 levels:
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to run, and
- if we're really unlucky, the same in nmi
at most one of which can be a kprobe or perf event.

There is also a comment

/*
* bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
* to avoid potential recursive reuse issue when/if tracepoints are added
* inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack */

that suggests that one day bpf_perf_event_output might grow a static tracepoint. However, if the program attached to such a hypothetical tracepoint were to call bpf_perf_event_output, that would infinitely recurse ... it seems fine to let that case return -EBUSY as well. It does make me wonder if I should do the same nesting for the pt_regs.

I've now got an experiment running with the counter approach, and the workload that I hit the original crash with seems to be fine with 2 layers' worth -- though if we decide that's the one I should move forward with, I'll probably bump it to a third to be safe for an NMI.

#12 lore raw

PATCH 1/1 bpf: fix nested bpf tracepoints with per-cpu data

BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as they do not increment bpf_prog_active while executing.

This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
(at most one of which may be a kprobe or perf event).

Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

---
This is more lines of code, but possibly less intrusive than the per-array-element approach.

I don't necessarily like that I duplicated the nest_level logic in two places, but I don't see a way to unify them:
- kprobes' bpf_perf_event_output doesn't use bpf_raw_tp_regs, and does use the perf_sample_data,
- raw tracepoints' bpf_get_stackid uses bpf_raw_tp_regs, but not the perf_sample_data, and
- raw tracepoints' bpf_perf_event_output uses both...

 kernel/trace/bpf_trace.c | 95 +++++++++++++++++++++++++++++++++-------
 1 file changed, 80 insertions(+), 15 deletions(-)
kernel/trace/bpf_trace.c +80 -15
index f92d6ad5e080..4f5419837ddd 100644
@@ -410,8 +410,6 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
 	.arg4_type	= ARG_CONST_SIZE,
 };
 
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
-
 static __always_inline u64
 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 			u64 flags, struct perf_sample_data *sd)
@@ -442,24 +440,47 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 	return perf_event_output(event, sd, regs);
 }
 
+/*
+ * Support executing tracepoints in normal, irq, and nmi context that each call
+ * bpf_perf_event_output
+ */
+struct bpf_trace_sample_data {
+	struct perf_sample_data sds[3];
+};
+
+static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);
+static DEFINE_PER_CPU(int, bpf_trace_nest_level);
 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
 	   u64, flags, void *, data, u64, size)
 {
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
+	struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds);
+	struct perf_sample_data *sd;
+	int nest_level = this_cpu_inc_return(bpf_trace_nest_level);
 	struct perf_raw_record raw = {
 		.frag = {
 			.size = size,
 			.data = data,
 		},
 	};
+	int err = -EBUSY;
 
+	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds)))
+		goto out;
+
+	sd = &sds->sds[nest_level - 1];
+
+	err = -EINVAL;
 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
-		return -EINVAL;
+		goto out;
 
 	perf_sample_data_init(sd, 0, 0);
 	sd->raw = &raw;
 
-	return __bpf_perf_event_output(regs, map, flags, sd);
+	err = __bpf_perf_event_output(regs, map, flags, sd);
+
+out:
+	this_cpu_dec(bpf_trace_nest_level);
+	return err;
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -822,16 +843,48 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 /*
  * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
  * to avoid potential recursive reuse issue when/if tracepoints are added
- * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack
+ * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack.
+ *
+ * Since raw tracepoints run despite bpf_prog_active, support concurrent usage
+ * in normal, irq, and nmi context.
  */
-static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs);
+struct bpf_raw_tp_regs {
+	struct pt_regs regs[3];
+};
+static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs);
+static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level);
+static struct pt_regs *get_bpf_raw_tp_regs(void)
+{
+	struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
+
+	if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) {
+		this_cpu_dec(bpf_raw_tp_nest_level);
+		return ERR_PTR(-EBUSY);
+	}
+
+	return &tp_regs->regs[nest_level - 1];
+}
+
+static void put_bpf_raw_tp_regs(void)
+{
+	this_cpu_dec(bpf_raw_tp_nest_level);
+}
+
 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   struct bpf_map *, map, u64, flags, void *, data, u64, size)
 {
-	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	struct pt_regs *regs = get_bpf_raw_tp_regs();
+	int ret;
+
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	perf_fetch_caller_regs(regs);
-	return ____bpf_perf_event_output(regs, map, flags, data, size);
+	ret = ____bpf_perf_event_output(regs, map, flags, data, size);
+
+	put_bpf_raw_tp_regs();
+	return ret;
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
@@ -848,12 +901,18 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   struct bpf_map *, map, u64, flags)
 {
-	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	struct pt_regs *regs = get_bpf_raw_tp_regs();
+	int ret;
+
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	perf_fetch_caller_regs(regs);
 	/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
-	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
-			       flags, 0, 0);
+	ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map,
+			      flags, 0, 0);
+	put_bpf_raw_tp_regs();
+	return ret;
 }
 
 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
@@ -868,11 +927,17 @@ static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   void *, buf, u32, size, u64, flags)
 {
-	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+	struct pt_regs *regs = get_bpf_raw_tp_regs();
+	int ret;
+
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	perf_fetch_caller_regs(regs);
-	return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
-			     (unsigned long) size, flags, 0);
+	ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf,
+			    (unsigned long) size, flags, 0);
+	put_bpf_raw_tp_regs();
+	return ret;
 }
 
 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
Signature · 1 line
2.17.1
#13 lore raw

Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data

Quoted 10 lines · Matt Mullins

BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as they do not increment bpf_prog_active while executing.

This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
(at most one of which may be a kprobe or perf event).

Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

No comment on the code, but you're definitely missing a sign-off.

Quoted 11 lines

---
This is more lines of code, but possibly less intrusive than the per-array-element approach.

I don't necessarily like that I duplicated the nest_level logic in two places, but I don't see a way to unify them:
- kprobes' bpf_perf_event_output doesn't use bpf_raw_tp_regs, and does use the perf_sample_data,
- raw tracepoints' bpf_get_stackid uses bpf_raw_tp_regs, but not the perf_sample_data, and
- raw tracepoints' bpf_perf_event_output uses both...

#14 lore raw

Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data

Quoted 10 lines · Jakub Kicinski
Quoted 9 lines · Matt Mullins

BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as they do not increment bpf_prog_active while executing.

This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context
(at most one of which may be a kprobe or perf event).

Quoted 3 lines
Quoted 1 line
Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

No comment on the code, but you're definitely missing a sign-off.

Oops, I totally am. I'll give it some more time for opinions to roll in, and I'll fix that before I resubmit :)

Quoted 14 lines
Quoted 12 lines

---
This is more lines of code, but possibly less intrusive than the per-array-element approach.

I don't necessarily like that I duplicated the nest_level logic in two places, but I don't see a way to unify them:
- kprobes' bpf_perf_event_output doesn't use bpf_raw_tp_regs, and does use the perf_sample_data,
- raw tracepoints' bpf_get_stackid uses bpf_raw_tp_regs, but not the perf_sample_data, and
- raw tracepoints' bpf_perf_event_output uses both...

#15 lore raw

Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data

In reply to Matt Mullins
Quoted 8 lines · Matt Mullins

BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as they do not increment bpf_prog_active while executing.

This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context

Can NMIs be nested?

Quoted 42 lines

(at most one of which may be a kprobe or perf event).

Fixes 20b9d7ac4852 ("bpf: avoid excessive stack usage for perf_sample_data")

---
This is more lines of code, but possibly less intrusive than the per-array-element approach.

I don't necessarily like that I duplicated the nest_level logic in two places, but I don't see a way to unify them:
- kprobes' bpf_perf_event_output doesn't use bpf_raw_tp_regs, and does use the perf_sample_data,
- raw tracepoints' bpf_get_stackid uses bpf_raw_tp_regs, but not the perf_sample_data, and
- raw tracepoints' bpf_perf_event_output uses both...

 kernel/trace/bpf_trace.c | 95 +++++++++++++++++++++++++++++++++-------
 1 file changed, 80 insertions(+), 15 deletions(-)
kernel/trace/bpf_trace.c +7 -2
index f92d6ad5e080..4f5419837ddd 100644
@@ -410,8 +410,6 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
        .arg4_type      = ARG_CONST_SIZE,
 };
 
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
-
 static __always_inline u64
 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
                        u64 flags, struct perf_sample_data *sd)
@@ -442,24 +440,47 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
        return perf_event_output(event, sd, regs);
 }
 
+/*
+ * Support executing tracepoints in normal, irq, and nmi context that each call
+ * bpf_perf_event_output
+ */
+struct bpf_trace_sample_data {
+       struct perf_sample_data sds[3];
+};

kernel/trace/bpf_trace.c __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, · line 458

Quoted 10 lines
kernel/trace/bpf_trace.c +6 -1
+
+static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds);
+static DEFINE_PER_CPU(int, bpf_trace_nest_level);
 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
           u64, flags, void *, data, u64, size)
 {
-       struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
+       struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds);
+       struct perf_sample_data *sd;
+       int nest_level = this_cpu_inc_return(bpf_trace_nest_level);

reverse Christmas tree?

kernel/trace/bpf_trace.c __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, · line 468

Quoted 10 lines
kernel/trace/bpf_trace.c +3 -0
        struct perf_raw_record raw = {
                .frag = {
                        .size = size,
                        .data = data,
                },
        };
+       int err = -EBUSY;
 
+       if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds)))
+               goto out;

consider this a nit, but I find it much simpler to follow when err is set just before goto, so that it's clear what's going to be returned:

int err;

if (something_bad) {
    err = -EBAD_ERR_CODE1;
    goto out;
}

kernel/trace/bpf_trace.c __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, · line 474

Quoted 7 lines
kernel/trace/bpf_trace.c +5 -1
+
+       sd = &sds->sds[nest_level - 1];
+
+       err = -EINVAL;
        if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
-               return -EINVAL;
+               goto out;

Same here.

Quoted 112 lines
kernel/trace/bpf_trace.c +59 -12
        perf_sample_data_init(sd, 0, 0);
        sd->raw = &raw;
 
-       return __bpf_perf_event_output(regs, map, flags, sd);
+       err = __bpf_perf_event_output(regs, map, flags, sd);
+
+out:
+       this_cpu_dec(bpf_trace_nest_level);
+       return err;
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -822,16 +843,48 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 /*
  * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
  * to avoid potential recursive reuse issue when/if tracepoints are added
- * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack
+ * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack.
+ *
+ * Since raw tracepoints run despite bpf_prog_active, support concurrent usage
+ * in normal, irq, and nmi context.
  */
-static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs);
+struct bpf_raw_tp_regs {
+       struct pt_regs regs[3];
+};
+static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs);
+static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level);
+static struct pt_regs *get_bpf_raw_tp_regs(void)
+{
+       struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
+       int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
+
+       if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) {
+               this_cpu_dec(bpf_raw_tp_nest_level);
+               return ERR_PTR(-EBUSY);
+       }
+
+       return &tp_regs->regs[nest_level - 1];
+}
+
+static void put_bpf_raw_tp_regs(void)
+{
+       this_cpu_dec(bpf_raw_tp_nest_level);
+}
+
 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
           struct bpf_map *, map, u64, flags, void *, data, u64, size)
 {
-       struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+       struct pt_regs *regs = get_bpf_raw_tp_regs();
+       int ret;
+
+       if (IS_ERR(regs))
+               return PTR_ERR(regs);
 
        perf_fetch_caller_regs(regs);
-       return ____bpf_perf_event_output(regs, map, flags, data, size);
+       ret = ____bpf_perf_event_output(regs, map, flags, data, size);
+
+       put_bpf_raw_tp_regs();
+       return ret;
 }
 
 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
@@ -848,12 +901,18 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
           struct bpf_map *, map, u64, flags)
 {
-       struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+       struct pt_regs *regs = get_bpf_raw_tp_regs();
+       int ret;
+
+       if (IS_ERR(regs))
+               return PTR_ERR(regs);
 
        perf_fetch_caller_regs(regs);
        /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
-       return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
-                              flags, 0, 0);
+       ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map,
+                             flags, 0, 0);
+       put_bpf_raw_tp_regs();
+       return ret;
 }
 
 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
@@ -868,11 +927,17 @@ static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
           void *, buf, u32, size, u64, flags)
 {
-       struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
+       struct pt_regs *regs = get_bpf_raw_tp_regs();
+       int ret;
+
+       if (IS_ERR(regs))
+               return PTR_ERR(regs);
 
        perf_fetch_caller_regs(regs);
-       return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
-                            (unsigned long) size, flags, 0);
+       ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf,
+                           (unsigned long) size, flags, 0);
+       put_bpf_raw_tp_regs();
+       return ret;
 }
 
 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
--

2.17.1

maintainer
#16 lore raw

Re: [PATCH bpf] bpf: fix nested bpf tracepoints with per-cpu data

Quoted 11 lines · On Thu, 6 Jun 2019 19:59:18 -0700 Andrii Nakryiko <[email protected]>…
Quoted 8 lines · Matt Mullins

BPF_PROG_TYPE_RAW_TRACEPOINTs can be executed nested on the same CPU, as they do not increment bpf_prog_active while executing.

This enables three levels of nesting, to support
- a kprobe or raw tp or perf event,
- another one of the above that irq context happens to call, and
- another one in nmi context

Can NMIs be nested?

No, otherwise several things in the kernel will break.

-- Steve