bpf_perf_event_output helper

This helper is used to send raw data from eBPF program into special PERF_TYPE_SOFTWARE/PERF_COUNT_SW_BPF_OUTPUT perf_event. User space needs to perf_event_open() it (either for one or all cpus) and store FD into perf_event_array (similar to bpf_perf_event_read() helper) before eBPF program can send data into it.

Today the programs triggered by kprobe collect the data and either store it into the maps or print it via bpf_trace_printk() where latter is the debug facility and not suitable to stream the data. This new helper replaces such bpf_trace_printk() usage and allows programs to have dedicated channel into user space for post-processing of the raw data collected.

Signed-off-by Alexei Starovoitov <[email protected]>

---
  include/uapi/linux/bpf.h        |   11 ++++++++++
  include/uapi/linux/perf_event.h |    1 +
  kernel/bpf/arraymap.c           |    2 ++
  kernel/bpf/verifier.c           |    3 ++-
  kernel/trace/bpf_trace.c        |   46 +++++++++++++++++++++++++++++++++++++++
  5 files changed, 62 insertions(+), 1 deletion(-)

include/uapi/linux/bpf.h +11 -0

index 564f1f091991..2e032426cfb7 100644
@@ -287,6 +287,17 @@ enum bpf_func_id {
  	 * Return: realm if != 0
  	 */
  	BPF_FUNC_get_route_realm,
+
+	/**
+	 * bpf_perf_event_output(ctx, map, index, data, size) - output perf raw sample
+	 * @ctx: struct pt_regs*
+	 * @map: pointer to perf_event_array map
+	 * @index: index of event in the map
+	 * @data: data on stack to be output as raw data
+	 * @size: size of data
+	 * Return: 0 on success
+	 */
+	BPF_FUNC_perf_event_output,
  	__BPF_FUNC_MAX_ID,
  };
 

include/uapi/linux/perf_event.h +1 -0

index 2881145cda86..d3c417615361 100644
@@ -110,6 +110,7 @@ enum perf_sw_ids {
  	PERF_COUNT_SW_ALIGNMENT_FAULTS		= 7,
  	PERF_COUNT_SW_EMULATION_FAULTS		= 8,
  	PERF_COUNT_SW_DUMMY			= 9,
+	PERF_COUNT_SW_BPF_OUTPUT		= 10,
 
  	PERF_COUNT_SW_MAX,			/* non-ABI */
  };

kernel/bpf/arraymap.c +2 -0

index f2d9e698c753..e3cfe46b074f 100644
@@ -295,6 +295,8 @@ static void *perf_event_fd_array_get_ptr(struct bpf_map *map, int fd)
  		return (void *)attr;
 
  	if (attr->type != PERF_TYPE_RAW &&
+	    !(attr->type == PERF_TYPE_SOFTWARE &&
+	      attr->config == PERF_COUNT_SW_BPF_OUTPUT) &&
  	    attr->type != PERF_TYPE_HARDWARE) {
  		perf_event_release_kernel(event);
  		return ERR_PTR(-EINVAL);

kernel/bpf/verifier.c +2 -1

index 1d6b97be79e1..b56cf51f8d42 100644
@@ -245,6 +245,7 @@ static const struct {
  } func_limit[] = {
  	{BPF_MAP_TYPE_PROG_ARRAY, BPF_FUNC_tail_call},
  	{BPF_MAP_TYPE_PERF_EVENT_ARRAY, BPF_FUNC_perf_event_read},
+	{BPF_MAP_TYPE_PERF_EVENT_ARRAY, BPF_FUNC_perf_event_output},
  };
 
  static void print_verifier_state(struct verifier_env *env)

@@ -910,7 +911,7 @@ static int check_map_func_compatibility(struct bpf_map *map, int func_id)
  		 * don't allow any other map type to be passed into
  		 * the special func;
  		 */
-		if (bool_map != bool_func)
+		if (bool_func && bool_map != bool_func)
  			return -EINVAL;
  	}
 

kernel/trace/bpf_trace.c +46 -0

index 0fe96c7c8803..47febbe7998e 100644
@@ -215,6 +215,50 @@ const struct bpf_func_proto bpf_perf_event_read_proto = {
  	.arg2_type	= ARG_ANYTHING,
  };
 
+static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 index, u64 r4, u64 size)
+{
+	struct pt_regs *regs = (struct pt_regs *) (long) r1;
+	struct bpf_map *map = (struct bpf_map *) (long) r2;
+	struct bpf_array *array = container_of(map, struct bpf_array, map);
+	void *data = (void *) (long) r4;
+	struct perf_sample_data sample_data;
+	struct perf_event *event;
+	struct perf_raw_record raw = {
+		.size = size,
+		.data = data,
+	};
+
+	if (unlikely(index >= array->map.max_entries))
+		return -E2BIG;
+
+	event = (struct perf_event *)array->ptrs[index];
+	if (unlikely(!event))
+		return -ENOENT;
+
+	if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
+		     event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
+		return -EINVAL;
+
+	if (unlikely(event->oncpu != smp_processor_id()))
+		return -EOPNOTSUPP;
+
+	perf_sample_data_init(&sample_data, 0, 0);
+	sample_data.raw = &raw;
+	perf_event_output(event, &sample_data, regs);
+	return 0;
+}
+
+static const struct bpf_func_proto bpf_perf_event_output_proto = {
+	.func		= bpf_perf_event_output,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_CONST_MAP_PTR,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_STACK,
+	.arg5_type	= ARG_CONST_STACK_SIZE,
+};
+
  static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func_id)
  {
  	switch (func_id) {

@@ -242,6 +286,8 @@ static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func
  		return &bpf_get_smp_processor_id_proto;
  	case BPF_FUNC_perf_event_read:
  		return &bpf_perf_event_read_proto;
+	case BPF_FUNC_perf_event_output:
+		return &bpf_perf_event_output_proto;
  	default:
  		return NULL;
  	}

hi, Alexei

I've tested the sample in next patch and it works well. I think more work on the perf side needs to be done for parsing PERF_COUNT_SW_BPF_OUTPUT event type, are you working on that?

Thank you.

Quoted 162 lines

This helper is used to send raw data from eBPF program into special PERF_TYPE_SOFTWARE/PERF_COUNT_SW_BPF_OUTPUT perf_event. User space needs to perf_event_open() it (either for one or all cpus) and store FD into perf_event_array (similar to bpf_perf_event_read() helper) before eBPF program can send data into it.

Today the programs triggered by kprobe collect the data and either store it into the maps or print it via bpf_trace_printk() where latter is the debug facility and not suitable to stream the data. This new helper replaces such bpf_trace_printk() usage and allows programs to have dedicated channel into user space for post-processing of the raw data collected.

Signed-off-by Alexei Starovoitov <[email protected]>

---
  include/uapi/linux/bpf.h        |   11 ++++++++++
  include/uapi/linux/perf_event.h |    1 +
  kernel/bpf/arraymap.c           |    2 ++
  kernel/bpf/verifier.c           |    3 ++-
  kernel/trace/bpf_trace.c        |   46 
+++++++++++++++++++++++++++++++++++++++
  5 files changed, 62 insertions(+), 1 deletion(-)

include/uapi/linux/bpf.h +3 -0

index 564f1f091991..2e032426cfb7 100644
@@ -287,6 +287,17 @@ enum bpf_func_id {
       * Return: realm if != 0
       */
      BPF_FUNC_get_route_realm,
+
+    /**
+     * bpf_perf_event_output(ctx, map, index, data, size) - output 

perf raw sample

include/uapi/linux/bpf.h +8 -0

+     * @ctx: struct pt_regs*
+     * @map: pointer to perf_event_array map
+     * @index: index of event in the map
+     * @data: data on stack to be output as raw data
+     * @size: size of data
+     * Return: 0 on success
+     */
+    BPF_FUNC_perf_event_output,
      __BPF_FUNC_MAX_ID,
  };

include/uapi/linux/perf_event.h +1 -0

index 2881145cda86..d3c417615361 100644
@@ -110,6 +110,7 @@ enum perf_sw_ids {
      PERF_COUNT_SW_ALIGNMENT_FAULTS        = 7,
      PERF_COUNT_SW_EMULATION_FAULTS        = 8,
      PERF_COUNT_SW_DUMMY            = 9,
+    PERF_COUNT_SW_BPF_OUTPUT        = 10,
 
      PERF_COUNT_SW_MAX,            /* non-ABI */
  };

kernel/bpf/arraymap.c +2 -0

index f2d9e698c753..e3cfe46b074f 100644
@@ -295,6 +295,8 @@ static void *perf_event_fd_array_get_ptr(struct 
 bpf_map *map, int fd)
          return (void *)attr;
 
      if (attr->type != PERF_TYPE_RAW &&
+        !(attr->type == PERF_TYPE_SOFTWARE &&
+          attr->config == PERF_COUNT_SW_BPF_OUTPUT) &&
          attr->type != PERF_TYPE_HARDWARE) {
          perf_event_release_kernel(event);

          return ERR_PTR(-EINVAL);

kernel/bpf/verifier.c +2 -1

index 1d6b97be79e1..b56cf51f8d42 100644
@@ -245,6 +245,7 @@ static const struct {
  } func_limit[] = {
      {BPF_MAP_TYPE_PROG_ARRAY, BPF_FUNC_tail_call},
      {BPF_MAP_TYPE_PERF_EVENT_ARRAY, BPF_FUNC_perf_event_read},
+    {BPF_MAP_TYPE_PERF_EVENT_ARRAY, BPF_FUNC_perf_event_output},
  };
 
  static void print_verifier_state(struct verifier_env *env)

@@ -910,7 +911,7 @@ static int check_map_func_compatibility(struct 
 bpf_map *map, int func_id)
           * don't allow any other map type to be passed into
           * the special func;
           */
-        if (bool_map != bool_func)
+        if (bool_func && bool_map != bool_func)
              return -EINVAL;
      }

kernel/trace/bpf_trace.c +44 -0

index 0fe96c7c8803..47febbe7998e 100644
@@ -215,6 +215,50 @@ const struct bpf_func_proto 
 bpf_perf_event_read_proto = {
      .arg2_type    = ARG_ANYTHING,
  };
 
+static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 index, u64 r4, 
 u64 size)
+{
+    struct pt_regs *regs = (struct pt_regs *) (long) r1;
+    struct bpf_map *map = (struct bpf_map *) (long) r2;
+    struct bpf_array *array = container_of(map, struct bpf_array, map);
+    void *data = (void *) (long) r4;
+    struct perf_sample_data sample_data;
+    struct perf_event *event;
+    struct perf_raw_record raw = {
+        .size = size,
+        .data = data,
+    };
+
+    if (unlikely(index >= array->map.max_entries))
+        return -E2BIG;
+
+    event = (struct perf_event *)array->ptrs[index];
+    if (unlikely(!event))
+        return -ENOENT;
+
+    if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
+             event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
+        return -EINVAL;
+
+    if (unlikely(event->oncpu != smp_processor_id()))
+        return -EOPNOTSUPP;
+
+    perf_sample_data_init(&sample_data, 0, 0);
+    sample_data.raw = &raw;
+    perf_event_output(event, &sample_data, regs);
+    return 0;
+}
+
+static const struct bpf_func_proto bpf_perf_event_output_proto = {
+    .func        = bpf_perf_event_output,
+    .gpl_only    = false,
+    .ret_type    = RET_INTEGER,
+    .arg1_type    = ARG_PTR_TO_CTX,
+    .arg2_type    = ARG_CONST_MAP_PTR,
+    .arg3_type    = ARG_ANYTHING,
+    .arg4_type    = ARG_PTR_TO_STACK,
+    .arg5_type    = ARG_CONST_STACK_SIZE,
+};
+
  static const struct bpf_func_proto *kprobe_prog_func_proto(enum 

bpf_func_id func_id)

  {
      switch (func_id) {

kernel/trace/bpf_trace.c +2 -0

@@ -242,6 +286,8 @@ static const struct bpf_func_proto 
 *kprobe_prog_func_proto(enum bpf_func_id func
          return &bpf_get_smp_processor_id_proto;
      case BPF_FUNC_perf_event_read:
          return &bpf_perf_event_read_proto;
+    case BPF_FUNC_perf_event_output:
+        return &bpf_perf_event_output_proto;
      default:
          return NULL;

      }

I think we can
make perf output raw data hexadamically at first.

This helper is used to send raw data from eBPF program into special PERF_TYPE_SOFTWARE/PERF_COUNT_SW_BPF_OUTPUT perf_event. User space needs to perf_event_open() it (either for one or all cpus) and store FD into perf_event_array (similar to bpf_perf_event_read() helper) before eBPF program can send data into it.

include/uapi/linux/perf_event.h +0 -0

index 2881145cda86..d3c417615361 100644

Quoted 5 lines

PERF_COUNT_SW_DUMMY = 9,

include/uapi/linux/perf_event.h enum perf_sw_ids { · line 116

Quoted 4 lines

include/uapi/linux/perf_event.h +1 -0

+	PERF_COUNT_SW_BPF_OUTPUT		= 10,
 
 	PERF_COUNT_SW_MAX,			/* non-ABI */
 };

Do you really need the new type? Can't you use DUMMY for this?

Quoted 6 lines · Peter Zijlstra

Quoted 5 lines

PERF_COUNT_SW_DUMMY = 9,

include/uapi/linux/perf_event.h enum perf_sw_ids { · line 116

Quoted 4 lines

include/uapi/linux/perf_event.h +1 -0

+	PERF_COUNT_SW_BPF_OUTPUT		= 10,
 
 	PERF_COUNT_SW_MAX,			/* non-ABI */
 };

Do you really need the new type? Can't you use DUMMY for this?

It works fine with dummy, but user space code looked odd when it perf_event_open-s dummy events and starts reading them. So I thought either to add alias:
PERF_COUNT_SW_BPF_OUTPUT = PERF_COUNT_SW_DUMMY
or
add new number like I did above.
New number is cheap and I saw that dummy used for probing, so went with new number to disambiguate.

+++ b/kernel/trace/bpf_trace.c

kernel/trace/bpf_trace.c +22 -0

@@ -215,6 +215,50 @@ const struct bpf_func_proto bpf_perf_event_read_proto = {
 	.arg2_type	= ARG_ANYTHING,
 };
 
+static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 index, u64 r4, u64 size)
+{
+	struct pt_regs *regs = (struct pt_regs *) (long) r1;
+	struct bpf_map *map = (struct bpf_map *) (long) r2;
+	struct bpf_array *array = container_of(map, struct bpf_array, map);
+	void *data = (void *) (long) r4;
+	struct perf_sample_data sample_data;
+	struct perf_event *event;
+	struct perf_raw_record raw = {
+		.size = size,
+		.data = data,
+	};
+
+	if (unlikely(index >= array->map.max_entries))
+		return -E2BIG;
+
+	event = (struct perf_event *)array->ptrs[index];
+	if (unlikely(!event))
+		return -ENOENT;
+
+	if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
+		     event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))

Quoted 8 lines

kernel/trace/bpf_trace.c +1 -0

+	if (unlikely(event->oncpu != smp_processor_id()))

kernel/trace/bpf_trace.c const struct bpf_func_proto bpf_perf_event_read_proto = { · line 249

Quoted 7 lines

kernel/trace/bpf_trace.c +7 -0

+		return -EOPNOTSUPP;
+
+	perf_sample_data_init(&sample_data, 0, 0);
+	sample_data.raw = &raw;
+	perf_event_output(event, &sample_data, regs);
+	return 0;
+}

Note that this function also very much requires event to be local.

kernel/trace/bpf_trace.c +7 -0

+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_CONST_MAP_PTR,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_STACK,
+	.arg5_type	= ARG_CONST_STACK_SIZE,
+};

Quoted 3 lines · Alexei Starovoitov

kernel/trace/bpf_trace.c const struct bpf_func_proto bpf_perf_event_read_proto = { · line 253

Quoted 3 lines

kernel/trace/bpf_trace.c +3 -0

+static const struct bpf_func_proto bpf_perf_event_output_proto = {
+	.func		= bpf_perf_event_output,
+	.gpl_only	= false,

Oh ?

Quoted 5 lines · Peter Zijlstra

Quoted 3 lines · Alexei Starovoitov

kernel/trace/bpf_trace.c const struct bpf_func_proto bpf_perf_event_read_proto = { · line 253

Quoted 3 lines

kernel/trace/bpf_trace.c +3 -0

+static const struct bpf_func_proto bpf_perf_event_output_proto = {
+	.func		= bpf_perf_event_output,
+	.gpl_only	= false,

Oh ?

no particular reason. key helper bpf_probe_read() is gpl, so all bpf for tracing progs have to be gpl.
If you feel strongly about it, I can change it.

Quoted 7 lines · Alexei Starovoitov

Quoted 5 lines · Peter Zijlstra

Quoted 3 lines · Alexei Starovoitov

kernel/trace/bpf_trace.c const struct bpf_func_proto bpf_perf_event_read_proto = { · line 253

Quoted 3 lines

kernel/trace/bpf_trace.c +3 -0

+static const struct bpf_func_proto bpf_perf_event_output_proto = {
+	.func		= bpf_perf_event_output,
+	.gpl_only	= false,

Oh ?

Quoted 3 lines

no particular reason. key helper bpf_probe_read() is gpl, so all bpf for tracing progs have to be gpl.
If you feel strongly about it, I can change it.

All the perf symbols are export GPL, so I suppose this should be true.

Quoted 8 lines · Peter Zijlstra

Quoted 7 lines · Alexei Starovoitov

Quoted 5 lines · Peter Zijlstra

Quoted 3 lines · Alexei Starovoitov

kernel/trace/bpf_trace.c const struct bpf_func_proto bpf_perf_event_read_proto = { · line 253

Quoted 3 lines

kernel/trace/bpf_trace.c +3 -0

+static const struct bpf_func_proto bpf_perf_event_output_proto = {
+    .func        = bpf_perf_event_output,
+    .gpl_only    = false,

Oh ?

Quoted 5 lines

Quoted 3 lines

no particular reason. key helper bpf_probe_read() is gpl, so all bpf for tracing progs have to be gpl.
If you feel strongly about it, I can change it.

All the perf symbols are export GPL, so I suppose this should be true.

ok. will send a patch.

Can we (or have we already) setup some rules for licensing? Which part should be GPL? Who has the response to decide it?

Over the last year there were multiple attempts to let eBPF programs output data into perf events by He Kuang and Wangnan.
The last one was:
https://lkml.org/lkml/2015/7/20/736
It was almost perfect with exception that all bpf programs would sent data into one global perf_event.
This patch set takes different approach by letting user space open independent PERF_COUNT_SW_BPF_OUTPUT events, so that program output won't collide.

Wangnan is working on corresponding perf patches.