bpf: Define new BPF_MAP_TYPE_USER_RINGBUF map type
Subsystems (7)
PATCH v4 1/4 bpf: Define new BPF_MAP_TYPE_USER_RINGBUF map type
We want to support a ringbuf map type where samples are published from user-space, to be consumed by BPF programs. BPF currently supports a kernel -> user-space circular ring buffer via the BPF_MAP_TYPE_RINGBUF map type. We'll need to define a new map type for user-space -> kernel, as none of the helpers exported for BPF_MAP_TYPE_RINGBUF will apply to a user-space producer ring buffer, and we'll want to add one or more helper functions that would not apply for a kernel-producer ring buffer.
This patch therefore adds a new BPF_MAP_TYPE_USER_RINGBUF map type definition. The map type is useless in its current form, as there is no way to access or use it for anything until we one or more BPF helpers. A follow-on patch will therefore add a new helper function that allows BPF programs to run callbacks on samples that are published to the ring buffer.
--- include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/ringbuf.c | 62 +++++++++++++++++-- kernel/bpf/verifier.c | 3 + .../bpf/bpftool/Documentation/bpftool-map.rst | 2 +- tools/bpf/bpftool/map.c | 2 +- tools/include/uapi/linux/bpf.h | 1 + tools/lib/bpf/libbpf.c | 1 + 8 files changed, 65 insertions(+), 8 deletions(-)
include/linux/bpf_types.h +1 -0
@@ -126,6 +126,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops)#endif BPF_MAP_TYPE(BPF_MAP_TYPE_RINGBUF, ringbuf_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_BLOOM_FILTER, bloom_filter_map_ops) +BPF_MAP_TYPE(BPF_MAP_TYPE_USER_RINGBUF, user_ringbuf_map_ops) BPF_LINK_TYPE(BPF_LINK_TYPE_RAW_TRACEPOINT, raw_tracepoint)BPF_LINK_TYPE (BPF_LINK_TYPE_TRACING , tracing)
include/uapi/linux/bpf.h +1 -0
@@ -928,6 +928,7 @@ enum bpf_map_type {BPF_MAP_TYPE_INODE_STORAGE ,BPF_MAP_TYPE_TASK_STORAGE ,BPF_MAP_TYPE_BLOOM_FILTER , + BPF_MAP_TYPE_USER_RINGBUF, };/* Note that tracing related programs such as
kernel/bpf/ringbuf.c +56 -6
92 lines not on this page · raw message
kernel/bpf/verifier.c +3 -0
@@ -6206,6 +6206,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, func_id != BPF_FUNC_ringbuf_discard_dynptr) goto error;break ; +case BPF_MAP_TYPE_USER_RINGBUF : + goto error;case BPF_MAP_TYPE_STACK_TRACE :if (func_id != BPF_FUNC_get_stackid) goto error;
@@ -12705,6 +12707,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, }break ;case BPF_MAP_TYPE_RINGBUF : +case BPF_MAP_TYPE_USER_RINGBUF :case BPF_MAP_TYPE_INODE_STORAGE :case BPF_MAP_TYPE_SK_STORAGE :case BPF_MAP_TYPE_TASK_STORAGE :
tools/bpf/bpftool/Documentation/bpftool-map.rst +1 -1
@@ -55,7 +55,7 @@ MAP COMMANDS | | **devmap** | **devmap_hash** | **sockmap** | **cpumap** | **xskmap** | **sockhash** | | **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage** | | **queue** | **stack** | **sk_storage** | **struct_ops** | **ringbuf** | **inode_storage** -| | **task_storage** | **bloom_filter** } +| | **task_storage** | **bloom_filter** | **user_ringbuf** } DESCRIPTION ===========
tools/bpf/bpftool/map.c +1 -1
@@ -1459,7 +1459,7 @@ static int do_help(int argc, char **argv)" devmap | devmap_hash | sockmap | cpumap | xskmap | sockhash |\n" " cgroup_storage | reuseport_sockarray | percpu_cgroup_storage |\n" " queue | stack | sk_storage | struct_ops | ringbuf | inode_storage |\n" -" task_storage | bloom_filter }\n" +" task_storage | bloom_filter | user_ringbuf }\n" " " HELP_SPEC_OPTIONS " |\n" " {-f|--bpffs} | {-n|--nomount} }\n" "",
tools/include/uapi/linux/bpf.h +1 -0
@@ -928,6 +928,7 @@ enum bpf_map_type {BPF_MAP_TYPE_INODE_STORAGE ,BPF_MAP_TYPE_TASK_STORAGE ,BPF_MAP_TYPE_BLOOM_FILTER , + BPF_MAP_TYPE_USER_RINGBUF, };/* Note that tracing related programs such as
tools/lib/bpf/libbpf.c +1 -0
@@ -163,6 +163,7 @@ static const char * const map_type_name[] = { [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", + [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", };static const char *const prog_type_name[] = {
Signature · 1 line
2.37.1
PATCH v4 2/4 bpf: Add bpf_user_ringbuf_drain() helper
In a prior change, we added a new BPF_MAP_TYPE_USER_RINGBUF map type which will allow user-space applications to publish messages to a ring buffer that is consumed by a BPF program in kernel-space. In order for this map-type to be useful, it will require a BPF helper function that BPF programs can invoke to drain samples from the ring buffer, and invoke callbacks on those samples. This change adds that capability via a new BPF helper function:
bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx,
u64 flags)
BPF programs may invoke this function to run callback_fn() on a series of samples in the ring buffer. callback_fn() has the following signature:
long callback_fn(struct bpf_dynptr *dynptr, void *context);
Samples are provided to the callback in the form of struct bpf_dynptr *'s, which the program can read using BPF helper functions for querying struct bpf_dynptr's.
In order to support bpf_ringbuf_drain(), a new PTR_TO_DYNPTR register type is added to the verifier to reflect a dynptr that was allocated by a helper function and passed to a BPF program. Unlike PTR_TO_STACK dynptrs which are allocated on the stack by a BPF program, PTR_TO_DYNPTR dynptrs need not use reference tracking, as the BPF helper is trusted to properly free the dynptr before returning. The verifier currently only supports PTR_TO_DYNPTR registers that are also DYNPTR_TYPE_LOCAL.
Note that while the corresponding user-space libbpf logic will be added in a subsequent patch, this patch does contain an implementation of the .map_poll() callback for BPF_MAP_TYPE_USER_RINGBUF maps. This .map_poll() callback guarantees that an epoll-waiting user-space producer will receive at least one event notification whenever at least one sample is drained in an invocation of bpf_user_ringbuf_drain(), provided that the function is not invoked with the BPF_RB_NO_WAKEUP flag.
Sending an event notification for every sample is not an option, as it could cause the system to hang due to invoking irq_work_queue() in too-frequent succession. So as to try and optimize for the common case, however, bpf_user_ringbuf_drain() will also send an event notification whenever a sample being drained causes the ring buffer to no longer be full. This heuristic may not help some user-space producers, as a producer can publish samples of varying size, and there may not be enough space in the ring buffer after the first sample is drained which causes it to no longer be full. In this case, the producer may have to wait until bpf_ringbuf_drain() returns to receive an event notification.
--- include/linux/bpf.h | 11 +- include/uapi/linux/bpf.h | 36 ++++++ kernel/bpf/helpers.c | 2 + kernel/bpf/ringbuf.c | 201 ++++++++++++++++++++++++++++++++- kernel/bpf/verifier.c | 61 +++++++++- tools/include/uapi/linux/bpf.h | 36 ++++++ 6 files changed, 335 insertions(+), 12 deletions(-)
include/linux/bpf.h +9 -2
@@ -402,7 +402,7 @@ enum bpf_type_flag {/* DYNPTR points to memory local to the bpf program. */ DYNPTR_TYPE_LOCAL =BIT (8 +BPF_BASE_TYPE_BITS ), -/* DYNPTR points to a ringbuf record. */ +/* DYNPTR points to a kernel-produced ringbuf record. */ DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),/* Size is known at compile time. */
@@ -607,6 +607,7 @@ enum bpf_reg_type {PTR_TO_MEM ,/* reg points to valid memory region */ PTR_TO_BUF ,/* reg points to a read/write buffer */ PTR_TO_FUNC ,/* reg points to a bpf program function */ +PTR_TO_DYNPTR ,/* reg points to a dynptr */ __BPF_REG_TYPE_MAX,/* Extended reg_types. */
@@ -1334,6 +1335,11 @@ struct bpf_array {#define BPF_MAP_CAN_READ BIT(0)#define BPF_MAP_CAN_WRITE BIT(1) +/* Maximum number of user-producer ringbuffer samples that can be drained in +* a call to bpf_user_ringbuf_drain(). +*/ +#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024) +static inline u32 bpf_map_flags_to_cap (struct bpf_map *map) {u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG |BPF_F_WRONLY_PROG );
@@ -2433,6 +2439,7 @@ extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_copy_from_user_task_proto;extern const struct bpf_func_proto bpf_set_retval_proto;extern const struct bpf_func_proto bpf_get_retval_proto; +extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
@@ -2577,7 +2584,7 @@ enum bpf_dynptr_type {BPF_DYNPTR_TYPE_INVALID ,/* Points to memory that is local to the bpf program */ BPF_DYNPTR_TYPE_LOCAL , -/* Underlying data is a ringbuf record */ +/* Underlying data is a kernel-produced ringbuf record */ BPF_DYNPTR_TYPE_RINGBUF, };
include/uapi/linux/bpf.h +36 -0
@@ -5387,6 +5387,41 @@ union bpf_attr {* Return * Current *ktime*. * +* long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags) +* Description +* Drain samples from the specified user ring buffer, and invoke +* the provided callback for each such sample: +* +* long (\*callback_fn)(struct bpf_dynptr \*dynptr, void \*ctx); +* +* If **callback_fn** returns 0, the helper will continue to try +* and drain the next sample, up to a maximum of +* BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, +* the helper will skip the rest of the samples and return. Other +* return values are not used now, and will be rejected by the +* verifier. +* Return +* The number of drained samples if no error was encountered while +* draining samples. If a user-space producer was epoll-waiting on +* this map, and at least one sample was drained, they will +* receive an event notification notifying them of available space +* in the ring buffer. If the BPF_RB_NO_WAKEUP flag is passed to +* this function, no wakeup notification will be sent. If there +* are no samples in the ring buffer, 0 is returned. +* +* On failure, the returned value is one of the following: +* +* **-EBUSY** if the ring buffer is contended, and another calling +* context was concurrently draining the ring buffer. +* +* **-EINVAL** if user-space is not properly tracking the ring +* buffer due to the producer position not being aligned to 8 +* bytes, a sample not being aligned to 8 bytes, or the producer +* position not matching the advertised length of a sample. +* +* **-E2BIG** if user-space has tried to publish a sample which is +* larger than the size of the ring buffer, or which cannot fit +* within a struct bpf_dynptr. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5598,6 +5633,7 @@ union bpf_attr {FN (tcp_raw_check_syncookie_ipv4), \FN (tcp_raw_check_syncookie_ipv6), \FN (ktime_get_tai_ns), \ +FN (user_ringbuf_drain), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +2 -0
@@ -1627,6 +1627,8 @@ bpf_base_func_proto(enum bpf_func_id func_id)return &bpf_dynptr_write_proto;case BPF_FUNC_dynptr_data:return &bpf_dynptr_data_proto; +case BPF_FUNC_user_ringbuf_drain: +return &bpf_user_ringbuf_drain_proto;default :break ; }
kernel/bpf/ringbuf.c +195 -6
258 lines not on this page · raw message
kernel/bpf/verifier.c +57 -4
119 lines not on this page · raw message
tools/include/uapi/linux/bpf.h +36 -0
@@ -5387,6 +5387,41 @@ union bpf_attr {* Return * Current *ktime*. * +* long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags) +* Description +* Drain samples from the specified user ring buffer, and invoke +* the provided callback for each such sample: +* +* long (\*callback_fn)(struct bpf_dynptr \*dynptr, void \*ctx); +* +* If **callback_fn** returns 0, the helper will continue to try +* and drain the next sample, up to a maximum of +* BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, +* the helper will skip the rest of the samples and return. Other +* return values are not used now, and will be rejected by the +* verifier. +* Return +* The number of drained samples if no error was encountered while +* draining samples. If a user-space producer was epoll-waiting on +* this map, and at least one sample was drained, they will +* receive an event notification notifying them of available space +* in the ring buffer. If the BPF_RB_NO_WAKEUP flag is passed to +* this function, no wakeup notification will be sent. If there +* are no samples in the ring buffer, 0 is returned. +* +* On failure, the returned value is one of the following: +* +* **-EBUSY** if the ring buffer is contended, and another calling +* context was concurrently draining the ring buffer. +* +* **-EINVAL** if user-space is not properly tracking the ring +* buffer due to the producer position not being aligned to 8 +* bytes, a sample not being aligned to 8 bytes, or the producer +* position not matching the advertised length of a sample. +* +* **-E2BIG** if user-space has tried to publish a sample which is +* larger than the size of the ring buffer, or which cannot fit +* within a struct bpf_dynptr. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5598,6 +5633,7 @@ union bpf_attr {FN (tcp_raw_check_syncookie_ipv4), \FN (tcp_raw_check_syncookie_ipv6), \FN (ktime_get_tai_ns), \ +FN (user_ringbuf_drain), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.37.1
PATCH v4 3/4 bpf: Add libbpf logic for user-space ring buffer
In reply to David VernetNow that all of the logic is in place in the kernel to support user-space produced ring buffers, we can add the user-space logic to libbpf. This patch therefore adds the following public symbols to libbpf:
struct user_ring_buffer *
user_ring_buffer__new(int map_fd,
const struct user_ring_buffer_opts *opts);
void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size);
void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb,
__u32 size, int timeout_ms);void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample); void user_ring_buffer__discard(struct user_ring_buffer *rb, void user_ring_buffer__free(struct user_ring_buffer *rb);
A user-space producer must first create a struct user_ring_buffer * object with user_ring_buffer__new(), and can then reserve samples in the ring buffer using one of the following two symbols:
void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size); void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb,
__u32 size, int timeout_ms);
With user_ring_buffer__reserve(), a pointer to a 'size' region of the ring buffer will be returned if sufficient space is available in the buffer. user_ring_buffer__reserve_blocking() provides similar semantics, but will block for up to 'timeout_ms' in epoll_wait if there is insufficient space in the buffer. This function has the guarantee from the kernel that it will receive at least one event-notification per invocation to bpf_ringbuf_drain(), provided that at least one sample is drained, and the BPF program did not pass the BPF_RB_NO_WAKEUP flag to bpf_ringbuf_drain().
Once a sample is reserved, it must either be committed to the ring buffer with user_ring_buffer__submit(), or discarded with user_ring_buffer__discard().
--- tools/lib/bpf/libbpf.c | 10 +- tools/lib/bpf/libbpf.h | 105 +++++++++++++ tools/lib/bpf/libbpf.map | 10 ++ tools/lib/bpf/libbpf_probes.c | 1 + tools/lib/bpf/libbpf_version.h | 2 +- tools/lib/bpf/ringbuf.c | 270 +++++++++++++++++++++++++++++++++ 6 files changed, 395 insertions(+), 3 deletions(-)
tools/lib/bpf/libbpf.c +8 -2
@@ -2373,6 +2373,12 @@ static size_t adjust_ringbuf_sz(size_t sz)return sz; } +static bool map_is_ringbuf (const struct bpf_map *map) +{ +return map->def.type ==BPF_MAP_TYPE_RINGBUF || + map->def.type ==BPF_MAP_TYPE_USER_RINGBUF ; +} +static void fill_map_from_def (struct bpf_map *map,const struct btf_map_def *def) { map->def.type = def->map_type;
@@ -2387,7 +2393,7 @@ static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def map->btf_value_type_id = def->value_type_id;/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ -if (map->def.type ==BPF_MAP_TYPE_RINGBUF ) +if (map_is_ringbuf (map)) map->def.max_entries =adjust_ringbuf_sz (map->def.max_entries);if (def->parts &MAP_DEF_MAP_TYPE )
@@ -4370,7 +4376,7 @@ int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) map->def.max_entries = max_entries;/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ -if (map->def.type ==BPF_MAP_TYPE_RINGBUF ) +if (map_is_ringbuf (map)) map->def.max_entries =adjust_ringbuf_sz (map->def.max_entries);return 0 ;
tools/lib/bpf/libbpf.h +105 -0
117 lines not on this page · raw message
tools/lib/bpf/libbpf.map +10 -0
@@ -368,3 +368,13 @@ LIBBPF_1.0.0 { libbpf_bpf_prog_type_str; perf_buffer__buffer; }; + +LIBBPF_1.1.0 { + global: + user_ring_buffer__discard; + user_ring_buffer__free; + user_ring_buffer__new; + user_ring_buffer__reserve; + user_ring_buffer__reserve_blocking; + user_ring_buffer__submit; +} LIBBPF_1.0.0;
tools/lib/bpf/libbpf_probes.c +1 -0
@@ -231,6 +231,7 @@ static int probe_map_create(enum bpf_map_type map_type)return btf_fd;break ;case BPF_MAP_TYPE_RINGBUF : +case BPF_MAP_TYPE_USER_RINGBUF : key_size =0 ; value_size =0 ; max_entries =4096 ;
tools/lib/bpf/libbpf_version.h +1 -1
@@ -4,6 +4,6 @@#define __LIBBPF_VERSION_H#define LIBBPF_MAJOR_VERSION 1 -#define LIBBPF_MINOR_VERSION 0 +#define LIBBPF_MINOR_VERSION 1#endif /* __LIBBPF_VERSION_H */
tools/lib/bpf/ringbuf.c +270 -0
285 lines not on this page · raw message
Signature · 1 line
2.37.1
PATCH v4 4/4 selftests/bpf: Add selftests validating the user ringbuf
In reply to David VernetThis change includes selftests that validate the expected behavior and APIs of the new BPF_MAP_TYPE_USER_RINGBUF map type.
--- .../selftests/bpf/prog_tests/user_ringbuf.c | 752 ++++++++++++++++++ .../selftests/bpf/progs/test_user_ringbuf.h | 35 + .../selftests/bpf/progs/user_ringbuf_fail.c | 177 +++++ .../bpf/progs/user_ringbuf_success.c | 218 +++++ 4 files changed, 1182 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/user_ringbuf.c create mode 100644 tools/testing/selftests/bpf/progs/test_user_ringbuf.h create mode 100644 tools/testing/selftests/bpf/progs/user_ringbuf_fail.c create mode 100644 tools/testing/selftests/bpf/progs/user_ringbuf_success.c
tools/testing/selftests/bpf/prog_tests/user_ringbuf.c new file +752 -0
@@ -0,0 +1,752 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#define _GNU_SOURCE +#include <linux/compiler.h> +#include <linux/ring_buffer.h> +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/sysinfo.h> +#include <test_progs.h> +#include <uapi/linux/bpf.h> +#include <unistd.h> + +#include "user_ringbuf_fail.skel.h" +#include "user_ringbuf_success.skel.h" + +#include "../progs/test_user_ringbuf.h" + +static size_t log_buf_sz =1 <<20 ;/* 1 MB */ +static char obj_log_buf[1048576 ]; +static const long c_sample_size =sizeof (struct sample ) +BPF_RINGBUF_HDR_SZ ; +static const long c_ringbuf_size =1 <<12 ;/* 1 small page */ +static const long c_max_entries = c_ringbuf_size / c_sample_size; + +static void drain_current_samples (void ) +{ +syscall (__NR_getpgid); +} + +static int write_samples (struct user_ring_buffer *ringbuf,uint32_t num_samples) +{ +int i, err =0 ; + +/* Write some number of samples to the ring buffer. */ +for (i =0 ; i < num_samples; i++) { +struct sample *entry; +int read; + + entry =user_ring_buffer__reserve (ringbuf,sizeof (*entry)); +if (!entry) { + err = -errno; + goto done; + } + + entry->pid =getpid (); + entry->seq = i; + entry->value = i * i; + + read =snprintf (entry->comm,sizeof (entry->comm),"%u" , i); +if (read <=0 ) { +/* Assert on the error path to avoid spamming logs with +* mostly success messages. +*/ +ASSERT_GT (read,0 ,"snprintf_comm" ); + err = read; +user_ring_buffer__discard (ringbuf, entry); + goto done; + } + +user_ring_buffer__submit (ringbuf, entry); + } + +done: +drain_current_samples (); + +return err; +} + +static struct user_ringbuf_success *open_load_ringbuf_skel (void ) +{ +struct user_ringbuf_success *skel; +int err; + + skel =user_ringbuf_success__open (); +if (!ASSERT_OK_PTR (skel,"skel_open" )) +return NULL ; + + err =bpf_map__set_max_entries (skel->maps.user_ringbuf, c_ringbuf_size); +if (!ASSERT_OK (err,"set_max_entries" )) + goto cleanup; + + err =bpf_map__set_max_entries (skel->maps.kernel_ringbuf, c_ringbuf_size); +if (!ASSERT_OK (err,"set_max_entries" )) + goto cleanup; + + err =user_ringbuf_success__load (skel); +if (!ASSERT_OK (err,"skel_load" )) + goto cleanup; + +return skel; + +cleanup: +user_ringbuf_success__destroy (skel); +return NULL ; +} + +static void test_user_ringbuf_mappings (void ) +{ +int err, rb_fd; +int page_size =getpagesize (); +void *mmap_ptr; +struct user_ringbuf_success *skel; + + skel =open_load_ringbuf_skel (); +if (!skel) +return ; + + rb_fd =bpf_map__fd (skel->maps.user_ringbuf); +/* cons_pos can be mapped R/O, can't add +X with mprotect. */ + mmap_ptr =mmap (NULL , page_size,PROT_READ ,MAP_SHARED , rb_fd,0 ); +ASSERT_OK_PTR (mmap_ptr,"ro_cons_pos" ); +ASSERT_ERR (mprotect (mmap_ptr, page_size,PROT_WRITE ),"write_cons_pos_protect" ); +ASSERT_ERR (mprotect (mmap_ptr, page_size,PROT_EXEC ),"exec_cons_pos_protect" ); +ASSERT_ERR_PTR (mremap (mmap_ptr,0 ,4 * page_size,MREMAP_MAYMOVE ),"wr_prod_pos" ); + err = -errno; +ASSERT_ERR (err,"wr_prod_pos_err" ); +ASSERT_OK (munmap (mmap_ptr, page_size),"unmap_ro_cons" ); + +/* prod_pos can be mapped RW, can't add +X with mprotect. */ + mmap_ptr =mmap (NULL , page_size,PROT_READ |PROT_WRITE ,MAP_SHARED , + rb_fd, page_size); +ASSERT_OK_PTR (mmap_ptr,"rw_prod_pos" ); +ASSERT_ERR (mprotect (mmap_ptr, page_size,PROT_EXEC ),"exec_prod_pos_protect" ); + err = -errno; +ASSERT_ERR (err,"wr_prod_pos_err" ); +ASSERT_OK (munmap (mmap_ptr, page_size),"unmap_rw_prod" ); + +/* data pages can be mapped RW, can't add +X with mprotect. */ + mmap_ptr =mmap (NULL , page_size,PROT_WRITE ,MAP_SHARED , rb_fd, +2 * page_size); +ASSERT_OK_PTR (mmap_ptr,"rw_data" ); +ASSERT_ERR (mprotect (mmap_ptr, page_size,PROT_EXEC ),"exec_data_protect" ); + err = -errno; +ASSERT_ERR (err,"exec_data_err" ); +ASSERT_OK (munmap (mmap_ptr, page_size),"unmap_rw_data" ); + +user_ringbuf_success__destroy (skel); +} + +static int load_skel_create_ringbufs (struct user_ringbuf_success **skel_out, +struct ring_buffer **kern_ringbuf_out, +ring_buffer_sample_fn callback, +struct user_ring_buffer **user_ringbuf_out) +{ +struct user_ringbuf_success *skel; +struct ring_buffer *kern_ringbuf =NULL ; +struct user_ring_buffer *user_ringbuf =NULL ; +int err = -ENOMEM , rb_fd; + + skel =open_load_ringbuf_skel (); +if (!skel) +return err; + +/* only trigger BPF program for current process */ + skel->bss->pid =getpid (); + +if (kern_ringbuf_out) { + rb_fd =bpf_map__fd (skel->maps.kernel_ringbuf); + kern_ringbuf =ring_buffer__new (rb_fd, callback, skel,NULL ); +if (!ASSERT_OK_PTR (kern_ringbuf,"kern_ringbuf_create" )) + goto cleanup; + + *kern_ringbuf_out = kern_ringbuf; + } + +if (user_ringbuf_out) { + rb_fd =bpf_map__fd (skel->maps.user_ringbuf); + user_ringbuf =user_ring_buffer__new (rb_fd,NULL ); +if (!ASSERT_OK_PTR (user_ringbuf,"user_ringbuf_create" )) + goto cleanup; + + *user_ringbuf_out = user_ringbuf; +ASSERT_EQ (skel->bss->read,0 ,"no_reads_after_load" ); + } + + err =user_ringbuf_success__attach (skel); +if (!ASSERT_OK (err,"skel_attach" )) + goto cleanup; + + *skel_out = skel; +return 0 ; + +cleanup: +if (kern_ringbuf_out) + *kern_ringbuf_out =NULL ; +if (user_ringbuf_out) + *user_ringbuf_out =NULL ; +ring_buffer__free (kern_ringbuf); +user_ring_buffer__free (user_ringbuf); +user_ringbuf_success__destroy (skel); +return err; +} + +static int load_skel_create_user_ringbuf (struct user_ringbuf_success **skel_out, +struct user_ring_buffer **ringbuf_out) +{ +return load_skel_create_ringbufs (skel_out,NULL ,NULL , ringbuf_out); +} + +static void manually_write_test_invalid_sample (struct user_ringbuf_success *skel, +__u32 size,__u64 producer_pos,int err) +{ +void *data_ptr; +__u64 *producer_pos_ptr; +int rb_fd, page_size =getpagesize (); + + rb_fd =bpf_map__fd (skel->maps.user_ringbuf); + +ASSERT_EQ (skel->bss->read,0 ,"num_samples_before_bad_sample" ); + +/* Map the producer_pos as RW. */ + producer_pos_ptr =mmap (NULL , page_size,PROT_READ |PROT_WRITE , +MAP_SHARED , rb_fd, page_size); +ASSERT_OK_PTR (producer_pos_ptr,"producer_pos_ptr" ); + +/* Map the data pages as RW. */ + data_ptr =mmap (NULL , page_size,PROT_WRITE ,MAP_SHARED , rb_fd,2 * page_size); +ASSERT_OK_PTR (data_ptr,"rw_data" ); + +memset (data_ptr,0 ,BPF_RINGBUF_HDR_SZ ); + *(__u32 *)data_ptr = size; + +/* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in the kernel. */ +smp_store_release (producer_pos_ptr, producer_pos +BPF_RINGBUF_HDR_SZ ); + +drain_current_samples (); +ASSERT_EQ (skel->bss->read,0 ,"num_samples_after_bad_sample" ); +ASSERT_EQ (skel->bss->err, err,"err_after_bad_sample" ); + +ASSERT_OK (munmap (producer_pos_ptr, page_size),"unmap_producer_pos" ); +ASSERT_OK (munmap (data_ptr, page_size),"unmap_data_ptr" ); +} + +static void test_user_ringbuf_post_misaligned (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err; +__u32 size = (1 <<5 ) +7 ; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (!ASSERT_OK (err,"misaligned_skel" )) +return ; + +manually_write_test_invalid_sample (skel, size, size, -EINVAL ); +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_post_producer_wrong_offset (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err; +__u32 size = (1 <<5 ); + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (!ASSERT_OK (err,"wrong_offset_skel" )) +return ; + +manually_write_test_invalid_sample (skel, size, size -8 , -EINVAL ); +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_post_larger_than_ringbuf_sz (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err; +__u32 size = c_ringbuf_size; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (!ASSERT_OK (err,"huge_sample_skel" )) +return ; + +manually_write_test_invalid_sample (skel, size, size, -E2BIG ); +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_basic (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (!ASSERT_OK (err,"ringbuf_basic_skel" )) +return ; + +ASSERT_EQ (skel->bss->read,0 ,"num_samples_read_before" ); + + err =write_samples (ringbuf,2 ); +if (!ASSERT_OK (err,"write_samples" )) + goto cleanup; + +ASSERT_EQ (skel->bss->read,2 ,"num_samples_read_after" ); + +cleanup: +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_sample_full_ringbuffer (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err; +void *sample; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (!ASSERT_OK (err,"ringbuf_full_sample_skel" )) +return ; + + sample =user_ring_buffer__reserve (ringbuf, c_ringbuf_size -BPF_RINGBUF_HDR_SZ ); +if (!ASSERT_OK_PTR (sample,"full_sample" )) + goto cleanup; + +user_ring_buffer__submit (ringbuf, sample); +ASSERT_EQ (skel->bss->read,0 ,"num_samples_read_before" ); +drain_current_samples (); +ASSERT_EQ (skel->bss->read,1 ,"num_samples_read_after" ); + +cleanup: +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_post_alignment_autoadjust (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +struct sample *sample; +int err; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (!ASSERT_OK (err,"ringbuf_align_autoadjust_skel" )) +return ; + +/* libbpf should automatically round any sample up to an 8-byte alignment. */ + sample =user_ring_buffer__reserve (ringbuf,sizeof (*sample) +1 ); +ASSERT_OK_PTR (sample,"reserve_autoaligned" ); +user_ring_buffer__submit (ringbuf, sample); + +ASSERT_EQ (skel->bss->read,0 ,"num_samples_read_before" ); +drain_current_samples (); +ASSERT_EQ (skel->bss->read,1 ,"num_samples_read_after" ); + +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_overfill (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (err) +return ; + + err =write_samples (ringbuf, c_max_entries *5 ); +ASSERT_ERR (err,"write_samples" ); +ASSERT_EQ (skel->bss->read, c_max_entries,"max_entries" ); + +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_discards_properly_ignored (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err, num_discarded =0 ; +__u64 *token; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (err) +return ; + +ASSERT_EQ (skel->bss->read,0 ,"num_samples_read_before" ); + +while (1 ) { +/* Write samples until the buffer is full. */ + token =user_ring_buffer__reserve (ringbuf,sizeof (*token)); +if (!token) +break ; + +user_ring_buffer__discard (ringbuf, token); + num_discarded++; + } + +if (!ASSERT_GE (num_discarded,0 ,"num_discarded" )) + goto cleanup; + +/* Should not read any samples, as they are all discarded. */ +ASSERT_EQ (skel->bss->read,0 ,"num_pre_kick" ); +drain_current_samples (); +ASSERT_EQ (skel->bss->read,0 ,"num_post_kick" ); + +/* Now that the ringbuffer has been drained, we should be able to reserve another token. */ + token =user_ring_buffer__reserve (ringbuf,sizeof (*token)); + +if (!ASSERT_OK_PTR (token,"new_token" )) + goto cleanup; + +user_ring_buffer__discard (ringbuf, token); +cleanup: +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void test_user_ringbuf_loop (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +uint32_t total_samples =8192 ; +uint32_t remaining_samples = total_samples; +int err; + +BUILD_BUG_ON (total_samples <= c_max_entries); + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (err) +return ; + +do { +uint32_t curr_samples; + + curr_samples = remaining_samples > c_max_entries + ? c_max_entries : remaining_samples; + err =write_samples (ringbuf, curr_samples); +if (err !=0 ) { +/* Assert inside of if statement to avoid flooding logs +* on the success path. +*/ +ASSERT_OK (err,"write_samples" ); + goto cleanup; + } + + remaining_samples -= curr_samples; +ASSERT_EQ (skel->bss->read, total_samples - remaining_samples, +"current_batched_entries" ); + }while (remaining_samples >0 ); +ASSERT_EQ (skel->bss->read, total_samples,"total_batched_entries" ); + +cleanup: +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static int send_test_message (struct user_ring_buffer *ringbuf, +enum test_msg_op op,s64 operand_64, +s32 operand_32) +{ +struct test_msg *msg; + + msg =user_ring_buffer__reserve (ringbuf,sizeof (*msg)); +if (!msg) { +/* Assert on the error path to avoid spamming logs with mostly +* success messages. +*/ +ASSERT_OK_PTR (msg,"reserve_msg" ); +return -ENOMEM ; + } + + msg->msg_op = op; + +switch (op) { +case TEST_MSG_OP_INC64 : +case TEST_MSG_OP_MUL64 : + msg->operand_64 = operand_64; +break ; +case TEST_MSG_OP_INC32 : +case TEST_MSG_OP_MUL32 : + msg->operand_32 = operand_32; +break ; +default : +PRINT_FAIL ("Invalid operand %d\n" , op); +user_ring_buffer__discard (ringbuf, msg); +return -EINVAL ; + } + +user_ring_buffer__submit (ringbuf, msg); + +return 0 ; +} + +static void kick_kernel_read_messages (void ) +{ +syscall (__NR_prctl); +} + +static int handle_kernel_msg (void *ctx,void *data,size_t len) +{ +struct user_ringbuf_success *skel = ctx; +struct test_msg *msg = data; + +switch (msg->msg_op) { +case TEST_MSG_OP_INC64 : + skel->bss->user_mutated += msg->operand_64; +return 0 ; +case TEST_MSG_OP_INC32 : + skel->bss->user_mutated += msg->operand_32; +return 0 ; +case TEST_MSG_OP_MUL64 : + skel->bss->user_mutated *= msg->operand_64; +return 0 ; +case TEST_MSG_OP_MUL32 : + skel->bss->user_mutated *= msg->operand_32; +return 0 ; +default : +fprintf (stderr,"Invalid operand %d\n" , msg->msg_op); +return -EINVAL ; + } +} + +static void drain_kernel_messages_buffer (struct ring_buffer *kern_ringbuf, +struct user_ringbuf_success *skel) +{ +int cnt; + + cnt =ring_buffer__consume (kern_ringbuf); +ASSERT_EQ (cnt,8 ,"consume_kern_ringbuf" ); +ASSERT_OK (skel->bss->err,"consume_kern_ringbuf_err" ); +} + +static void test_user_ringbuf_msg_protocol (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *user_ringbuf; +struct ring_buffer *kern_ringbuf; +int err, i; +__u64 expected_kern =0 ; + + err =load_skel_create_ringbufs (&skel, &kern_ringbuf, handle_kernel_msg, &user_ringbuf); +if (!ASSERT_OK (err,"create_ringbufs" )) +return ; + +for (i =0 ; i <64 ; i++) { +enum test_msg_op op = i %TEST_MSG_OP_NUM_OPS ; +__u64 operand_64 =TEST_OP_64 ; +__u32 operand_32 =TEST_OP_32 ; + + err =send_test_message (user_ringbuf, op, operand_64, operand_32); +if (err) { +/* Only assert on a failure to avoid spamming success logs. */ +ASSERT_OK (err,"send_test_message" ); + goto cleanup; + } + +switch (op) { +case TEST_MSG_OP_INC64 : + expected_kern += operand_64; +break ; +case TEST_MSG_OP_INC32 : + expected_kern += operand_32; +break ; +case TEST_MSG_OP_MUL64 : + expected_kern *= operand_64; +break ; +case TEST_MSG_OP_MUL32 : + expected_kern *= operand_32; +break ; +default : +PRINT_FAIL ("Unexpected op %d\n" , op); + goto cleanup; + } + +if (i %8 ==0 ) { +kick_kernel_read_messages (); +ASSERT_EQ (skel->bss->kern_mutated, expected_kern,"expected_kern" ); +ASSERT_EQ (skel->bss->err,0 ,"bpf_prog_err" ); +drain_kernel_messages_buffer (kern_ringbuf, skel); + } + } + +cleanup: +ring_buffer__free (kern_ringbuf); +user_ring_buffer__free (user_ringbuf); +user_ringbuf_success__destroy (skel); +} + +static void *kick_kernel_cb (void *arg) +{ +/* Kick the kernel, causing it to drain the ringbuffer and then wake up +* the test thread waiting on epoll. +*/ +syscall (__NR_getrlimit); + +return NULL ; +} + +static int spawn_kick_thread_for_poll (void ) +{ +pthread_t thread; + +return pthread_create (&thread,NULL , kick_kernel_cb,NULL ); +} + +static void test_user_ringbuf_blocking_reserve (void ) +{ +struct user_ringbuf_success *skel; +struct user_ring_buffer *ringbuf; +int err, num_written =0 ; +__u64 *token; + + err =load_skel_create_user_ringbuf (&skel, &ringbuf); +if (err) +return ; + +ASSERT_EQ (skel->bss->read,0 ,"num_samples_read_before" ); + +while (1 ) { +/* Write samples until the buffer is full. */ + token =user_ring_buffer__reserve (ringbuf,sizeof (*token)); +if (!token) +break ; + + *token =0xdeadbeef ; + +user_ring_buffer__submit (ringbuf, token); + num_written++; + } + +if (!ASSERT_GE (num_written,0 ,"num_written" )) + goto cleanup; + +/* Should not have read any samples until the kernel is kicked. */ +ASSERT_EQ (skel->bss->read,0 ,"num_pre_kick" ); + +/* We correctly time out after 1 second, without a sample. */ + token =user_ring_buffer__reserve_blocking (ringbuf,sizeof (*token),1000 ); +if (!ASSERT_EQ (token,NULL ,"pre_kick_timeout_token" )) + goto cleanup; + + err =spawn_kick_thread_for_poll (); +if (!ASSERT_EQ (err,0 ,"deferred_kick_thread\n" )) + goto cleanup; + +/* After spawning another thread that asychronously kicks the kernel to +* drain the messages, we're able to block and successfully get a +* sample once we receive an event notification. +*/ + token =user_ring_buffer__reserve_blocking (ringbuf,sizeof (*token),10000 ); + +if (!ASSERT_OK_PTR (token,"block_token" )) + goto cleanup; + +ASSERT_GT (skel->bss->read,0 ,"num_post_kill" ); +ASSERT_LE (skel->bss->read, num_written,"num_post_kill" ); +ASSERT_EQ (skel->bss->err,0 ,"err_post_poll" ); +user_ring_buffer__discard (ringbuf, token); + +cleanup: +user_ring_buffer__free (ringbuf); +user_ringbuf_success__destroy (skel); +} + +static struct { +const char *prog_name; +const char *expected_err_msg; +} failure_tests[] = { +/* failure cases */ + {"user_ringbuf_callback_bad_access1" ,"negative offset dynptr_ptr ptr" }, + {"user_ringbuf_callback_bad_access2" ,"dereference of modified dynptr_ptr ptr" }, + {"user_ringbuf_callback_write_forbidden" ,"invalid mem access 'dynptr_ptr'" }, + {"user_ringbuf_callback_null_context_write" ,"invalid mem access 'scalar'" }, + {"user_ringbuf_callback_null_context_read" ,"invalid mem access 'scalar'" }, + {"user_ringbuf_callback_discard_dynptr" ,"arg 1 is an unacquired reference" }, + {"user_ringbuf_callback_submit_dynptr" ,"arg 1 is an unacquired reference" }, + {"user_ringbuf_callback_invalid_return" ,"At callback return the register R0 has value" }, +}; + +#define SUCCESS_TEST (_func) { _func, #_func } + +static struct { +void (*test_callback)(void ); +const char *test_name; +} success_tests[] = { +SUCCESS_TEST (test_user_ringbuf_mappings), +SUCCESS_TEST (test_user_ringbuf_post_misaligned), +SUCCESS_TEST (test_user_ringbuf_post_producer_wrong_offset), +SUCCESS_TEST (test_user_ringbuf_post_larger_than_ringbuf_sz), +SUCCESS_TEST (test_user_ringbuf_basic), +SUCCESS_TEST (test_user_ringbuf_sample_full_ringbuffer), +SUCCESS_TEST (test_user_ringbuf_post_alignment_autoadjust), +SUCCESS_TEST (test_user_ringbuf_overfill), +SUCCESS_TEST (test_user_ringbuf_discards_properly_ignored), +SUCCESS_TEST (test_user_ringbuf_loop), +SUCCESS_TEST (test_user_ringbuf_msg_protocol), +SUCCESS_TEST (test_user_ringbuf_blocking_reserve), +}; + +static void verify_fail (const char *prog_name,const char *expected_err_msg) +{ +LIBBPF_OPTS (bpf_object_open_opts, opts); +struct bpf_program *prog; +struct user_ringbuf_fail *skel; +int err; + + opts.kernel_log_buf = obj_log_buf; + opts.kernel_log_size = log_buf_sz; + opts.kernel_log_level =1 ; + + skel =user_ringbuf_fail__open_opts (&opts); +if (!ASSERT_OK_PTR (skel,"dynptr_fail__open_opts" )) + goto cleanup; + + prog =bpf_object__find_program_by_name (skel->obj, prog_name); +if (!ASSERT_OK_PTR (prog,"bpf_object__find_program_by_name" )) + goto cleanup; + +bpf_program__set_autoload (prog, true); + +bpf_map__set_max_entries (skel->maps.user_ringbuf,getpagesize ()); + + err =user_ringbuf_fail__load (skel); +if (!ASSERT_ERR (err,"unexpected load success" )) + goto cleanup; + +if (!ASSERT_OK_PTR (strstr (obj_log_buf, expected_err_msg),"expected_err_msg" )) { +fprintf (stderr,"Expected err_msg: %s\n" , expected_err_msg); +fprintf (stderr,"Verifier output: %s\n" , obj_log_buf); + } + +cleanup: +user_ringbuf_fail__destroy (skel); +} + +void test_user_ringbuf (void ) +{ +int i; + +for (i =0 ; i <ARRAY_SIZE (success_tests); i++) { +if (!test__start_subtest (success_tests[i].test_name)) +continue ; + + success_tests[i].test_callback (); + } + +for (i =0 ; i <ARRAY_SIZE (failure_tests); i++) { +if (!test__start_subtest (failure_tests[i].prog_name)) +continue ; + +verify_fail (failure_tests[i].prog_name, failure_tests[i].expected_err_msg); + } +}
tools/testing/selftests/bpf/progs/test_user_ringbuf.h new file +35 -0
@@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#ifndef _TEST_USER_RINGBUF_H +#define _TEST_USER_RINGBUF_H + +#define TEST_OP_64 4 +#define TEST_OP_32 2 + +enum test_msg_op { +TEST_MSG_OP_INC64 , +TEST_MSG_OP_INC32 , +TEST_MSG_OP_MUL64 , +TEST_MSG_OP_MUL32 , + +// Must come last. +TEST_MSG_OP_NUM_OPS , +}; + +struct test_msg { +enum test_msg_op msg_op; +union { +__s64 operand_64; +__s32 operand_32; + }; +}; + +struct sample { +int pid; +int seq; +long value; +char comm[16 ]; +}; + +#endif /* _TEST_USER_RINGBUF_H */
tools/testing/selftests/bpf/progs/user_ringbuf_fail.c new file +177 -0
177 lines not on this page · raw message
tools/testing/selftests/bpf/progs/user_ringbuf_success.c new file +218 -0
218 lines not on this page · raw message
Signature · 1 line
2.37.1