io_uring: add IORING_OP_BPF for extending io_uring
v3 cover io_uring: add IORING_OP_BPF for extending io_uring
Hello,
Add IORING_OP_BPF for extending io_uring operations, follows typical cases:
- buffer registered zero copy [1]
Also there are some RAID like ublk servers which needs to generate data parity in case of ublk zero copy
- extend io_uring operations from application
Easy to add one new syscall with IORING_OP_BPF
- extend 64 byte SQE
bpf map can store IO data conveniently
- communicate in IO chain
IORING_OP_BPF can be used for communicate among IOs seamlessly without requiring extra syscall
- pretty handy to inject error for test purpose
Any comments & feedback are welcome!
[1] lpc2024: ublk based zero copy I/O - use case in Android
https://lpc.events/event/18/contributions/1710/attachments/1440/3070/LPC2024_ublk_zero_copy.pdf
V3:
- add per-buffer iterator kfuncs for addressing Caleb's concern about complicated kfunc interface, kernel mapped buffer produced by the iterator can be read/written from bpf prog in zero copy style
- rename to bpf_ext
- change tests to copy between uring_buf and arena, which may match with raid's use case
- misc cleanup
V2:
- per-ring struct ops (Stefan Metzmacher, Caleb Sander Mateos)
- refactor io_import_fixed()/io_prep_reg_iovec()/io_import_reg_vec() for allowing to handle multiple buffers for single request
- kernel selftests
- all kinds of comments from Caleb Sander Mateos
- support vectored and registered vector buffer
Ming Lei (12):
io_uring: make io_import_fixed() global
io_uring: refactor io_prep_reg_iovec() for BPF kfunc use
io_uring: refactor io_import_reg_vec() for BPF kfunc use
io_uring: prepare for extending io_uring with bpf
io_uring: bpf: extend io_uring with bpf struct_ops
io_uring: bpf: implement struct_ops registration
io_uring: bpf: add BPF buffer descriptor for IORING_OP_BPF
io_uring: bpf: add per-buffer iterator kfuncs
bpf: add bpf_uring_buf_dynptr to special_kfunc_list selftests/io_uring: add io_uring_unregister_buffers() selftests/io_uring: add BPF struct_ops and kfunc tests selftests/io_uring: add buffer iterator selftest with BPF arena
include/linux/io_uring_types.h | 14 +- include/uapi/linux/io_uring.h | 40 + init/Kconfig | 7 + io_uring/Makefile | 1 + io_uring/bpf-ops.c | 7 +- io_uring/bpf_ext.c | 755 ++++++++++++++++++ io_uring/bpf_ext.h | 71 ++ io_uring/io_uring.c | 9 +- io_uring/io_uring.h | 6 +- io_uring/opdef.c | 16 + io_uring/rsrc.c | 46 +- io_uring/rsrc.h | 68 +- kernel/bpf/verifier.c | 12 + tools/include/io_uring/mini_liburing.h | 10 + tools/testing/selftests/Makefile | 3 +- tools/testing/selftests/io_uring/.gitignore | 2 + tools/testing/selftests/io_uring/Makefile | 173 ++++ .../selftests/io_uring/bpf_ext_basic.bpf.c | 94 +++ .../selftests/io_uring/bpf_ext_basic.c | 215 +++++ .../selftests/io_uring/bpf_ext_memcpy.bpf.c | 305 +++++++ .../selftests/io_uring/bpf_ext_memcpy.c | 517 ++++++++++++ .../io_uring/include/bpf_ext_memcpy_defs.h | 18 + .../selftests/io_uring/include/iou_test.h | 98 +++ tools/testing/selftests/io_uring/runner.c | 206 +++++ 24 files changed, 2646 insertions(+), 47 deletions(-) create mode 100644 io_uring/bpf_ext.c create mode 100644 io_uring/bpf_ext.h create mode 100644 tools/testing/selftests/io_uring/.gitignore create mode 100644 tools/testing/selftests/io_uring/Makefile create mode 100644 tools/testing/selftests/io_uring/bpf_ext_basic.bpf.c create mode 100644 tools/testing/selftests/io_uring/bpf_ext_basic.c create mode 100644 tools/testing/selftests/io_uring/bpf_ext_memcpy.bpf.c create mode 100644 tools/testing/selftests/io_uring/bpf_ext_memcpy.c create mode 100644 tools/testing/selftests/io_uring/include/bpf_ext_memcpy_defs.h create mode 100644 tools/testing/selftests/io_uring/include/iou_test.h create mode 100644 tools/testing/selftests/io_uring/runner.c
Signature · 1 line
2.53.0
PATCH v3 1/12 io_uring: make io_import_fixed() global
Refactor buffer import functions:
- Make io_import_fixed() global so BPF kfuncs can use it directly
- Make io_import_reg_buf() static inline in rsrc.h
This allows BPF kfuncs to import buffers without associating them with a request, useful when one request has multiple buffers.
--- io_uring/rsrc.c | 17 +++-------------- io_uring/rsrc.h | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 17 deletions(-)
io_uring/rsrc.c +3 -14
@@ -1046,9 +1046,9 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter,return 0 ; } -static int io_import_fixed (int ddir,struct iov_iter *iter, -struct io_mapped_ubuf *imu, -u64 buf_addr,size_t len) +int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len) {const struct bio_vec *bvec;size_t folio_mask;
@@ -1117,17 +1117,6 @@ inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,return NULL ; } -int io_import_reg_buf (struct io_kiocb *req,struct iov_iter *iter, -u64 buf_addr,size_t len,int ddir, -unsigned issue_flags) -{ -struct io_rsrc_node *node; - - node =io_find_buf_node (req, issue_flags); -if (!node) -return -EFAULT ; -return io_import_fixed (ddir, iter, node->buf, buf_addr, len); -}/* Lock two rings at once. The rings must be different! */ static void lock_two_rings (struct io_ring_ctx *ctx1,struct io_ring_ctx *ctx2)
io_uring/rsrc.h +15 -3
@@ -65,9 +65,21 @@ int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);struct io_rsrc_node *io_find_buf_node (struct io_kiocb *req,unsigned issue_flags); -int io_import_reg_buf (struct io_kiocb *req,struct iov_iter *iter, -u64 buf_addr,size_t len,int ddir, -unsigned issue_flags); +int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len); + +static inline int io_import_reg_buf (struct io_kiocb *req,struct iov_iter *iter, +u64 buf_addr,size_t len,int ddir, +unsigned issue_flags) +{ +struct io_rsrc_node *node; + + node =io_find_buf_node (req, issue_flags); +if (!node) +return -EFAULT ; +return io_import_fixed (ddir, iter, node->buf, buf_addr, len); +}int io_import_reg_vec (int ddir,struct iov_iter *iter,struct io_kiocb *req,struct iou_vec *vec,unsigned nr_iovs,unsigned issue_flags);
Signature · 1 line
2.53.0
PATCH v3 2/12 io_uring: refactor io_prep_reg_iovec() for BPF kfunc use
In reply to Ming LeiSplit io_prep_reg_iovec() into:
- __io_prep_reg_iovec(): core logic without request association
- io_prep_reg_iovec(): inline wrapper handling request flags
The core function takes explicit 'compat' and 'need_clean' parameters instead of accessing req directly. This allows BPF kfuncs to prepare vectored buffers without request association, enabling support for multiple buffers per request.
--- io_uring/rsrc.c | 11 +++++------ io_uring/rsrc.h | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 8 deletions(-)
io_uring/rsrc.c +5 -6
@@ -1517,8 +1517,8 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,return io_vec_fill_bvec (ddir, iter, imu, iov, nr_iovs, vec); } -int io_prep_reg_iovec (struct io_kiocb *req,struct iou_vec *iv, -const struct iovec __user *uvec,size_t uvec_segs) +int __io_prep_reg_iovec (struct iou_vec *iv,const struct iovec __user *uvec, +size_t uvec_segs,bool compat,bool *need_clean) {struct iovec *iov;int iovec_off, ret;
@@ -1528,17 +1528,16 @@ int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv, ret =io_vec_realloc (iv, uvec_segs);if (ret)return ret; - req->flags |=REQ_F_NEED_CLEANUP ; +if (need_clean) + *need_clean = true; }/* pad iovec to the right */ iovec_off = iv->nr - uvec_segs; iov = iv->iovec + iovec_off; - res =iovec_from_user (uvec, uvec_segs, uvec_segs, iov, -io_is_compat (req->ctx)); + res =iovec_from_user (uvec, uvec_segs, uvec_segs, iov, compat);if (IS_ERR (res))return PTR_ERR (res); - req->flags |=REQ_F_IMPORT_BUFFER ;return 0 ; }
io_uring/rsrc.h +19 -2
@@ -4,6 +4,7 @@#include <linux/io_uring_types.h> #include <linux/lockdep.h> +#include "io_uring.h" #define IO_VEC_CACHE_SOFT_CAP 256
@@ -83,8 +84,24 @@ static inline int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,int io_import_reg_vec (int ddir,struct iov_iter *iter,struct io_kiocb *req,struct iou_vec *vec,unsigned nr_iovs,unsigned issue_flags); -int io_prep_reg_iovec (struct io_kiocb *req,struct iou_vec *iv, -const struct iovec __user *uvec,size_t uvec_segs); +int __io_prep_reg_iovec (struct iou_vec *iv,const struct iovec __user *uvec, +size_t uvec_segs,bool compat,bool *need_clean); + +static inline int io_prep_reg_iovec (struct io_kiocb *req,struct iou_vec *iv, +const struct iovec __user *uvec, +size_t uvec_segs) +{ +bool need_clean = false; +int ret; + + ret =__io_prep_reg_iovec (iv, uvec, uvec_segs, +io_is_compat (req->ctx), &need_clean); +if (need_clean) + req->flags |=REQ_F_NEED_CLEANUP ; +if (ret >=0 ) + req->flags |=REQ_F_IMPORT_BUFFER ; +return ret; +}int io_register_clone_buffers (struct io_ring_ctx *ctx,void __user *arg);int io_sqe_buffers_unregister (struct io_ring_ctx *ctx);
Signature · 1 line
2.53.0
PATCH v3 3/12 io_uring: refactor io_import_reg_vec() for BPF kfunc use
In reply to Ming LeiSplit io_import_reg_vec() into:
- __io_import_reg_vec(): core logic taking io_mapped_ubuf directly
- io_import_reg_vec(): inline wrapper handling buffer lookup and request flags
The core function takes 'imu' and 'need_clean' parameters instead of accessing req directly. This allows BPF kfuncs to import vectored buffers without request association, enabling support for multiple buffers per request.
--- io_uring/rsrc.c | 18 +++++------------- io_uring/rsrc.h | 29 ++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 16 deletions(-)
io_uring/rsrc.c +5 -13
@@ -1453,23 +1453,14 @@ static int io_kern_bvec_size(struct iovec *iov, unsigned nr_iovs,return 0 ; } -int io_import_reg_vec (int ddir,struct iov_iter *iter, -struct io_kiocb *req,struct iou_vec *vec, -unsigned nr_iovs,unsigned issue_flags) +int __io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu,struct iou_vec *vec, +unsigned nr_iovs,bool *need_clean) { -struct io_rsrc_node *node; -struct io_mapped_ubuf *imu;unsigned iovec_off;struct iovec *iov;unsigned nr_segs; - node =io_find_buf_node (req, issue_flags); -if (!node) -return -EFAULT ; - imu = node->buf; -if (!(imu->dir & (1 << ddir))) -return -EFAULT ; - iovec_off = vec->nr - nr_iovs; iov = vec->iovec + iovec_off;
@@ -1508,7 +1499,8 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter, *vec = tmp_vec; iov = vec->iovec + iovec_off; - req->flags |=REQ_F_NEED_CLEANUP ; +if (need_clean) + *need_clean = true; }if (imu->flags &IO_REGBUF_F_KBUF )
io_uring/rsrc.h +26 -3
@@ -81,9 +81,32 @@ static inline int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,return -EFAULT ;return io_import_fixed (ddir, iter, node->buf, buf_addr, len); } -int io_import_reg_vec (int ddir,struct iov_iter *iter, -struct io_kiocb *req,struct iou_vec *vec, -unsigned nr_iovs,unsigned issue_flags); +int __io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu,struct iou_vec *vec, +unsigned nr_iovs,bool *need_clean); + +static inline int io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_kiocb *req,struct iou_vec *vec, +unsigned nr_iovs,unsigned issue_flags) +{ +struct io_rsrc_node *node; +struct io_mapped_ubuf *imu; +bool need_clean = false; +int ret; + + node =io_find_buf_node (req, issue_flags); +if (!node) +return -EFAULT ; + imu = node->buf; +if (!(imu->dir & (1 << ddir))) +return -EFAULT ; + + ret =__io_import_reg_vec (ddir, iter, imu, vec, nr_iovs, &need_clean); +if (need_clean) + req->flags |=REQ_F_NEED_CLEANUP ; +return ret; +} +int __io_prep_reg_iovec (struct iou_vec *iv,const struct iovec __user *uvec,size_t uvec_segs,bool compat,bool *need_clean);
Signature · 1 line
2.53.0
PATCH v3 4/12 io_uring: prepare for extending io_uring with bpf
In reply to Ming LeiAdd one bpf operation & related framework and prepare for extending io_uring with bpf.
--- include/uapi/linux/io_uring.h | 1 + init/Kconfig | 7 +++++++ io_uring/Makefile | 1 + io_uring/bpf_ext.c | 26 ++++++++++++++++++++++++++ io_uring/bpf_ext.h | 15 +++++++++++++++ io_uring/opdef.c | 16 ++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 io_uring/bpf_ext.c create mode 100644 io_uring/bpf_ext.h
include/uapi/linux/io_uring.h +1 -0
@@ -318,6 +318,7 @@ enum io_uring_op {IORING_OP_PIPE ,IORING_OP_NOP128 ,IORING_OP_URING_CMD128 , +IORING_OP_BPF ,/* this goes last, obviously */ IORING_OP_LAST,
init/Kconfig +7 -0
@@ -1884,6 +1884,13 @@ config IO_URING applications to submit and complete IO through submission and completion rings that are shared between the kernel and application. +config IO_URING_BPF_EXT + bool "Enable IO uring bpf extension" if EXPERT + depends on IO_URING && BPF + help + This option enables bpf extension for the io_uring interface, so + application can define its own io_uring operation by bpf program. + config GCOV_PROFILE_URING bool "Enable GCOV profiling on the io_uring subsystem" depends on IO_URING && GCOV_KERNEL
io_uring/Makefile +1 -0
@@ -26,3 +26,4 @@ obj-$(CONFIG_PROC_FS) += fdinfo.o obj-$(CONFIG_IO_URING_MOCK_FILE) += mock_file.o obj-$(CONFIG_IO_URING_BPF) += bpf_filter.o obj-$(CONFIG_IO_URING_BPF_OPS) += bpf-ops.o +obj-$(CONFIG_IO_URING_BPF_EXT) += bpf_ext.o
io_uring/bpf_ext.c new file +26 -0
@@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Red Hat */ + +#include <linux/kernel.h> +#include <linux/errno.h> +#include <uapi/linux/io_uring.h> +#include "io_uring.h" +#include "bpf_ext.h" + +int io_uring_bpf_issue (struct io_kiocb *req,unsigned int issue_flags) +{ +return -EOPNOTSUPP ; +} + +int io_uring_bpf_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +return -EOPNOTSUPP ; +} + +void io_uring_bpf_fail (struct io_kiocb *req) +{ +} + +void io_uring_bpf_cleanup (struct io_kiocb *req) +{ +}
io_uring/bpf_ext.h new file +15 -0
@@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef IOU_BPF_OP_H +#define IOU_BPF_OP_H + +struct io_kiocb ; +struct io_uring_sqe ; + +#ifdef CONFIG_IO_URING_BPF_EXT +int io_uring_bpf_issue (struct io_kiocb *req,unsigned int issue_flags); +int io_uring_bpf_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe); +void io_uring_bpf_fail (struct io_kiocb *req); +void io_uring_bpf_cleanup (struct io_kiocb *req); +#endif + +#endif
io_uring/opdef.c +16 -0
@@ -38,6 +38,7 @@#include "futex.h" #include "truncate.h" #include "zcrx.h" +#include "bpf_ext.h" static int io_no_issue (struct io_kiocb *req,unsigned int issue_flags) {
@@ -589,6 +590,14 @@ const struct io_issue_def io_issue_defs[] = { .prep = io_uring_cmd_prep, .issue = io_uring_cmd, }, + [IORING_OP_BPF] = { +#if defined(CONFIG_IO_URING_BPF_EXT ) + .prep = io_uring_bpf_prep, + .issue = io_uring_bpf_issue, +#else + .prep = io_eopnotsupp_prep, +#endif + }, }; const struct io_cold_def io_cold_defs[] = {
@@ -847,6 +856,13 @@ const struct io_cold_def io_cold_defs[] = { .sqe_copy = io_uring_cmd_sqe_copy, .cleanup = io_uring_cmd_cleanup, }, + [IORING_OP_BPF] = { + .name = "BPF", +#if defined(CONFIG_IO_URING_BPF_EXT ) + .cleanup = io_uring_bpf_cleanup, + .fail = io_uring_bpf_fail, +#endif + }, };const char *io_uring_get_opcode (u8 opcode)
Signature · 1 line
2.53.0
PATCH v3 5/12 io_uring: bpf: extend io_uring with bpf struct_ops
In reply to Ming Leiio_uring can be extended with bpf struct_ops in the following ways:
1) add new io_uring operation from application
- one typical use case is for operating device zero-copy buffer, which belongs to kernel, and not visible or too expensive to export to userspace, such as supporting copy data from this buffer to userspace, decompressing data to zero-copy buffer in Android case[1][2], or checksum/decrypting.
[1] https://lpc.events/event/18/contributions/1710/attachments/1440/3070/LPC2024_ublk_zero_copy.pdf
2) extend 64 byte SQE, since bpf map can be used to store IO data conveniently
3) communicate in IO chain, since bpf map can be shared among IOs, when one bpf IO is completed, data can be written to IO chain wide bpf map, then the following bpf IO can retrieve the data from this bpf map, this way is more flexible than io_uring built-in buffer
4) pretty handy to inject error for test purpose
bpf struct_ops is one very handy way to attach bpf prog with kernel, and this patch simply wires existed io_uring operation callbacks with added uring bpf struct_ops, so application can define its own uring bpf operations.
--- include/linux/io_uring_types.h | 12 +- include/uapi/linux/io_uring.h | 12 ++ io_uring/bpf-ops.c | 7 +- io_uring/bpf_ext.c | 234 ++++++++++++++++++++++++++++++++- io_uring/bpf_ext.h | 41 ++++++ io_uring/io_uring.c | 9 +- io_uring/io_uring.h | 6 +- 7 files changed, 314 insertions(+), 7 deletions(-)
include/linux/io_uring_types.h +11 -1
@@ -10,6 +10,7 @@struct iou_loop_params ;struct io_uring_bpf_ops ; +struct uring_bpf_ops_kern ;enum {/*
@@ -493,7 +494,16 @@ struct io_ring_ctx {DECLARE_HASHTABLE (napi_ht,4 );#endif -struct io_uring_bpf_ops *bpf_ops; +/* +* bpf_ops and bpf_ext_ops are mutually exclusive: bpf_ops is used +* for io_uring_bpf_ops struct_ops, while bpf_ext_ops provides +* per-opcode BPF extension operations (IORING_SETUP_BPF_EXT). +* The two cannot be active at the same time on the same ring. +*/ +union { +struct io_uring_bpf_ops *bpf_ops; +struct uring_bpf_ops_kern *bpf_ext_ops; + };/* * Protection for resize vs mmap races - both the mmap and resize
include/uapi/linux/io_uring.h +12 -0
@@ -76,6 +76,7 @@ struct io_uring_sqe {__u32 install_fd_flags;__u32 nop_flags;__u32 pipe_flags; +__u32 bpf_op_flags; };__u64 user_data;/* data to be passed back at completion time */ /* pack this to avoid bogus arm OABI complaints */
@@ -252,6 +253,9 @@ enum io_uring_sqe_flags_bit {*/ #define IORING_SETUP_SQ_REWIND (1U << 20) +/* Allow userspace to define io_uring operation by BPF prog */ +#define IORING_SETUP_BPF_EXT (1U << 21) +enum io_uring_op {IORING_OP_NOP ,IORING_OP_READV ,
@@ -442,6 +446,13 @@ enum io_uring_op {#define IORING_RECVSEND_BUNDLE (1U << 4)#define IORING_SEND_VECTORIZED (1U << 5) +/* +* sqe->bpf_op_flags top 8bits is for storing bpf prog sub op +* The other 24bits are used for bpf prog +*/ +#define IORING_BPF_OP_BITS 8 +#define IORING_BPF_OP_SHIFT 24 +/* * cqe.res for IORING_CQE_F_NOTIF if * IORING_SEND_ZC_REPORT_USAGE was requested
@@ -646,6 +657,7 @@ struct io_uring_params {#define IORING_FEAT_MIN_TIMEOUT (1U << 15)#define IORING_FEAT_RW_ATTR (1U << 16)#define IORING_FEAT_NO_IOWAIT (1U << 17) +#define IORING_FEAT_BPF (1U << 18)/* * io_uring_register(2) opcodes and arguments
io_uring/bpf-ops.c +6 -1
@@ -162,7 +162,6 @@ static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops)return -EOPNOTSUPP ;if (!(ctx->flags &IORING_SETUP_DEFER_TASKRUN ))return -EOPNOTSUPP ; -if (ctx->bpf_ops)return -EBUSY ;if (WARN_ON_ONCE (!ops->loop_step))
@@ -186,6 +185,12 @@ static int bpf_io_reg(void *kdata, struct bpf_link *link)return PTR_ERR (file); ctx = file->private_data; +/* bpf_ops and bpf_ext_ops share storage and are mutually exclusive */ +if (ctx->flags &IORING_SETUP_BPF_EXT ) { +fput (file); +return -EINVAL ; + } + scoped_guard(mutex, &io_bpf_ctrl_mutex) {guard (mutex)(&ctx->uring_lock); ret =io_install_bpf (ctx, ops);
io_uring/bpf_ext.c +232 -2
256 lines not on this page · raw message
io_uring/bpf_ext.h +41 -0
53 lines not on this page · raw message
io_uring/io_uring.c +8 -1
@@ -97,6 +97,7 @@#include "wait.h" #include "bpf_filter.h" #include "loop.h" +#include "bpf_ext.h" #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \ IOSQE_IO_HARDLINK | IOSQE_ASYNC)
@@ -294,6 +295,9 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)io_napi_init (ctx);mutex_init (&ctx->mmap_lock); +if (io_bpf_alloc (ctx)) + goto free_ref; +return ctx; free_ref:
@@ -2150,7 +2154,9 @@ static __cold void io_req_caches_free(struct io_ring_ctx *ctx)static __cold void io_ring_ctx_free (struct io_ring_ctx *ctx) { -io_unregister_bpf_ops (ctx); +/* bpf_ops and bpf_ext_ops share storage; skip if bpf_ext_ops is active */ +if (!(ctx->flags &IORING_SETUP_BPF_EXT )) +io_unregister_bpf_ops (ctx);io_sq_thread_finish (ctx);mutex_lock (&ctx->uring_lock);
@@ -2196,6 +2202,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)if (ctx->hash_map)io_wq_put_hash (ctx->hash_map);io_napi_free (ctx); +io_bpf_free (ctx);kvfree (ctx->cancel_table.hbs);xa_destroy (&ctx->io_bl_xa);kfree (ctx);
io_uring/io_uring.h +4 -2
@@ -49,7 +49,8 @@ struct io_ctx_config { IORING_FEAT_RECVSEND_BUNDLE |\ IORING_FEAT_MIN_TIMEOUT |\ IORING_FEAT_RW_ATTR |\ - IORING_FEAT_NO_IOWAIT) + IORING_FEAT_NO_IOWAIT |\ + IORING_FEAT_BPF)#define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ IORING_SETUP_SQPOLL |\
@@ -71,7 +72,8 @@ struct io_ctx_config { IORING_SETUP_HYBRID_IOPOLL |\ IORING_SETUP_CQE_MIXED |\ IORING_SETUP_SQE_MIXED |\ - IORING_SETUP_SQ_REWIND) + IORING_SETUP_SQ_REWIND |\ + IORING_SETUP_BPF_EXT)#define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\ IORING_ENTER_SQ_WAKEUP |\
Signature · 1 line
2.53.0
Jens Axboe
maintainerRe: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 5 lines · Ming Lei
include/linux/io_uring_types.h +0
-1@@ -493,7 +494,16 @@ struct io_ring_ctx {DECLARE_HASHTABLE (napi_ht,4 );#endif -struct io_uring_bpf_ops *bpf_ops;
include/linux/io_uring_types.h struct io_ring_ctx { · line 506
Quoted 10 lines
include/linux/io_uring_types.h +10
-0+/* +* bpf_ops and bpf_ext_ops are mutually exclusive: bpf_ops is used +* for io_uring_bpf_ops struct_ops, while bpf_ext_ops provides +* per-opcode BPF extension operations (IORING_SETUP_BPF_EXT). +* The two cannot be active at the same time on the same ring. +*/ +union { +struct io_uring_bpf_ops *bpf_ops; +struct uring_bpf_ops_kern *bpf_ext_ops; + };
What am I missing here, why is this the case? What makes the use of both at the same time impossible?
io_uring/bpf-ops.c static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) · line 167
Quoted 12 lines
io_uring/bpf-ops.c +0
-1@@ -162,7 +162,6 @@ static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops)return -EOPNOTSUPP ;if (!(ctx->flags &IORING_SETUP_DEFER_TASKRUN ))return -EOPNOTSUPP ; -if (ctx->bpf_ops)return -EBUSY ;if (WARN_ON_ONCE (!ops->loop_step))
Spurious whitespace change.
io_uring/io_uring.h struct io_ctx_config { · line 53
Quoted 11 lines
io_uring/io_uring.h +2
-1@@ -49,7 +49,8 @@ struct io_ctx_config {IORING_FEAT_RECVSEND_BUNDLE |\IORING_FEAT_MIN_TIMEOUT |\IORING_FEAT_RW_ATTR |\ - IORING_FEAT_NO_IOWAIT) +IORING_FEAT_NO_IOWAIT |\ + IORING_FEAT_BPF)
Do we need this FEAT flag? If you think so, then it should at least be dependent on whether the kernel supports this feature, eg if CONFIG_IO_URING_BPF_EXT is set
Signature · 1 line
Jens Axboe
Re: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 5 lines · Jens Axboe
Quoted 4 lines · Ming Lei
include/linux/io_uring_types.h +0
-0@@ -493,7 +494,16 @@ struct io_ring_ctx {DECLARE_HASHTABLE (napi_ht,4 );#endif
Quoted 14 lines
include/linux/io_uring_types.h struct io_ring_ctx { · line 506
Quoted 11 lines
include/linux/io_uring_types.h +10
-1-struct io_uring_bpf_ops *bpf_ops; +/* +* bpf_ops and bpf_ext_ops are mutually exclusive: bpf_ops is used +* for io_uring_bpf_ops struct_ops, while bpf_ext_ops provides +* per-opcode BPF extension operations (IORING_SETUP_BPF_EXT). +* The two cannot be active at the same time on the same ring. +*/ +union { +struct io_uring_bpf_ops *bpf_ops; +struct uring_bpf_ops_kern *bpf_ext_ops; + };What am I missing here, why is this the case? What makes the use of both at the same time impossible?
Please see the following code:
static inline bool io_has_loop_ops(struct io_ring_ctx *ctx)
{
return data_race(ctx->loop_step);
}io_uring_enter():
...
if (io_has_loop_ops(ctx)) {
ret = io_run_loop(ctx);
goto out;
}
...So if ->loop_step is assigned from io_install_bpf() called from bpf_ops registration, traditional userspace SQE submission and CQE reap are bypassed completely, then IORING_OP_BPF and any other OP can't be handled at all.
Quoted 15 lines
io_uring/bpf-ops.c static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) · line 167
Quoted 13 lines
io_uring/bpf-ops.c +0
-1@@ -162,7 +162,6 @@ static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops)return -EOPNOTSUPP ;if (!(ctx->flags &IORING_SETUP_DEFER_TASKRUN ))return -EOPNOTSUPP ; -if (ctx->bpf_ops)return -EBUSY ;if (WARN_ON_ONCE (!ops->loop_step))Spurious whitespace change.
Will remove it in next version.
Quoted 16 lines
io_uring/io_uring.h struct io_ctx_config { · line 53
Quoted 12 lines
io_uring/io_uring.h +2
-1@@ -49,7 +49,8 @@ struct io_ctx_config {IORING_FEAT_RECVSEND_BUNDLE |\IORING_FEAT_MIN_TIMEOUT |\IORING_FEAT_RW_ATTR |\ - IORING_FEAT_NO_IOWAIT) +IORING_FEAT_NO_IOWAIT |\ + IORING_FEAT_BPF)Do we need this FEAT flag? If you think so, then it should at least be dependent on whether the kernel supports this feature, eg if
CONFIG_IO_URING_BPF_EXTis set
Good catch!
Thanks,
Ming
Jens Axboe
maintainerRe: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 36 lines · Ming Lei
Quoted 5 lines · Jens Axboe
Quoted 4 lines · Ming Lei
include/linux/io_uring_types.h +0
-0@@ -493,7 +494,16 @@ struct io_ring_ctx {DECLARE_HASHTABLE (napi_ht,4 );#endif Quoted 14 lines
include/linux/io_uring_types.h struct io_ring_ctx { · line 506
Quoted 11 lines
include/linux/io_uring_types.h +10
-1-struct io_uring_bpf_ops *bpf_ops; +/* +* bpf_ops and bpf_ext_ops are mutually exclusive: bpf_ops is used +* for io_uring_bpf_ops struct_ops, while bpf_ext_ops provides +* per-opcode BPF extension operations (IORING_SETUP_BPF_EXT). +* The two cannot be active at the same time on the same ring. +*/ +union { +struct io_uring_bpf_ops *bpf_ops; +struct uring_bpf_ops_kern *bpf_ext_ops; + };What am I missing here, why is this the case? What makes the use of both at the same time impossible?
Please see the following code:
static inline bool
io_has_loop_ops(struct io_ring_ctx *ctx){ return data_race(ctx->loop_step); }
io_uring_enter():... if (io_has_loop_ops(ctx)) { ret = io_run_loop(ctx); goto out; } ...
Quoted 4 lines
So if ->loop_step is assigned from
io_install_bpf()called frombpf_opsregistration, traditional userspace SQE submission and CQE reap are bypassed completely, thenIORING_OP_BPFand any other OP can't be handled at all.
It ends up calling io_submit_sqes() all the same, so not sure I follow the problem here. Seems to me that the only thing that is making it mutually exclusive is the fact that you unionized the ops.
Signature · 1 line
Jens Axboe
Re: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 37 lines · Jens Axboe
Quoted 36 lines · Ming Lei
Quoted 5 lines · Jens Axboe
Quoted 4 lines · Ming Lei
include/linux/io_uring_types.h +0
-0@@ -493,7 +494,16 @@ struct io_ring_ctx {DECLARE_HASHTABLE (napi_ht,4 );#endif Quoted 14 lines
include/linux/io_uring_types.h struct io_ring_ctx { · line 506
Quoted 11 lines
include/linux/io_uring_types.h +10
-1-struct io_uring_bpf_ops *bpf_ops; +/* +* bpf_ops and bpf_ext_ops are mutually exclusive: bpf_ops is used +* for io_uring_bpf_ops struct_ops, while bpf_ext_ops provides +* per-opcode BPF extension operations (IORING_SETUP_BPF_EXT). +* The two cannot be active at the same time on the same ring. +*/ +union { +struct io_uring_bpf_ops *bpf_ops; +struct uring_bpf_ops_kern *bpf_ext_ops; + };What am I missing here, why is this the case? What makes the use of both at the same time impossible?
Please see the following code:
static inline bool
io_has_loop_ops(struct io_ring_ctx *ctx){ return data_race(ctx->loop_step); }
io_uring_enter():... if (io_has_loop_ops(ctx)) { ret = io_run_loop(ctx); goto out; } ...
Quoted 8 lines
Quoted 4 lines
So if ->loop_step is assigned from
io_install_bpf()called frombpf_opsregistration, traditional userspace SQE submission and CQE reap are bypassed completely, thenIORING_OP_BPFand any other OP can't be handled at all.It ends up calling
io_submit_sqes()all the same, so not sure I follow the problem here. Seems to me that the only thing that is making it mutually exclusive is the fact that you unionized the ops.
Looks I miss the point, will switch to non-exclusive in next version.
Thanks,
Ming
Jens Axboe
maintainerRe: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
In reply to Ming LeiQuoted 11 lines · Ming Lei
int
io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){io_uring/bpf_ext.c +9
-0+ struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data); +u32 opf =READ_ONCE (sqe->bpf_op_flags); +unsigned char bpf_op =uring_bpf_get_op (opf); +const struct uring_bpf_ops *ops; + +if (unlikely (!(req->ctx->flags &IORING_SETUP_BPF_EXT ))) + goto fail; + +if (bpf_op >=IO_RING_MAX_BPF_OPS )
io_uring/bpf_ext.c line 47
Quoted 10 lines
io_uring/bpf_ext.c +8
-0+return -EINVAL ; + + ops = req->ctx->bpf_ext_ops[bpf_op].ops; + data->opf = opf; + data->ops = ops; +if (ops && ops->prep_fn) +return ops->prep_fn (data, sqe); +fail:return -EOPNOTSUPP ; }
Any early exit should ensure 'data' is sane, so that the cleanup doesn't potentially touch uninitialized crap. This is something that has bit us in the past. Not an issue for this patch that adds the code, but it will be once the next patch is applied. Better to clear ->opf/ops here upfront, so that we never leave this function without 'data' being fully initialized.
Signature · 1 line
Jens Axboe
Re: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 21 lines · Jens Axboe
Quoted 20 lines · Ming Lei
int
io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){io_uring/bpf_ext.c +17
-0+ struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data); +u32 opf =READ_ONCE (sqe->bpf_op_flags); +unsigned char bpf_op =uring_bpf_get_op (opf); +const struct uring_bpf_ops *ops; + +if (unlikely (!(req->ctx->flags &IORING_SETUP_BPF_EXT ))) + goto fail; + +if (bpf_op >=IO_RING_MAX_BPF_OPS ) +return -EINVAL ; + + ops = req->ctx->bpf_ext_ops[bpf_op].ops; + data->opf = opf; + data->ops = ops; +if (ops && ops->prep_fn) +return ops->prep_fn (data, sqe); +fail:return -EOPNOTSUPP ;
Quoted 8 lines
Quoted 1 line
}
Any early exit should ensure 'data' is sane, so that the cleanup doesn't potentially touch uninitialized crap. This is something that has bit us in the past. Not an issue for this patch that adds the code, but it will be once the next patch is applied. Better to clear ->opf/ops here upfront, so that we never leave this function without 'data' being fully initialized.
But ->cleanup() is only called in case of REQ_F_NEED_CLEANUP.
Or maybe you mean other cleanup instead of ->cleanup()?
Thanks,
Ming
Jens Axboe
maintainerRe: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 31 lines · Ming Lei
Quoted 21 lines · Jens Axboe
Quoted 20 lines · Ming Lei
int
io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){io_uring/bpf_ext.c +17
-0+ struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data); +u32 opf =READ_ONCE (sqe->bpf_op_flags); +unsigned char bpf_op =uring_bpf_get_op (opf); +const struct uring_bpf_ops *ops; + +if (unlikely (!(req->ctx->flags &IORING_SETUP_BPF_EXT ))) + goto fail; + +if (bpf_op >=IO_RING_MAX_BPF_OPS ) +return -EINVAL ; + + ops = req->ctx->bpf_ext_ops[bpf_op].ops; + data->opf = opf; + data->ops = ops; +if (ops && ops->prep_fn) +return ops->prep_fn (data, sqe); +fail:return -EOPNOTSUPP ;Quoted 8 lines
Quoted 1 line
}
Any early exit should ensure 'data' is sane, so that the cleanup doesn't potentially touch uninitialized crap. This is something that has bit us in the past. Not an issue for this patch that adds the code, but it will be once the next patch is applied. Better to clear ->opf/ops here upfront, so that we never leave this function without 'data' being fully initialized.
Quoted 3 lines
But ->cleanup() is only called in case of
REQ_F_NEED_CLEANUP.Or maybe you mean other cleanup instead of ->cleanup()?
I do mean ->cleanup() - what I'm trying to say here is that we've had cases of REQ_F_NEED_CLEANUP being set late, and hence missing cleanup for easily hit error conditions, and non-initialized data being exposed in cleanup. It's very easy to miss for later patches that adds another error condition. My recommendation is to fully initialize 'data' and set REQ_F_NEED_CLEANUP early, which handily avoids that for future changes too.
Signature · 1 line
Jens Axboe
Re: [PATCH V3 05/12] io_uring: bpf: extend io_uring with bpf struct_ops
Quoted 36 lines · Jens Axboe
Quoted 31 lines · Ming Lei
Quoted 21 lines · Jens Axboe
Quoted 20 lines · Ming Lei
int
io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){io_uring/bpf_ext.c +17
-0+ struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data); +u32 opf =READ_ONCE (sqe->bpf_op_flags); +unsigned char bpf_op =uring_bpf_get_op (opf); +const struct uring_bpf_ops *ops; + +if (unlikely (!(req->ctx->flags &IORING_SETUP_BPF_EXT ))) + goto fail; + +if (bpf_op >=IO_RING_MAX_BPF_OPS ) +return -EINVAL ; + + ops = req->ctx->bpf_ext_ops[bpf_op].ops; + data->opf = opf; + data->ops = ops; +if (ops && ops->prep_fn) +return ops->prep_fn (data, sqe); +fail:return -EOPNOTSUPP ;Quoted 8 lines
Quoted 1 line
}
Any early exit should ensure 'data' is sane, so that the cleanup doesn't potentially touch uninitialized crap. This is something that has bit us in the past. Not an issue for this patch that adds the code, but it will be once the next patch is applied. Better to clear ->opf/ops here upfront, so that we never leave this function without 'data' being fully initialized.
Quoted 3 lines
But ->cleanup() is only called in case of
REQ_F_NEED_CLEANUP.Or maybe you mean other cleanup instead of ->cleanup()?
Quoted 7 lines
I do mean ->cleanup() - what I'm trying to say here is that we've had cases of
REQ_F_NEED_CLEANUPbeing set late, and hence missing cleanup for easily hit error conditions, and non-initialized data being exposed in cleanup. It's very easy to miss for later patches that adds another error condition. My recommendation is to fully initialize 'data' and setREQ_F_NEED_CLEANUPearly, which handily avoids that for future changes too.
OK, your recommendation is definitely clean & safe, will take it.
Thanks,
Ming
PATCH v3 6/12 io_uring: bpf: implement struct_ops registration
In reply to Ming LeiComplete the BPF struct_ops registration mechanism by implementing refcount-based lifecycle management:
- Add refcount field to struct uring_bpf_ops_kern for tracking active requests
- Add wait_queue_head_t bpf_wq to struct io_ring_ctx for synchronizing unregistration with in-flight requests
- Implement io_bpf_reg_unreg() to handle registration (refcount=1) and unregistration (wait for in-flight requests to complete)
- Update io_uring_bpf_prep() to increment refcount on success and reject new requests when refcount is zero (unregistration in progress)
- Update io_uring_bpf_cleanup() to decrement refcount and wake up waiters when it reaches zero
--- include/linux/io_uring_types.h | 2 + io_uring/bpf_ext.c | 104 ++++++++++++++++++++++++++++++++- io_uring/bpf_ext.h | 3 + 3 files changed, 106 insertions(+), 3 deletions(-)
include/linux/io_uring_types.h +2 -0
@@ -516,6 +516,8 @@ struct io_ring_ctx {struct io_mapped_region ring_region;/* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; + +wait_queue_head_t bpf_wq; };/*
io_uring/bpf_ext.c +101 -3
162 lines not on this page · raw message
io_uring/bpf_ext.h +3 -0
@@ -27,14 +27,17 @@ typedef void (*uring_bpf_cleanup_t)(struct uring_bpf_data *data);struct uring_bpf_ops {unsigned short id; +int ring_fd;uring_bpf_prep_t prep_fn;uring_bpf_issue_t issue_fn;uring_bpf_fail_t fail_fn;uring_bpf_cleanup_t cleanup_fn; }; +/* TODO: manage it via `io_rsrc_node` */ struct uring_bpf_ops_kern {const struct uring_bpf_ops *ops; +int refcount; };#ifdef CONFIG_IO_URING_BPF_EXT int io_uring_bpf_issue (struct io_kiocb *req,unsigned int issue_flags);
Signature · 1 line
2.53.0
PATCH v3 7/12 io_uring: bpf: add BPF buffer descriptor for IORING_OP_BPF
In reply to Ming LeiAdd io_bpf_buf_desc struct and io_bpf_buf_type enum to describe buffer parameters for IORING_OP_BPF kfuncs. Supports plain userspace, registered, vectored, and registered vectored buffer types.
Registered buffers (FIXED, KFIXED, REG_VEC) refer to buffers pre-registered with io_uring and can be either userspace or kernel buffers depending on how they were registered.
--- include/uapi/linux/io_uring.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+)
include/uapi/linux/io_uring.h +27 -0
@@ -453,6 +453,33 @@ enum io_uring_op {#define IORING_BPF_OP_BITS 8#define IORING_BPF_OP_SHIFT 24 +/* +* BPF buffer descriptor types. +* +* Registered buffers (FIXED, KFIXED, REG_VEC) refer to buffers pre-registered +* with io_uring. These can be either userspace or kernel buffers depending on +* how they were registered. +* +* For KFIXED, addr is an offset from the registered buffer start. +* For REG_VEC with kernel buffers, each iov.iov_base is offset-based. +*/ +enum io_bpf_buf_type { +IO_BPF_BUF_USER =0 ,/* plain userspace buffer */ +IO_BPF_BUF_FIXED =1 ,/* registered buffer (absolute address) */ +IO_BPF_BUF_VEC =2 ,/* vectored buffer (iovec array) */ +IO_BPF_BUF_KFIXED =3 ,/* registered buffer (offset-based) */ +IO_BPF_BUF_REG_VEC =4 ,/* registered vectored buffer */ +}; + +/* BPF buffer descriptor for IORING_OP_BPF */ +struct io_bpf_buf_desc { +__u8 type;/* IO_BPF_BUF_* */ +__u8 reserved; +__u16 buf_index;/* registered buffer index (FIXED/KFIXED/REG_VEC) */ +__u32 len;/* length (non-vec) or nr_vecs (vec types) */ +__u64 addr;/* userspace address, iovec ptr, or offset (KFIXED) */ +}; +/* * cqe.res for IORING_CQE_F_NOTIF if * IORING_SEND_ZC_REPORT_USAGE was requested
Signature · 1 line
2.53.0
PATCH v3 8/12 io_uring: bpf: add per-buffer iterator kfuncs
In reply to Ming LeiAdd per-buffer KF_ITER kfuncs for page-level access from BPF programs. Each buffer gets its own iterator on the BPF stack, and the BPF program coordinates multiple iterators for multi-buffer operations. The verifier enforces proper iterator lifecycle via KF_ITER_NEW/NEXT/DESTROY.
kfunc API:
- bpf_iter_uring_buf_new(iter, data, desc, direction): import one buffer, take submit lock (refcounted via data->lock_depth). Supports all 5 buffer types (USER, FIXED, VEC, KFIXED, REG_VEC) and both directions (ITER_SOURCE for reading, ITER_DEST for writing).
- bpf_iter_uring_buf_next(iter): extract the next page, kmap it, return int * pointing to the avail byte count (non-NULL = more data, NULL = done). The actual page data is accessed via bpf_uring_buf_dynptr() or bpf_uring_buf_dynptr_rdwr().
- bpf_iter_uring_buf_destroy(iter): unmap page, free resources, release submit lock when lock_depth reaches zero.
- bpf_uring_buf_dynptr(it__iter, ptr__uninit): populate a read-only LOCAL dynptr bounded to avail bytes, preventing data leaks beyond valid data.
- bpf_uring_buf_dynptr_rdwr(it__iter, ptr__uninit): populate a writable LOCAL dynptr, requires direction == ITER_DEST (checked via iter.data_source).
The dynptr approach replaces the old uring_buf_page_t typedef which exposed PAGE_SIZE bytes to BPF programs even when only avail bytes contained valid data.
Note: bpf_dynptr_slice() requires a compile-time constant size, so BPF programs typically process pages in fixed-size chunks (e.g. 512 bytes). Buffer addresses and lengths should be at least 512-byte aligned for efficient access.
Add helper functions for buffer import:
- io_bpf_import_fixed_buf(): handles FIXED/KFIXED types
- io_bpf_import_reg_vec(): handles REG_VEC type
- io_bpf_import_vec_buf(): handles VEC type
- io_bpf_import_buffer(): unified dispatcher for all buffer types
--- io_uring/bpf_ext.c | 413 ++++++++++++++++++++++++++++++++++++++++++++- io_uring/bpf_ext.h | 16 +- 2 files changed, 421 insertions(+), 8 deletions(-)
io_uring/bpf_ext.c +407 -6
@@ -10,9 +10,12 @@#include <linux/btf.h> #include <linux/btf_ids.h> #include <linux/filter.h> +#include <linux/uio.h> +#include <linux/highmem.h> #include <uapi/linux/io_uring.h> #include "io_uring.h" #include "register.h" +#include "rsrc.h" #include "bpf_ext.h" static inline unsigned char uring_bpf_get_op (u32 op_flags)
@@ -20,11 +23,6 @@ static inline unsigned char uring_bpf_get_op(u32 op_flags)return (unsigned char )(op_flags >>IORING_BPF_OP_SHIFT ); } -static inline unsigned int uring_bpf_get_flags (u32 op_flags) -{ -return op_flags & ((1U <<IORING_BPF_OP_SHIFT ) -1 ); -} -int io_uring_bpf_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) { struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
@@ -47,6 +45,8 @@ int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) data->opf = opf; data->ops = ops; + data->issue_flags =0 ; + data->lock_depth =0 ; ret = ops->prep_fn (data, sqe);if (!ret) {/* Only increment refcount on success (uring_lock already held) */
@@ -74,7 +74,13 @@ static int __io_uring_bpf_issue(struct io_kiocb *req)int io_uring_bpf_issue (struct io_kiocb *req,unsigned int issue_flags) { -return __io_uring_bpf_issue (req); + struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data); +int ret; + + data->issue_flags = issue_flags; + ret =__io_uring_bpf_issue (req); + data->issue_flags =0 ; +return ret; }void io_uring_bpf_fail (struct io_kiocb *req)
@@ -291,6 +297,206 @@ static struct bpf_struct_ops bpf_uring_bpf_ops = { .owner = THIS_MODULE, }; +/* +* Per-buffer iterator kernel state (heap-allocated, one per buffer). +*/ +struct bpf_iter_uring_buf_kern { +struct uring_bpf_data *data; +struct iov_iter iter; +struct iou_vec vec; +struct io_rsrc_node *node; +struct page *page;/* current extracted page */ +void *kmap_base;/* kmap_local_page() + offset */ +int avail;/* valid bytes in current page */ +}; + +static inline struct bpf_iter_uring_buf_kern * +iter_kern (const struct bpf_iter_uring_buf *iter) +{ +return (struct bpf_iter_uring_buf_kern *)&iter->__opaque[0 ]; +} + +static void iter_unmap_page (struct bpf_iter_uring_buf_kern *kern) +{ +if (kern->kmap_base) { +kunmap_local (kern->kmap_base); + kern->kmap_base =NULL ; + } +if (kern->page &&iov_iter_extract_will_pin (&kern->iter)) { +unpin_user_page (kern->page); + kern->page =NULL ; + } +} + +/* +* Helper to import fixed buffer (FIXED or KFIXED). +* Must be called with submit lock held. +* +* FIXED: addr is absolute userspace address within buffer +* KFIXED: addr is offset from buffer start +* +* Returns node with incremented refcount on success, ERR_PTR on failure. +*/ +static struct io_rsrc_node *io_bpf_import_fixed_buf (struct io_ring_ctx *ctx, +struct iov_iter *iter, +const struct io_bpf_buf_desc *desc, +int ddir) +{ +struct io_rsrc_node *node; +struct io_mapped_ubuf *imu; +int ret; + + node =io_rsrc_node_lookup (&ctx->buf_table, desc->buf_index); +if (!node) +return ERR_PTR (-EFAULT ); + + imu = node->buf; +if (!(imu->dir & (1 << ddir))) +return ERR_PTR (-EFAULT ); + + node->refs++; + + ret =io_import_fixed (ddir, iter, imu, desc->addr, desc->len); +if (ret) { + node->refs--; +return ERR_PTR (ret); + } + +return node; +} + +/* +* Helper to import registered vectored buffer (REG_VEC). +* Must be called with submit lock held. +* +* addr: userspace iovec pointer +* len: number of iovecs +* buf_index: registered buffer index +* +* Returns node with incremented refcount on success, ERR_PTR on failure. +* Caller must call io_vec_free(vec) after use. +*/ +static struct io_rsrc_node *io_bpf_import_reg_vec (struct io_ring_ctx *ctx, +struct iov_iter *iter, +const struct io_bpf_buf_desc *desc, +int ddir,struct iou_vec *vec) +{ +struct io_rsrc_node *node; +struct io_mapped_ubuf *imu; +int ret; + + node =io_rsrc_node_lookup (&ctx->buf_table, desc->buf_index); +if (!node) +return ERR_PTR (-EFAULT ); + + imu = node->buf; +if (!(imu->dir & (1 << ddir))) +return ERR_PTR (-EFAULT ); + + node->refs++; + +/* Prepare iovec from userspace */ + ret =__io_prep_reg_iovec (vec,u64_to_user_ptr (desc->addr), + desc->len,io_is_compat (ctx),NULL ); +if (ret) + goto err; + +/* Import vectored buffer from registered buffer */ + ret =__io_import_reg_vec (ddir, iter, imu, vec, desc->len,NULL ); +if (ret) + goto err; + +return node; +err: + node->refs--; +return ERR_PTR (ret); +} + +/* +* Helper to import a vectored user buffer (VEC) into iou_vec. +* Allocates space in vec and copies iovec from userspace. +* +* Returns 0 on success, negative error code on failure. +* Caller must call io_vec_free(vec) after use. +*/ +static int io_bpf_import_vec_buf (struct io_ring_ctx *ctx, +struct iov_iter *iter, +const struct io_bpf_buf_desc *desc, +int ddir,struct iou_vec *vec) +{ +unsigned nr_vecs = desc->len; +struct iovec *iov; +size_t total_len =0 ; +void *res; +int ret, i; + +if (nr_vecs > vec->nr) { + ret =io_vec_realloc (vec, nr_vecs); +if (ret) +return ret; + } + + iov = vec->iovec; + res =iovec_from_user (u64_to_user_ptr (desc->addr), nr_vecs, + nr_vecs, iov,io_is_compat (ctx)); +if (IS_ERR (res)) +return PTR_ERR (res); + +for (i =0 ; i < nr_vecs; i++) + total_len += iov[i].iov_len; + +iov_iter_init (iter, ddir, iov, nr_vecs, total_len); +return 0 ; +} + +/* +* Helper to import a buffer into an iov_iter based on io_bpf_buf_desc. +* Supports all 5 buffer types: USER, FIXED, VEC, KFIXED, REG_VEC. +* Must be called with submit lock held for FIXED/KFIXED/REG_VEC types. +* +* @ctx: ring context +* @iter: output iterator +* @desc: buffer descriptor +* @ddir: direction (ITER_SOURCE for source, ITER_DEST for destination) +* @vec: iou_vec for VEC/REG_VEC types (caller must call io_vec_free after use) +* +* Returns node pointer (may be NULL for USER/VEC), or ERR_PTR on failure. +* Caller must drop node reference when done if non-NULL. +*/ +static struct io_rsrc_node *io_bpf_import_buffer (struct io_ring_ctx *ctx, +struct iov_iter *iter, +const struct io_bpf_buf_desc *desc, +int ddir,struct iou_vec *vec) +{ +int ret; + +switch (desc->type) { +case IO_BPF_BUF_USER : +/* Plain user buffer */ + ret =import_ubuf (ddir,u64_to_user_ptr (desc->addr), + desc->len, iter); +return ret ?ERR_PTR (ret) :NULL ; + +case IO_BPF_BUF_FIXED : +case IO_BPF_BUF_KFIXED : +/* FIXED: addr is absolute address within buffer */ +/* KFIXED: addr is offset from buffer start */ +return io_bpf_import_fixed_buf (ctx, iter, desc, ddir); + +case IO_BPF_BUF_VEC : +/* Vectored user buffer - addr is iovec ptr, len is nr_vecs */ + ret =io_bpf_import_vec_buf (ctx, iter, desc, ddir, vec); +return ret ?ERR_PTR (ret) :NULL ; + +case IO_BPF_BUF_REG_VEC : +/* Registered vectored buffer */ +return io_bpf_import_reg_vec (ctx, iter, desc, ddir, vec); + +default : +return ERR_PTR (-EINVAL ); + } +} +__bpf_kfunc_start_defs (); __bpf_kfunc void uring_bpf_set_result(struct uring_bpf_data *data, int res) {
@@ -300,10 +506,205 @@ __bpf_kfunc void uring_bpf_set_result(struct uring_bpf_data *data, int res)req_set_fail (req);io_req_set_res (req, res,0 ); } + +/** +* bpf_iter_uring_buf_new - Initialize per-buffer iterator (KF_ITER_NEW) +* @iter: BPF-visible iterator state (on BPF stack) +* @data: BPF request data containing request context +* @desc: Single buffer descriptor +* @direction: ITER_SOURCE (read from buffer) or ITER_DEST (write to buffer) +* +* Takes the submit lock (refcounted via data->lock_depth, first caller +* acquires, last _destroy releases). +* +* Returns 0 on success, negative error code on failure. +*/ +__bpf_kfunc int bpf_iter_uring_buf_new(struct bpf_iter_uring_buf *iter, +struct uring_bpf_data *data, +struct io_bpf_buf_desc *desc, +int direction) +{ +struct io_kiocb *req =cmd_to_io_kiocb (data); +struct io_ring_ctx *ctx = req->ctx; +struct bpf_iter_uring_buf_kern *kern =iter_kern (iter); +struct io_rsrc_node *node; + +BUILD_BUG_ON (sizeof (struct bpf_iter_uring_buf_kern ) > +sizeof (struct bpf_iter_uring_buf )); +BUILD_BUG_ON (__alignof__(struct bpf_iter_uring_buf_kern ) != + __alignof__(struct bpf_iter_uring_buf )); + +memset (kern,0 ,sizeof (*kern)); + +if (desc->type >IO_BPF_BUF_REG_VEC ) +return -EINVAL ; +if (direction !=ITER_SOURCE && direction !=ITER_DEST ) +return -EINVAL ; + + kern->data = data; + +if (data->lock_depth++ ==0 ) +io_ring_submit_lock (ctx, data->issue_flags); + + node =io_bpf_import_buffer (ctx, &kern->iter, desc, + direction, &kern->vec); +if (IS_ERR (node)) { +if (--data->lock_depth ==0 ) +io_ring_submit_unlock (ctx, data->issue_flags); + kern->data =NULL ; +return PTR_ERR (node); + } + + kern->node = node; +return 0 ; +} + +/** +* bpf_iter_uring_buf_next - Get next page chunk (KF_ITER_NEXT) +* @iter: BPF-visible iterator state +* +* Unmaps the previous page and extracts the next one. +* +* Returns a non-NULL pointer when data is available, NULL when done. +* The returned pointer is to the avail count (int); the actual page data +* must be obtained via bpf_uring_buf_dynptr(). +* +* Note: bpf_dynptr_slice() requires a compile-time constant size, so BPF +* programs typically process pages in fixed-size chunks (e.g. 512 bytes). +* If the page offset or extracted length is not aligned to the chunk size, +* the trailing bytes cannot be accessed via bpf_dynptr_slice() and are +* silently dropped. For best efficiency, callers should ensure buffer +* addresses and lengths are at least 512-byte aligned. +*/ +__bpf_kfunc int * +bpf_iter_uring_buf_next (struct bpf_iter_uring_buf *iter) +{ +struct bpf_iter_uring_buf_kern *kern =iter_kern (iter); +struct page *pages[1 ]; +struct page **pp = pages; +size_t offset; +ssize_t extracted; + +if (!kern->data) +return NULL ; + +/* Unmap and release previous page */ +iter_unmap_page (kern); + kern->avail =0 ; + +if (iov_iter_count (&kern->iter) ==0 ) +return NULL ; + +/* Extract next page */ + extracted =iov_iter_extract_pages (&kern->iter, &pp, +PAGE_SIZE ,1 ,0 , &offset); +if (extracted <=0 ) +return NULL ; + + kern->page = pp[0 ]; + kern->kmap_base =kmap_local_page (kern->page) + offset; + kern->avail = extracted; + +return &kern->avail; +} + +/** +* bpf_uring_buf_dynptr - Get dynptr for current page data +* @it__iter: Buffer iterator (must have a current page from _next()) +* @ptr__uninit: Dynptr to initialize (LOCAL type, read-only) +* +* Initializes @ptr__uninit as a read-only LOCAL dynptr whose size equals +* the valid byte count in the current page chunk. This prevents reads +* beyond the actual buffer data, unlike the old uring_buf_page_t approach +* which exposed a full PAGE_SIZE pointer. +* +* Returns 0 on success, -EINVAL if no current page is available. +*/ +__bpf_kfunc int bpf_uring_buf_dynptr(struct bpf_iter_uring_buf *it__iter, +struct bpf_dynptr *ptr__uninit) +{ +struct bpf_dynptr_kern *dynptr = (struct bpf_dynptr_kern *)ptr__uninit; +struct bpf_iter_uring_buf_kern *kern =iter_kern (it__iter); + +if (!kern->kmap_base || kern->avail <=0 ) { +bpf_dynptr_set_null (dynptr); +return -EINVAL ; + } + +bpf_dynptr_init (dynptr, kern->kmap_base, +BPF_DYNPTR_TYPE_LOCAL ,0 , kern->avail); +bpf_dynptr_set_rdonly (dynptr); +return 0 ; +} + +/** +* bpf_uring_buf_dynptr_rdwr - Get writable dynptr for current page data +* @it__iter: Buffer iterator (must have a current page from _next()) +* @ptr__uninit: Dynptr to initialize (LOCAL type, read-write) +* +* Like bpf_uring_buf_dynptr() but returns a writable dynptr. The iterator +* must have been created with direction == ITER_DEST; otherwise returns +* -EPERM. This allows writing data into user buffers (e.g. copying from +* BPF arena to a user-provided destination buffer). +* +* Returns 0 on success, -EINVAL if no current page, -EPERM if not ITER_DEST. +*/ +__bpf_kfunc int bpf_uring_buf_dynptr_rdwr(struct bpf_iter_uring_buf *it__iter, +struct bpf_dynptr *ptr__uninit) +{ +struct bpf_dynptr_kern *dynptr = (struct bpf_dynptr_kern *)ptr__uninit; +struct bpf_iter_uring_buf_kern *kern =iter_kern (it__iter); + +if (!kern->kmap_base || kern->avail <=0 ) { +bpf_dynptr_set_null (dynptr); +return -EINVAL ; + } + +if (kern->iter.data_source) {/* ITER_SOURCE — read-only buffer */ +bpf_dynptr_set_null (dynptr); +return -EPERM ; + } + +bpf_dynptr_init (dynptr, kern->kmap_base, +BPF_DYNPTR_TYPE_LOCAL ,0 , kern->avail); +return 0 ; +} + +/** +* bpf_iter_uring_buf_destroy - Destroy per-buffer iterator (KF_ITER_DESTROY) +* @iter: BPF-visible iterator state +* +* Unmaps page, frees resources, releases submit lock if this +* iterator owns it. +*/ +__bpf_kfunc void bpf_iter_uring_buf_destroy(struct bpf_iter_uring_buf *iter) +{ +struct bpf_iter_uring_buf_kern *kern =iter_kern (iter); +struct io_ring_ctx *ctx; + +if (!kern->data) +return ; + + ctx =cmd_to_io_kiocb (kern->data)->ctx; + +iter_unmap_page (kern); +io_vec_free (&kern->vec); +if (kern->node) +io_put_rsrc_node (ctx, kern->node); +if (--kern->data->lock_depth ==0 ) +io_ring_submit_unlock (ctx, kern->data->issue_flags); + kern->data =NULL ; +} +__bpf_kfunc_end_defs ();BTF_KFUNCS_START (uring_bpf_kfuncs)BTF_ID_FLAGS (func ,uring_bpf_set_result ) +BTF_ID_FLAGS (func, bpf_iter_uring_buf_new,KF_ITER_NEW ) +BTF_ID_FLAGS (func, bpf_iter_uring_buf_next,KF_ITER_NEXT |KF_RET_NULL ) +BTF_ID_FLAGS (func, bpf_iter_uring_buf_destroy,KF_ITER_DESTROY ) +BTF_ID_FLAGS (func, bpf_uring_buf_dynptr) +BTF_ID_FLAGS (func, bpf_uring_buf_dynptr_rdwr)BTF_KFUNCS_END (uring_bpf_kfuncs) static const struct btf_kfunc_id_set uring_kfunc_set = {
io_uring/bpf_ext.h +14 -2
@@ -13,10 +13,13 @@ struct uring_bpf_data {void *req_data;/* not for bpf prog */ const struct uring_bpf_ops *ops;u32 opf; +u32 issue_flags;/* io_uring issue flags */ +unsigned int lock_depth;/* not for bpf prog */ /* writeable for bpf prog */ u8 pdu[64 -sizeof (void *) - -sizeof (struct uring_bpf_ops *) -sizeof (u32)]; +sizeof (struct uring_bpf_ops *) -2 *sizeof (u32) - +sizeof (unsigned int )]; }; typedef int (*uring_bpf_prep_t)(struct uring_bpf_data *data,
@@ -37,8 +40,17 @@ struct uring_bpf_ops {/* TODO: manage it via `io_rsrc_node` */ struct uring_bpf_ops_kern {const struct uring_bpf_ops *ops; -int refcount; +int refcount;/* Protected by ctx->uring_lock */ }; + +/* +* Per-buffer BPF iterator state (lives on BPF stack). +* Uses bpf_iter_ prefix for KF_ITER verifier enforcement. +* Kernel-internal state is stored inline in the __opaque[] array. +*/ +struct bpf_iter_uring_buf { +__u64 __opaque[12 ]; +}__aligned(8) ;#ifdef CONFIG_IO_URING_BPF_EXT int io_uring_bpf_issue (struct io_kiocb *req,unsigned int issue_flags);int io_uring_bpf_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);
Signature · 1 line
2.53.0
PATCH v3 9/12 bpf: add bpf_uring_buf_dynptr to special_kfunc_list
In reply to Ming LeiRegister bpf_uring_buf_dynptr() and bpf_uring_buf_dynptr_rdwr() in the verifier's special_kfunc_list so they can produce LOCAL dynptrs. Without these entries, the verifier cannot determine the dynptr type for the __uninit output parameter.
We cannot reuse the existing bpf_dynptr_from_mem() helper because kfunc PTR_TO_MEM returns are always marked MEM_RDONLY by the verifier (meta.r0_rdonly = true), but bpf_dynptr_from_mem() requires writable memory (ARG_PTR_TO_UNINIT_MEM). A dedicated kfunc with a special_kfunc_list entry is the only way to produce a dynptr from read-only kernel-provided memory.
--- kernel/bpf/verifier.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
kernel/bpf/verifier.c +12 -0
@@ -12531,6 +12531,8 @@ enum special_kfunc_type { KF_bpf_session_is_return, KF_bpf_stream_vprintk, KF_bpf_stream_print_stack, + KF_bpf_uring_buf_dynptr, + KF_bpf_uring_buf_dynptr_rdwr, };BTF_ID_LIST (special_kfunc_list)
@@ -12611,6 +12613,13 @@ BTF_ID(func, bpf_arena_reserve_pages) BTF_ID(func, bpf_session_is_return) BTF_ID(func, bpf_stream_vprintk) BTF_ID(func, bpf_stream_print_stack) +#ifdef CONFIG_IO_URING_BPF_EXT +BTF_ID(func, bpf_uring_buf_dynptr) +BTF_ID(func, bpf_uring_buf_dynptr_rdwr) +#else +BTF_ID_UNUSED +BTF_ID_UNUSED +#endif static bool is_task_work_add_kfunc (u32 func_id) {
@@ -13590,6 +13599,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ dynptr_arg_type |=DYNPTR_TYPE_SKB_META ; }else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { dynptr_arg_type |=DYNPTR_TYPE_FILE ; + }else if (meta->func_id == special_kfunc_list[KF_bpf_uring_buf_dynptr] || + meta->func_id == special_kfunc_list[KF_bpf_uring_buf_dynptr_rdwr]) { + dynptr_arg_type |=DYNPTR_TYPE_LOCAL ; }else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { dynptr_arg_type |=DYNPTR_TYPE_FILE ; meta->release_regno = regno;
Signature · 1 line
2.53.0
PATCH v3 10/12 selftests/io_uring: add io_uring_unregister_buffers()
In reply to Ming LeiAdd io_uring_unregister_buffers(), so that kernel selftest can call it explicitly.
--- tools/include/io_uring/mini_liburing.h | 10 ++++++++++ 1 file changed, 10 insertions(+)
tools/include/io_uring/mini_liburing.h +10 -0
@@ -1,6 +1,7 @@/* SPDX-License-Identifier: MIT */ #include <linux/io_uring.h> +#include <signal.h> #include <sys/mman.h> #include <sys/syscall.h> #include <stdio.h>
@@ -284,6 +285,15 @@ static inline int io_uring_register_buffers(struct io_uring *ring,return (ret <0 ) ? -errno : ret; } +static inline int io_uring_unregister_buffers (struct io_uring *ring) +{ +int ret; + + ret =syscall (__NR_io_uring_register, ring->ring_fd, +IORING_UNREGISTER_BUFFERS ,NULL ,0 ); +return (ret <0 ) ? -errno : ret; +} +static inline void io_uring_prep_send (struct io_uring_sqe *sqe,int sockfd,const void *buf,size_t len,int flags) {
Signature · 1 line
2.53.0
PATCH v3 11/12 selftests/io_uring: add BPF struct_ops and kfunc tests
In reply to Ming LeiAdd selftests for io_uring BPF struct_ops and kfunc functionality:
- bpf_ext_basic: Tests IORING_OP_BPF struct_ops registration and execution with multiple struct_ops support
The test framework includes:
- runner.c: Main test runner with auto-discovery
- iou_test.h: Common test infrastructure
- Makefile: Build system with BPF skeleton generation
--- tools/testing/selftests/io_uring/.gitignore | 2 + tools/testing/selftests/io_uring/Makefile | 173 ++++++++++++++ .../selftests/io_uring/bpf_ext_basic.bpf.c | 94 ++++++++ .../selftests/io_uring/bpf_ext_basic.c | 215 ++++++++++++++++++ .../selftests/io_uring/include/iou_test.h | 98 ++++++++ tools/testing/selftests/io_uring/runner.c | 206 +++++++++++++++++ 6 files changed, 788 insertions(+) create mode 100644 tools/testing/selftests/io_uring/.gitignore create mode 100644 tools/testing/selftests/io_uring/Makefile create mode 100644 tools/testing/selftests/io_uring/bpf_ext_basic.bpf.c create mode 100644 tools/testing/selftests/io_uring/bpf_ext_basic.c create mode 100644 tools/testing/selftests/io_uring/include/iou_test.h create mode 100644 tools/testing/selftests/io_uring/runner.c
tools/testing/selftests/io_uring/.gitignore new file +2 -0
@@ -0,0 +1,2 @@ +/build/ +/runner
tools/testing/selftests/io_uring/Makefile new file +173 -0
173 lines not on this page · raw message
tools/testing/selftests/io_uring/bpf_ext_basic.bpf.c new file +94 -0
94 lines not on this page · raw message
tools/testing/selftests/io_uring/bpf_ext_basic.c new file +215 -0
215 lines not on this page · raw message
tools/testing/selftests/io_uring/include/iou_test.h new file +98 -0
98 lines not on this page · raw message
tools/testing/selftests/io_uring/runner.c new file +206 -0
206 lines not on this page · raw message
Signature · 1 line
2.53.0
PATCH v3 12/12 selftests/io_uring: add buffer iterator selftest with BPF arena
In reply to Ming LeiAdd bpf_ext_memcpy test for per-buffer iterator kfuncs with dynptr. Tests both read (buffer -> arena) and write (arena -> buffer) directions across all buffer types (USER, VEC, FIXED, REG_VEC) with 1MB+ buffers.
--- tools/testing/selftests/Makefile | 3 +- .../selftests/io_uring/bpf_ext_memcpy.bpf.c | 305 +++++++++++ .../selftests/io_uring/bpf_ext_memcpy.c | 517 ++++++++++++++++++ .../io_uring/include/bpf_ext_memcpy_defs.h | 18 + 4 files changed, 842 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/io_uring/bpf_ext_memcpy.bpf.c create mode 100644 tools/testing/selftests/io_uring/bpf_ext_memcpy.c create mode 100644 tools/testing/selftests/io_uring/include/bpf_ext_memcpy_defs.h
tools/testing/selftests/Makefile +2 -1
@@ -45,6 +45,7 @@ TARGETS += futex TARGETS += gpio TARGETS += hid TARGETS += intel_pstate +TARGETS += io_uring TARGETS += iommu TARGETS += ipc TARGETS += ir
@@ -148,7 +149,7 @@ endif # User can optionally provide a TARGETS skiplist. By default we skip # targets using BPF since it has cutting edge build time dependencies # which require more effort to install. -SKIP_TARGETS ?= bpf sched_ext +SKIP_TARGETS ?= bpf io_uring sched_ext ifneq ($(SKIP_TARGETS),) TMP := $(filter-out $(SKIP_TARGETS), $(TARGETS)) override TARGETS := $(TMP)
tools/testing/selftests/io_uring/bpf_ext_memcpy.bpf.c new file +305 -0
305 lines not on this page · raw message
tools/testing/selftests/io_uring/bpf_ext_memcpy.c new file +517 -0
517 lines not on this page · raw message
tools/testing/selftests/io_uring/include/bpf_ext_memcpy_defs.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef BPF_MEMCPY_DEFS_H +#define BPF_MEMCPY_DEFS_H + +/* +* Shared definitions between bpf_memcpy.bpf.c (BPF program) and +* bpf_memcpy.c (userspace test). +* +* Buffer size must be CHUNK_SIZE-aligned so every dynptr slice is a +* clean multiple of CHUNK_SIZE with no partial tail. +*/ +#define CHUNK_SIZE 512 +/* Must be divisible by CHUNK_SIZE and nr_vecs (5). +* 512 * 2050 = 1049600, 1049600 / 5 = 209920 = 512 * 410. +*/ +#define TEST_BUF_SIZE (CHUNK_SIZE * 2050)/* ~1MB */ + +#endif /* BPF_MEMCPY_DEFS_H */
Signature · 1 line
2.53.0