add basic zero copy receive support
v2 cover add basic zero copy receive support
Add basic support for io_uring zero copy receive in liburing. Besides the mandatory syncing of necessary liburing.h headers, add a thin wrapper around the registration op and a unit test.
Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct io_uring_zcrx_rq.
In the future, I'll add code to hide the implementation details. But for now, this unblocks the kernel selftest.
Changes in v2:
--------------
* Split out linux/io_uring.h header sync changes
* Move exporting io_uring_register_ifq() to LIBURING_2.10 section
David Wei (4):
liburing: sync io_uring headers
zcrx: sync io_uring headers
zcrx: add basic support
zcrx: add unit test
src/include/liburing.h | 12 + src/include/liburing/io_uring.h | 62 ++- src/liburing-ffi.map | 2 + src/liburing.map | 2 + src/register.c | 6 + test/Makefile | 1 + test/zcrx.c | 918 ++++++++++++++++++++++++++++++++ 7 files changed, 1001 insertions(+), 2 deletions(-) create mode 100644 test/zcrx.c
Signature · 1 line
2.43.5
PATCH v2 1/4 liburing: sync io_uring headers
In preparation of syncing headers w/ zero copy rx changes, first sync other unrelated changes.
--- src/include/liburing/io_uring.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
src/include/liburing/io_uring.h +6 -2
@@ -364,7 +364,7 @@ enum io_uring_op {* result will be the number of buffers send, with * the starting buffer ID in cqe->flags as per * usual for provided buffer usage. The buffers -* will be contiguous from the starting buffer ID. +* will be contigious from the starting buffer ID. */ #define IORING_RECVSEND_POLL_FIRST (1U << 0)#define IORING_RECV_MULTISHOT (1U << 1)
@@ -424,7 +424,7 @@ enum io_uring_msg_ring_flags {* IO completion data structure (Completion Queue Entry) */ struct io_uring_cqe { -__u64 user_data;/* sqe->user_data submission passed back */ +__u64 user_data;/* sqe->user_data value passed back */ __s32 res;/* result code for this event */ __u32 flags;
@@ -616,6 +616,10 @@ enum io_uring_register_op {/* clone registered buffers from source ring to current ring */ IORING_REGISTER_CLONE_BUFFERS =30 , +/* send MSG_RING without having a ring */ +IORING_REGISTER_SEND_MSG_RING =31 , + +/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS =33 , IORING_REGISTER_MEM_REGION = 34,
Signature · 1 line
2.43.5
PATCH v2 2/4 zcrx: sync io_uring headers
In reply to David WeiSync linux/io_uring.h with zcrx changes.
--- src/include/liburing/io_uring.h | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+)
src/include/liburing/io_uring.h +54 -0
@@ -87,6 +87,7 @@ struct io_uring_sqe {union {__s32 splice_fd_in;__u32 file_index; +__u32 zcrx_ifq_idx;__u32 optlen;struct {__u16 addr_len;
@@ -262,6 +263,7 @@ enum io_uring_op {IORING_OP_FTRUNCATE ,IORING_OP_BIND ,IORING_OP_LISTEN , +IORING_OP_RECV_ZC ,/* this goes last, obviously */ IORING_OP_LAST,
@@ -619,6 +621,9 @@ enum io_uring_register_op {/* send MSG_RING without having a ring */ IORING_REGISTER_SEND_MSG_RING =31 , +/* register a netdev hw rx queue for zerocopy */ +IORING_REGISTER_ZCRX_IFQ =32 , +/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS = 33,
@@ -920,6 +925,55 @@ enum io_uring_socket_op { SOCKET_URING_OP_SETSOCKOPT, }; +/* Zero copy receive refill queue entry */ +struct io_uring_zcrx_rqe { +__u64 off; +__u32 len; +__u32 __pad; +}; + +struct io_uring_zcrx_cqe { +__u64 off; +__u64 __pad; +}; + +/* The bit from which area id is encoded into offsets */ +#define IORING_ZCRX_AREA_SHIFT 48 +#define IORING_ZCRX_AREA_MASK (~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1)) + +struct io_uring_zcrx_offsets { +__u32 head; +__u32 tail; +__u32 rqes; +__u32 __resv2; +__u64 __resv[2 ]; +}; + +struct io_uring_zcrx_area_reg { +__u64 addr; +__u64 len; +__u64 rq_area_token; +__u32 flags; +__u32 __resv1; +__u64 __resv2[2 ]; +}; + +/* +* Argument for IORING_REGISTER_ZCRX_IFQ +*/ +struct io_uring_zcrx_ifq_reg { +__u32 if_idx; +__u32 if_rxq; +__u32 rq_entries; +__u32 flags; + +__u64 area_ptr;/* pointer to struct io_uring_zcrx_area_reg */ +__u64 region_ptr;/* struct io_uring_region_desc * */ + +struct io_uring_zcrx_offsets offsets; +__u64 __resv[4 ]; +}; + #ifdef __cplusplus }#endif
Signature · 1 line
2.43.5
PATCH v2 3/4 zcrx: add basic support
In reply to David WeiAdd basic support for zcrx with a thin wrapper around IORING_REGISTER_ZCRX_IFQ and a struct for the refill queue.
--- src/include/liburing.h | 12 ++++++++++++ src/liburing-ffi.map | 2 ++ src/liburing.map | 2 ++ src/register.c | 6 ++++++ 4 files changed, 22 insertions(+)
src/include/liburing.h +12 -0
@@ -132,6 +132,16 @@ struct io_uring {unsigned pad2; }; +struct io_uring_zcrx_rq { +__u32 *khead; +__u32 *ktail; +__u32 rq_tail; +unsigned ring_entries; + +struct io_uring_zcrx_rqe *rqes; +void *ring_ptr; +}; +/* * Library interface */
@@ -265,6 +275,8 @@ int io_uring_register_file_alloc_range(struct io_uring *ring,int io_uring_register_napi (struct io_uring *ring,struct io_uring_napi *napi);int io_uring_unregister_napi (struct io_uring *ring,struct io_uring_napi *napi); +int io_uring_register_ifq (struct io_uring *ring, +struct io_uring_zcrx_ifq_reg *reg);int io_uring_register_clock (struct io_uring *ring,struct io_uring_clock_register *arg);
src/liburing-ffi.map +2 -0
@@ -227,4 +227,6 @@ LIBURING_2.9 { } LIBURING_2.8; LIBURING_2.10 { + global: + io_uring_register_ifq; } LIBURING_2.9;
src/liburing.map +2 -0
@@ -113,4 +113,6 @@ LIBURING_2.9 { } LIBURING_2.8; LIBURING_2.10 { + global: + io_uring_register_ifq; } LIBURING_2.9;
src/register.c +6 -0
@@ -422,6 +422,12 @@ int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)return io_uring_clone_buffers_offset (dst, src,0 ,0 ,0 ,0 ); } +int io_uring_register_ifq (struct io_uring *ring, +struct io_uring_zcrx_ifq_reg *reg) +{ +return do_register (ring,IORING_REGISTER_ZCRX_IFQ , reg,1 ); +} +int io_uring_resize_rings (struct io_uring *ring,struct io_uring_params *p) {unsigned sq_head, sq_tail;
Signature · 1 line
2.43.5
PATCH v2 4/4 zcrx: add unit test
In reply to David WeiTests for registration and a basic recv test. No zero copy is actually happening but it does test the copy fallback.
--- test/Makefile | 1 + test/zcrx.c | 918 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 919 insertions(+) create mode 100644 test/zcrx.c
test/Makefile +1 -0
@@ -239,6 +239,7 @@ test_srcs := \ wakeup-hang.c \ wq-aff.c \ xattr.c \ + zcrx.c \ # EOL # Please keep this list sorted alphabetically.
test/zcrx.c new file +918 -0
918 lines not on this page · raw message
Signature · 1 line
2.43.5
Quoted 9 lines · David Wei
Add basic support for
io_uringzero copy receive in liburing. Besides the mandatory syncing of necessaryliburing.hheaders, add a thin wrapper around the registration op and a unit test.Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct
io_uring_zcrx_rq.[...]
Applied, thanks!
[1/4] liburing: sync io_uring headers
commit: 48d8d54e524a9c37e0cc52921bb41070156a597f
[2/4] zcrx: sync io_uring headers
commit: ce3a65747d43a405cc19a630d5f8a0f613293f5c
[3/4] zcrx: add basic support
commit: d7ec4ce3421fbbdaba07426d589b72e204ac92e9
[4/4] zcrx: add unit test
commit: 30bd7baa5627f066fc0bfbca293f950fdf3ef3f1Best regards,
Signature · 1 line
Jens Axboe