add basic zero copy receive support
cover add basic zero copy receive support
Add basic support for io_uring zero copy receive in liburing. Besides the mandatory syncing of necessary liburing.h headers, add a thin wrapper around the registration op and a unit test.
Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct io_uring_zcrx_rq.
In the future, I'll add code to hide the implementation details. But for now, this unblocks the kernel selftest.
David Wei (3):
zcrx: sync kernel headers
zcrx: add basic support
zcrx: add unit test
src/include/liburing.h | 12 + src/include/liburing/io_uring.h | 62 ++- src/liburing-ffi.map | 1 + src/liburing.map | 1 + src/register.c | 6 + test/Makefile | 1 + test/zcrx.c | 918 ++++++++++++++++++++++++++++++++ 7 files changed, 999 insertions(+), 2 deletions(-) create mode 100644 test/zcrx.c
Signature · 1 line
2.43.5
PATCH 1/3 zcrx: sync kernel headers
Sync linux/io_uring.h with zcrx changes.
--- src/include/liburing/io_uring.h | 62 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-)
src/include/liburing/io_uring.h +60 -2
@@ -87,6 +87,7 @@ struct io_uring_sqe {union {__s32 splice_fd_in;__u32 file_index; +__u32 zcrx_ifq_idx;__u32 optlen;struct {__u16 addr_len;
@@ -262,6 +263,7 @@ enum io_uring_op {IORING_OP_FTRUNCATE ,IORING_OP_BIND ,IORING_OP_LISTEN , +IORING_OP_RECV_ZC ,/* this goes last, obviously */ IORING_OP_LAST,
@@ -364,7 +366,7 @@ enum io_uring_op {* result will be the number of buffers send, with * the starting buffer ID in cqe->flags as per * usual for provided buffer usage. The buffers -* will be contiguous from the starting buffer ID. +* will be contigious from the starting buffer ID. */ #define IORING_RECVSEND_POLL_FIRST (1U << 0)#define IORING_RECV_MULTISHOT (1U << 1)
@@ -424,7 +426,7 @@ enum io_uring_msg_ring_flags {* IO completion data structure (Completion Queue Entry) */ struct io_uring_cqe { -__u64 user_data;/* sqe->user_data submission passed back */ +__u64 user_data;/* sqe->user_data value passed back */ __s32 res;/* result code for this event */ __u32 flags;
@@ -616,6 +618,13 @@ enum io_uring_register_op {/* clone registered buffers from source ring to current ring */ IORING_REGISTER_CLONE_BUFFERS =30 , +/* send MSG_RING without having a ring */ +IORING_REGISTER_SEND_MSG_RING =31 , + +/* register a netdev hw rx queue for zerocopy */ +IORING_REGISTER_ZCRX_IFQ =32 , + +/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS =33 , IORING_REGISTER_MEM_REGION = 34,
@@ -916,6 +925,55 @@ enum io_uring_socket_op { SOCKET_URING_OP_SETSOCKOPT, }; +/* Zero copy receive refill queue entry */ +struct io_uring_zcrx_rqe { +__u64 off; +__u32 len; +__u32 __pad; +}; + +struct io_uring_zcrx_cqe { +__u64 off; +__u64 __pad; +}; + +/* The bit from which area id is encoded into offsets */ +#define IORING_ZCRX_AREA_SHIFT 48 +#define IORING_ZCRX_AREA_MASK (~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1)) + +struct io_uring_zcrx_offsets { +__u32 head; +__u32 tail; +__u32 rqes; +__u32 __resv2; +__u64 __resv[2 ]; +}; + +struct io_uring_zcrx_area_reg { +__u64 addr; +__u64 len; +__u64 rq_area_token; +__u32 flags; +__u32 __resv1; +__u64 __resv2[2 ]; +}; + +/* +* Argument for IORING_REGISTER_ZCRX_IFQ +*/ +struct io_uring_zcrx_ifq_reg { +__u32 if_idx; +__u32 if_rxq; +__u32 rq_entries; +__u32 flags; + +__u64 area_ptr;/* pointer to struct io_uring_zcrx_area_reg */ +__u64 region_ptr;/* struct io_uring_region_desc * */ + +struct io_uring_zcrx_offsets offsets; +__u64 __resv[4 ]; +}; + #ifdef __cplusplus }#endif
Signature · 1 line
2.43.5
Jens Axboe
maintainerRe: [PATCH liburing v1 1/3] zcrx: sync kernel headers
Quoted 1 line · David Wei
Sync
linux/io_uring.hwith zcrx changes.
Some of these hunks seem like generic changes from io_uring.h not related to the zcrx work? Not a huge deal, but would be nice if they were split from the main change.
Signature · 1 line
Jens Axboe
Re: [PATCH liburing v1 1/3] zcrx: sync kernel headers
Quoted 7 lines · Jens Axboe
Quoted 1 line · David Wei
Sync
linux/io_uring.hwith zcrx changes.Some of these hunks seem like generic changes from
io_uring.hnot related to the zcrx work? Not a huge deal, but would be nice if they were split from the main change.
No problem, I'll split them out.
PATCH 2/3 zcrx: add basic support
In reply to David WeiAdd basic support for zcrx with a thin wrapper around IORING_REGISTER_ZCRX_IFQ and a struct for the refill queue.
--- src/include/liburing.h | 12 ++++++++++++ src/liburing-ffi.map | 1 + src/liburing.map | 1 + src/register.c | 6 ++++++ 4 files changed, 20 insertions(+)
src/include/liburing.h +12 -0
@@ -132,6 +132,16 @@ struct io_uring {unsigned pad2; }; +struct io_uring_zcrx_rq { +__u32 *khead; +__u32 *ktail; +__u32 rq_tail; +unsigned ring_entries; + +struct io_uring_zcrx_rqe *rqes; +void *ring_ptr; +}; +/* * Library interface */
@@ -265,6 +275,8 @@ int io_uring_register_file_alloc_range(struct io_uring *ring,int io_uring_register_napi (struct io_uring *ring,struct io_uring_napi *napi);int io_uring_unregister_napi (struct io_uring *ring,struct io_uring_napi *napi); +int io_uring_register_ifq (struct io_uring *ring, +struct io_uring_zcrx_ifq_reg *reg);int io_uring_register_clock (struct io_uring *ring,struct io_uring_clock_register *arg);
src/liburing-ffi.map +1 -0
@@ -222,6 +222,7 @@ LIBURING_2.9 { io_uring_register_wait_reg; io_uring_submit_and_wait_reg; io_uring_clone_buffers_offset; + io_uring_register_ifq; io_uring_register_region; io_uring_sqe_set_buf_group; } LIBURING_2.8;
src/liburing.map +1 -0
@@ -109,5 +109,6 @@ LIBURING_2.9 { io_uring_register_wait_reg; io_uring_submit_and_wait_reg; io_uring_clone_buffers_offset; + io_uring_register_ifq; io_uring_register_region; } LIBURING_2.8;
src/register.c +6 -0
@@ -422,6 +422,12 @@ int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)return io_uring_clone_buffers_offset (dst, src,0 ,0 ,0 ,0 ); } +int io_uring_register_ifq (struct io_uring *ring, +struct io_uring_zcrx_ifq_reg *reg) +{ +return do_register (ring,IORING_REGISTER_ZCRX_IFQ , reg,1 ); +} +int io_uring_resize_rings (struct io_uring *ring,struct io_uring_params *p) {unsigned sq_head, sq_tail;
Signature · 1 line
2.43.5
Jens Axboe
maintainerRe: [PATCH liburing v1 2/3] zcrx: add basic support
Quoted 17 lines · David Wei
src/liburing-ffi.map +1
-0@@ -222,6 +222,7 @@ LIBURING_2.9 { io_uring_register_wait_reg; io_uring_submit_and_wait_reg; io_uring_clone_buffers_offset; + io_uring_register_ifq; io_uring_register_region; io_uring_sqe_set_buf_group; } LIBURING_2.8;src/liburing.map +0
-0
src/liburing.map LIBURING_2.9 { · line 114
Quoted 6 lines
src/liburing.map +1
-0@@ -109,5 +109,6 @@ LIBURING_2.9 { io_uring_register_wait_reg; io_uring_submit_and_wait_reg; io_uring_clone_buffers_offset; + io_uring_register_ifq; io_uring_register_region; } LIBURING_2.8;
This isn't right - 2.9 has already been released, you can't add new symbols to an existing release. I usually bump the version post release, but looks like I didn't this time - corrected now, with a new empty section for 2.10 symbols as well where these should go.
Signature · 1 line
Jens Axboe
Re: [PATCH liburing v1 2/3] zcrx: add basic support
Quoted 21 lines · Jens Axboe
Quoted 20 lines · David Wei
src/liburing-ffi.map +1
-0@@ -222,6 +222,7 @@ LIBURING_2.9 { io_uring_register_wait_reg; io_uring_submit_and_wait_reg; io_uring_clone_buffers_offset; + io_uring_register_ifq; io_uring_register_region; io_uring_sqe_set_buf_group; } LIBURING_2.8;src/liburing.map +0
-0@@ -109,5 +109,6 @@ LIBURING_2.9 { io_uring_register_wait_reg; io_uring_submit_and_wait_reg; io_uring_clone_buffers_offset;
Quoted 9 lines
src/liburing.map LIBURING_2.9 { · line 114
Quoted 3 lines
src/liburing.map +1
-0+ io_uring_register_ifq; io_uring_register_region; } LIBURING_2.8;This isn't right - 2.9 has already been released, you can't add new symbols to an existing release. I usually bump the version post release, but looks like I didn't this time - corrected now, with a new empty section for 2.10 symbols as well where these should go.
Thanks, I'll rebase and add to 2.10.
PATCH 3/3 zcrx: add unit test
In reply to David WeiTests for registration and a basic recv test. No zero copy is actually happening but it does test the copy fallback.
--- test/Makefile | 1 + test/zcrx.c | 918 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 919 insertions(+) create mode 100644 test/zcrx.c
test/Makefile +1 -0
@@ -239,6 +239,7 @@ test_srcs := \ wakeup-hang.c \ wq-aff.c \ xattr.c \ + zcrx.c \ # EOL # Please keep this list sorted alphabetically.
test/zcrx.c new file +918 -0
918 lines not on this page · raw message
Signature · 1 line
2.43.5
Jens Axboe
maintainerQuoted 10 lines · David Wei
Add basic support for
io_uringzero copy receive in liburing. Besides the mandatory syncing of necessaryliburing.hheaders, add a thin wrapper around the registration op and a unit test.Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct
io_uring_zcrx_rq.In the future, I'll add code to hide the implementation details. But for now, this unblocks the kernel selftest.
man pages coming for this too?
Signature · 1 line
Jens Axboe
Quoted 5 lines · Jens Axboe
Quoted 4 lines · David Wei
Add basic support for
io_uringzero copy receive in liburing. Besides the mandatory syncing of necessaryliburing.hheaders, add a thin wrapper around the registration op and a unit test.
Quoted 9 lines
Quoted 6 lines
Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct
io_uring_zcrx_rq.In the future, I'll add code to hide the implementation details. But for now, this unblocks the kernel selftest.
man pages coming for this too?
I'm working on higher level abstractions so that users hopefully won't need to call it directly. Could I defer manpages until these take shape?
Jens Axboe
maintainerQuoted 10 lines · David Wei
Quoted 9 lines · Jens Axboe
Quoted 8 lines · David Wei
Add basic support for
io_uringzero copy receive in liburing. Besides the mandatory syncing of necessaryliburing.hheaders, add a thin wrapper around the registration op and a unit test.Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct
io_uring_zcrx_rq.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
In the future, I'll add code to hide the implementation details. But for now, this unblocks the kernel selftest.
man pages coming for this too?
I'm working on higher level abstractions so that users hopefully won't need to call it directly. Could I defer manpages until these take shape?
Yep that's fine, thanks!
Signature · 1 line
Jens Axboe