Skip to contentSkip to conversation

add basic zero copy receive support

PATCH Applied David Wei 2 participants 6 messages Feb 18, 2025 io_uring

lore
Share & tools
lore

David Wei

author
#0 lore raw

v2 cover add basic zero copy receive support

Add basic support for io_uring zero copy receive in liburing. Besides the mandatory syncing of necessary liburing.h headers, add a thin wrapper around the registration op and a unit test.

Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct io_uring_zcrx_rq.

In the future, I'll add code to hide the implementation details. But for now, this unblocks the kernel selftest.

Changes in v2:
--------------
* Split out linux/io_uring.h header sync changes
* Move exporting io_uring_register_ifq() to LIBURING_2.10 section

David Wei (4):
liburing: sync io_uring headers
zcrx: sync io_uring headers
zcrx: add basic support
zcrx: add unit test

 src/include/liburing.h          |  12 +
 src/include/liburing/io_uring.h |  62 ++-
 src/liburing-ffi.map            |   2 +
 src/liburing.map                |   2 +
 src/register.c                  |   6 +
 test/Makefile                   |   1 +
 test/zcrx.c                     | 918 ++++++++++++++++++++++++++++++++
 7 files changed, 1001 insertions(+), 2 deletions(-)
 create mode 100644 test/zcrx.c
Signature · 1 line
2.43.5
author
#1 lore raw

PATCH v2 1/4 liburing: sync io_uring headers

In preparation of syncing headers w/ zero copy rx changes, first sync other unrelated changes.

---
 src/include/liburing/io_uring.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
src/include/liburing/io_uring.h +6 -2
index 765919883cff..452240a6ebb4 100644
@@ -364,7 +364,7 @@ enum io_uring_op {
  *				result 	will be the number of buffers send, with
  *				the starting buffer ID in cqe->flags as per
  *				usual for provided buffer usage. The buffers
- *				will be contiguous from the starting buffer ID.
+ *				will be	contigious from the starting buffer ID.
  */
 #define IORING_RECVSEND_POLL_FIRST	(1U << 0)
 #define IORING_RECV_MULTISHOT		(1U << 1)
@@ -424,7 +424,7 @@ enum io_uring_msg_ring_flags {
  * IO completion data structure (Completion Queue Entry)
  */
 struct io_uring_cqe {
-	__u64	user_data;	/* sqe->user_data submission passed back */
+	__u64	user_data;	/* sqe->user_data value passed back */
 	__s32	res;		/* result code for this event */
 	__u32	flags;
 
@@ -616,6 +616,10 @@ enum io_uring_register_op {
 	/* clone registered buffers from source ring to current ring */
 	IORING_REGISTER_CLONE_BUFFERS		= 30,
 
+	/* send MSG_RING without having a ring */
+	IORING_REGISTER_SEND_MSG_RING		= 31,
+
+	/* resize CQ ring */
 	IORING_REGISTER_RESIZE_RINGS		= 33,
 
 	IORING_REGISTER_MEM_REGION		= 34,
Signature · 1 line
2.43.5
author
#2 lore raw

PATCH v2 2/4 zcrx: sync io_uring headers

In reply to David Wei

Sync linux/io_uring.h with zcrx changes.

---
 src/include/liburing/io_uring.h | 54 +++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
src/include/liburing/io_uring.h +54 -0
index 452240a6ebb4..d2fcd1d22ea0 100644
@@ -87,6 +87,7 @@ struct io_uring_sqe {
 	union {
 		__s32	splice_fd_in;
 		__u32	file_index;
+		__u32	zcrx_ifq_idx;
 		__u32	optlen;
 		struct {
 			__u16	addr_len;
@@ -262,6 +263,7 @@ enum io_uring_op {
 	IORING_OP_FTRUNCATE,
 	IORING_OP_BIND,
 	IORING_OP_LISTEN,
+	IORING_OP_RECV_ZC,
 
 	/* this goes last, obviously */
 	IORING_OP_LAST,
@@ -619,6 +621,9 @@ enum io_uring_register_op {
 	/* send MSG_RING without having a ring */
 	IORING_REGISTER_SEND_MSG_RING		= 31,
 
+	/* register a netdev hw rx queue for zerocopy */
+	IORING_REGISTER_ZCRX_IFQ		= 32,
+
 	/* resize CQ ring */
 	IORING_REGISTER_RESIZE_RINGS		= 33,
 
@@ -920,6 +925,55 @@ enum io_uring_socket_op {
 	SOCKET_URING_OP_SETSOCKOPT,
 };
 
+/* Zero copy receive refill queue entry */
+struct io_uring_zcrx_rqe {
+	__u64	off;
+	__u32	len;
+	__u32	__pad;
+};
+
+struct io_uring_zcrx_cqe {
+	__u64	off;
+	__u64	__pad;
+};
+
+/* The bit from which area id is encoded into offsets */
+#define IORING_ZCRX_AREA_SHIFT	48
+#define IORING_ZCRX_AREA_MASK	(~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1))
+
+struct io_uring_zcrx_offsets {
+	__u32	head;
+	__u32	tail;
+	__u32	rqes;
+	__u32	__resv2;
+	__u64	__resv[2];
+};
+
+struct io_uring_zcrx_area_reg {
+	__u64	addr;
+	__u64	len;
+	__u64	rq_area_token;
+	__u32	flags;
+	__u32	__resv1;
+	__u64	__resv2[2];
+};
+
+/*
+ * Argument for IORING_REGISTER_ZCRX_IFQ
+ */
+struct io_uring_zcrx_ifq_reg {
+	__u32	if_idx;
+	__u32	if_rxq;
+	__u32	rq_entries;
+	__u32	flags;
+
+	__u64	area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
+	__u64	region_ptr; /* struct io_uring_region_desc * */
+
+	struct io_uring_zcrx_offsets offsets;
+	__u64	__resv[4];
+};
+
 #ifdef __cplusplus
 }
 #endif
Signature · 1 line
2.43.5
author
#3 lore raw

PATCH v2 3/4 zcrx: add basic support

In reply to David Wei

Add basic support for zcrx with a thin wrapper around IORING_REGISTER_ZCRX_IFQ and a struct for the refill queue.

---
 src/include/liburing.h | 12 ++++++++++++
 src/liburing-ffi.map   |  2 ++
 src/liburing.map       |  2 ++
 src/register.c         |  6 ++++++
 4 files changed, 22 insertions(+)
src/include/liburing.h +12 -0
index 49b4edf437b2..6393599cb3bf 100644
@@ -132,6 +132,16 @@ struct io_uring {
 	unsigned pad2;
 };
 
+struct io_uring_zcrx_rq {
+	__u32 *khead;
+	__u32 *ktail;
+	__u32 rq_tail;
+	unsigned ring_entries;
+
+	struct io_uring_zcrx_rqe *rqes;
+	void *ring_ptr;
+};
+
 /*
  * Library interface
  */
@@ -265,6 +275,8 @@ int io_uring_register_file_alloc_range(struct io_uring *ring,
 
 int io_uring_register_napi(struct io_uring *ring, struct io_uring_napi *napi);
 int io_uring_unregister_napi(struct io_uring *ring, struct io_uring_napi *napi);
+int io_uring_register_ifq(struct io_uring *ring,
+			  struct io_uring_zcrx_ifq_reg *reg);
 
 int io_uring_register_clock(struct io_uring *ring,
 			    struct io_uring_clock_register *arg);
src/liburing-ffi.map +2 -0
index 968ccbc67366..fe14adb6d83f 100644
@@ -227,4 +227,6 @@ LIBURING_2.9 {
 } LIBURING_2.8;
 
 LIBURING_2.10 {
+	global:
+		io_uring_register_ifq;
 } LIBURING_2.9;
src/liburing.map +2 -0
index 264a94946e90..d1661d9d61f9 100644
@@ -113,4 +113,6 @@ LIBURING_2.9 {
 } LIBURING_2.8;
 
 LIBURING_2.10 {
+	global:
+		io_uring_register_ifq;
 } LIBURING_2.9;
src/register.c +6 -0
index 0fff208cd5f5..99337d13135d 100644
@@ -422,6 +422,12 @@ int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)
 	return io_uring_clone_buffers_offset(dst, src, 0, 0, 0, 0);
 }
 
+int io_uring_register_ifq(struct io_uring *ring,
+			  struct io_uring_zcrx_ifq_reg *reg)
+{
+	return do_register(ring, IORING_REGISTER_ZCRX_IFQ, reg, 1);
+}
+
 int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p)
 {
 	unsigned sq_head, sq_tail;
Signature · 1 line
2.43.5
author
#4 lore raw

PATCH v2 4/4 zcrx: add unit test

In reply to David Wei

Tests for registration and a basic recv test. No zero copy is actually happening but it does test the copy fallback.

---
 test/Makefile |   1 +
 test/zcrx.c   | 918 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 919 insertions(+)
 create mode 100644 test/zcrx.c
test/Makefile +1 -0
index 64e1480867d4..d92379ad9ed0 100644
@@ -239,6 +239,7 @@ test_srcs := \
 	wakeup-hang.c \
 	wq-aff.c \
 	xattr.c \
+	zcrx.c \
 	# EOL
 
 # Please keep this list sorted alphabetically.
test/zcrx.c new file +918 -0

918 lines not on this page · raw message

Signature · 1 line
2.43.5
maintainer Applied
#5 lore raw
In reply to David Wei
Quoted 9 lines · David Wei

Add basic support for io_uring zero copy receive in liburing. Besides the mandatory syncing of necessary liburing.h headers, add a thin wrapper around the registration op and a unit test.

Users still need to setup by hand e.g. mmap, setup the registration structs, do the registration and then setup the refill queue struct io_uring_zcrx_rq.

[...]

Applied, thanks!

[1/4] liburing: sync io_uring headers

      commit: 48d8d54e524a9c37e0cc52921bb41070156a597f
[2/4] zcrx: sync io_uring headers
      commit: ce3a65747d43a405cc19a630d5f8a0f613293f5c
[3/4] zcrx: add basic support
      commit: d7ec4ce3421fbbdaba07426d589b72e204ac92e9
[4/4] zcrx: add unit test
      commit: 30bd7baa5627f066fc0bfbca293f950fdf3ef3f1

Best regards,

Signature · 1 line
Jens Axboe