Skip to contentSkip to conversation

zcrx refill queue allocation modes

PATCH Applied Pavel Begunkov 2 participants 6 messages Apr 21, 2025 io_uring

lore
Share & tools
lore
#0 lore raw

v2 cover zcrx refill queue allocation modes

Random patches for zcrx. Path 1 removes oneshot mode as it doesn't give a good example, and Patch 3 allows to kernel allocations for the refill queue.

v2: rework oneshot/size limiting instead of removal
fix flipped allocation mode check

Pavel Begunkov (4):
examples/zcrx: consolidate add_recvzc variants examples/zcrx: rework size limiting examples/zcrx: constants for request types examples/zcrx: add refill queue allocation modes

 examples/zcrx.c | 119 +++++++++++++++++++++++++++---------------------
 1 file changed, 68 insertions(+), 51 deletions(-)
Signature · 1 line
2.48.1
#1 lore raw

PATCH v2 1/4 examples/zcrx: consolidate add_recvzc variants

Pass len to add_recvzc, 0 means it's unlimited, and kill the oneshot variant.

---
 examples/zcrx.c | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)
examples/zcrx.c +4 -17
index 32d9e4ae..727943c4 100644
@@ -140,17 +140,7 @@ static void add_accept(struct io_uring *ring, int sockfd)
 	sqe->user_data = 1;
 }
 
-static void add_recvzc(struct io_uring *ring, int sockfd)
-{
-	struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
-
-	io_uring_prep_rw(IORING_OP_RECV_ZC, sqe, sockfd, NULL, 0, 0);
-	sqe->ioprio |= IORING_RECV_MULTISHOT;
-	sqe->zcrx_ifq_idx = zcrx_id;
-	sqe->user_data = 2;
-}
-
-static void add_recvzc_oneshot(struct io_uring *ring, int sockfd, size_t len)
+static void add_recvzc(struct io_uring *ring, int sockfd, size_t len)
 {
 	struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
 
@@ -168,10 +158,7 @@ static void process_accept(struct io_uring *ring, struct io_uring_cqe *cqe)
 		t_error(1, 0, "Unexpected second connection");
 
 	connfd = cqe->res;
-	if (cfg_oneshot)
-		add_recvzc_oneshot(ring, connfd, page_size);
-	else
-		add_recvzc(ring, connfd);
+	add_recvzc(ring, connfd, cfg_oneshot ? page_size : 0);
 }
 
 static void verify_data(char *data, size_t size, unsigned long seq)
@@ -207,11 +194,11 @@ static void process_recvzc(struct io_uring *ring, struct io_uring_cqe *cqe)
 
 	if (cfg_oneshot) {
 		if (cqe->res == 0 && cqe->flags == 0 && cfg_oneshot_recvs) {
-			add_recvzc_oneshot(ring, connfd, page_size);
+			add_recvzc(ring, connfd, page_size);
 			cfg_oneshot_recvs--;
 		}
 	} else if (!(cqe->flags & IORING_CQE_F_MORE)) {
-		add_recvzc(ring, connfd);
+		add_recvzc(ring, connfd, 0);
 	}
 
 	rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1);
Signature · 1 line
2.48.1
#2 lore raw

PATCH v2 2/4 examples/zcrx: rework size limiting

In reply to Pavel Begunkov

zcrx doesn't work in a single shot mode, we can limit the number of bytes it processes but not the number of cqes. Replace cfg_oneshot with total byte size limiting.

---
 examples/zcrx.c | 38 +++++++++++++++-----------------------
 1 file changed, 15 insertions(+), 23 deletions(-)
examples/zcrx.c +15 -23
index 727943c4..5b06bb4c 100644
@@ -46,9 +46,8 @@ static long page_size;
 static int cfg_port = 8000;
 static const char *cfg_ifname;
 static int cfg_queue_id = -1;
-static bool cfg_oneshot;
-static int cfg_oneshot_recvs;
 static bool cfg_verify_data = false;
+static size_t cfg_size = 0;
 static struct sockaddr_in6 cfg_addr;
 
 static void *area_ptr;
@@ -158,7 +157,7 @@ static void process_accept(struct io_uring *ring, struct io_uring_cqe *cqe)
 		t_error(1, 0, "Unexpected second connection");
 
 	connfd = cqe->res;
-	add_recvzc(ring, connfd, cfg_oneshot ? page_size : 0);
+	add_recvzc(ring, connfd, cfg_size);
 }
 
 static void verify_data(char *data, size_t size, unsigned long seq)
@@ -176,7 +175,8 @@ static void verify_data(char *data, size_t size, unsigned long seq)
 	}
 }
 
-static void process_recvzc(struct io_uring *ring, struct io_uring_cqe *cqe)
+static void process_recvzc(struct io_uring __attribute__((unused)) *ring,
+			   struct io_uring_cqe *cqe)
 {
 	unsigned rq_mask = rq_ring.ring_entries - 1;
 	struct io_uring_zcrx_cqe *rcqe;
@@ -184,22 +184,16 @@ static void process_recvzc(struct io_uring *ring, struct io_uring_cqe *cqe)
 	uint64_t mask;
 	char *data;
 
-	if (cqe->res < 0)
-		t_error(1, 0, "recvzc(): %d", cqe->res);
-
-	if (cqe->res == 0 && cqe->flags == 0 && cfg_oneshot_recvs == 0) {
+	if (!(cqe->flags & IORING_CQE_F_MORE)) {
+		if (!cfg_size || cqe->res != 0)
+			t_error(1, 0, "invalid final recvzc ret %i", cqe->res);
+		if (received != cfg_size)
+			t_error(1, 0, "total receive size mismatch %lu / %lu",
+				received, cfg_size);
 		stop = true;
-		return;
-	}
-
-	if (cfg_oneshot) {
-		if (cqe->res == 0 && cqe->flags == 0 && cfg_oneshot_recvs) {
-			add_recvzc(ring, connfd, page_size);
-			cfg_oneshot_recvs--;
-		}
-	} else if (!(cqe->flags & IORING_CQE_F_MORE)) {
-		add_recvzc(ring, connfd, 0);
 	}
+	if (cqe->res < 0)
+		t_error(1, 0, "recvzc(): %d", cqe->res);
 
 	rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1);
 	mask = (1ULL << IORING_ZCRX_AREA_SHIFT) - 1;
@@ -286,7 +280,7 @@ static void parse_opts(int argc, char **argv)
 	if (argc <= 1)
 		usage(argv[0]);
 
-	while ((c = getopt(argc, argv, "vp:i:q:o:")) != -1) {
+	while ((c = getopt(argc, argv, "vp:i:q:s:")) != -1) {
 		switch (c) {
 		case 'p':
 			cfg_port = strtoul(optarg, NULL, 0);
@@ -294,11 +288,9 @@ static void parse_opts(int argc, char **argv)
 		case 'i':
 			cfg_ifname = optarg;
 			break;
-		case 'o': {
-			cfg_oneshot = true;
-			cfg_oneshot_recvs = strtoul(optarg, NULL, 0);
+		case 's':
+			cfg_size = strtoul(optarg, NULL, 0);
 			break;
-		}
 		case 'q':
 			cfg_queue_id = strtoul(optarg, NULL, 0);
 			break;
Signature · 1 line
2.48.1
#3 lore raw

PATCH v2 3/4 examples/zcrx: constants for request types

In reply to Pavel Begunkov

Instead of hard coding user_data, name request types we need and use them.

---
 examples/zcrx.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)
examples/zcrx.c +17 -5
index 5b06bb4c..e5c3c6ec 100644
@@ -43,6 +43,14 @@ static long page_size;
 #define AREA_SIZE (8192 * page_size)
 #define SEND_SIZE (512 * 4096)
 
+#define REQ_TYPE_SHIFT	3
+#define REQ_TYPE_MASK	((1UL << REQ_TYPE_SHIFT) - 1)
+
+enum request_type {
+	REQ_TYPE_ACCEPT		= 1,
+	REQ_TYPE_RX		= 2,
+};
+
 static int cfg_port = 8000;
 static const char *cfg_ifname;
 static int cfg_queue_id = -1;
@@ -136,7 +144,7 @@ static void add_accept(struct io_uring *ring, int sockfd)
 	struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
 
 	io_uring_prep_accept(sqe, sockfd, NULL, NULL, 0);
-	sqe->user_data = 1;
+	sqe->user_data = REQ_TYPE_ACCEPT;
 }
 
 static void add_recvzc(struct io_uring *ring, int sockfd, size_t len)
@@ -146,7 +154,7 @@ static void add_recvzc(struct io_uring *ring, int sockfd, size_t len)
 	io_uring_prep_rw(IORING_OP_RECV_ZC, sqe, sockfd, NULL, len, 0);
 	sqe->ioprio |= IORING_RECV_MULTISHOT;
 	sqe->zcrx_ifq_idx = zcrx_id;
-	sqe->user_data = 2;
+	sqe->user_data = REQ_TYPE_RX;
 }
 
 static void process_accept(struct io_uring *ring, struct io_uring_cqe *cqe)
@@ -217,12 +225,16 @@ static void server_loop(struct io_uring *ring)
 	io_uring_submit_and_wait(ring, 1);
 
 	io_uring_for_each_cqe(ring, head, cqe) {
-		if (cqe->user_data == 1)
+		switch (cqe->user_data & REQ_TYPE_MASK) {
+		case REQ_TYPE_ACCEPT:
 			process_accept(ring, cqe);
-		else if (cqe->user_data == 2)
+			break;
+		case REQ_TYPE_RX:
 			process_recvzc(ring, cqe);
-		else
+			break;
+		default:
 			t_error(1, 0, "unknown cqe");
+		}
 		count++;
 	}
 	io_uring_cq_advance(ring, count);
Signature · 1 line
2.48.1
#4 lore raw

PATCH v2 4/4 examples/zcrx: add refill queue allocation modes

In reply to Pavel Begunkov

Refill queue creating is backed by the region api, which can either be user or kernel allocated. Add an option to switch between the modes.

---
 examples/zcrx.c | 46 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 10 deletions(-)
examples/zcrx.c +36 -10
index e5c3c6ec..40a4f0ad 100644
@@ -39,6 +39,13 @@
 #include "liburing.h"
 #include "helpers.h"
 
+enum {
+	RQ_ALLOC_USER,
+	RQ_ALLOC_KERNEL,
+
+	__RQ_ALLOC_MAX,
+};
+
 static long page_size;
 #define AREA_SIZE (8192 * page_size)
 #define SEND_SIZE (512 * 4096)
@@ -56,6 +63,7 @@ static const char *cfg_ifname;
 static int cfg_queue_id = -1;
 static bool cfg_verify_data = false;
 static size_t cfg_size = 0;
+static unsigned cfg_rq_alloc_mode = RQ_ALLOC_USER;
 static struct sockaddr_in6 cfg_addr;
 
 static void *area_ptr;
@@ -80,6 +88,7 @@ static void setup_zcrx(struct io_uring *ring)
 {
 	unsigned int ifindex;
 	unsigned int rq_entries = 4096;
+	unsigned rq_flags = 0;
 	int ret;
 
 	ifindex = if_nametoindex(cfg_ifname);
@@ -96,19 +105,22 @@ static void setup_zcrx(struct io_uring *ring)
 		t_error(1, 0, "mmap(): zero copy area");
 
 	ring_size = get_refill_ring_size(rq_entries);
-	ring_ptr = mmap(NULL,
-			ring_size,
-			PROT_READ | PROT_WRITE,
-			MAP_ANONYMOUS | MAP_PRIVATE,
-			0,
-			0);
-	if (ring_ptr == MAP_FAILED)
-		t_error(1, 0, "mmap(): refill ring");
+
+	ring_ptr = NULL;
+	if (cfg_rq_alloc_mode == RQ_ALLOC_USER) {
+		ring_ptr = mmap(NULL, ring_size,
+				PROT_READ | PROT_WRITE,
+				MAP_ANONYMOUS | MAP_PRIVATE,
+				0, 0);
+		if (ring_ptr == MAP_FAILED)
+			t_error(1, 0, "mmap(): refill ring");
+		rq_flags |= IORING_MEM_REGION_TYPE_USER;
+	}
 
 	struct io_uring_region_desc region_reg = {
 		.size = ring_size,
 		.user_addr = (__u64)(unsigned long)ring_ptr,
-		.flags = IORING_MEM_REGION_TYPE_USER,
+		.flags = rq_flags,
 	};
 
 	struct io_uring_zcrx_area_reg area_reg = {
@@ -129,6 +141,15 @@ static void setup_zcrx(struct io_uring *ring)
 	if (ret)
 		t_error(1, 0, "io_uring_register_ifq(): %d", ret);
 
+	if (cfg_rq_alloc_mode == RQ_ALLOC_KERNEL) {
+		ring_ptr = mmap(NULL, ring_size,
+				PROT_READ | PROT_WRITE,
+				MAP_SHARED | MAP_POPULATE,
+				ring->ring_fd, region_reg.mmap_offset);
+		if (ring_ptr == MAP_FAILED)
+			t_error(1, 0, "mmap(): refill ring");
+	}
+
 	rq_ring.khead = (unsigned int *)((char *)ring_ptr + reg.offsets.head);
 	rq_ring.ktail = (unsigned int *)((char *)ring_ptr + reg.offsets.tail);
 	rq_ring.rqes = (struct io_uring_zcrx_rqe *)((char *)ring_ptr + reg.offsets.rqes);
@@ -292,7 +313,7 @@ static void parse_opts(int argc, char **argv)
 	if (argc <= 1)
 		usage(argv[0]);
 
-	while ((c = getopt(argc, argv, "vp:i:q:s:")) != -1) {
+	while ((c = getopt(argc, argv, "vp:i:q:s:r:")) != -1) {
 		switch (c) {
 		case 'p':
 			cfg_port = strtoul(optarg, NULL, 0);
@@ -309,6 +330,11 @@ static void parse_opts(int argc, char **argv)
 		case 'v':
 			cfg_verify_data = true;
 			break;
+		case 'r':
+			cfg_rq_alloc_mode = strtoul(optarg, NULL, 0);
+			if (cfg_rq_alloc_mode >= __RQ_ALLOC_MAX)
+				t_error(1, 0, "invalid RQ allocation mode");
+			break;
 		}
 	}
 
Signature · 1 line
2.48.1
maintainer Applied
#5 lore raw
In reply to Pavel Begunkov
Quoted 8 lines · Pavel Begunkov

Random patches for zcrx. Path 1 removes oneshot mode as it doesn't give a good example, and Patch 3 allows to kernel allocations for the refill queue.

v2: rework oneshot/size limiting instead of removal
fix flipped allocation mode check

[...]

Applied, thanks!

[1/4] examples/zcrx: consolidate add_recvzc variants

      commit: 17286dbf9017ae6f4c864216b59ab9919268aff6
[2/4] examples/zcrx: rework size limiting
      commit: b10430f9b004a4f1f54c1a4721c80ce3a98996a0
[3/4] examples/zcrx: constants for request types
      commit: 8cee8f8cbf9f91c4dca9298159f2c5c64f898080
[4/4] examples/zcrx: add refill queue allocation modes
      commit: 353fc7dcc61e059ac8890916f41d39df10e5bfa5

Best regards,

Signature · 1 line
Jens Axboe