Skip to contentSkip to conversation

vectored registered buffer support and tests

PATCH Superseded Pavel Begunkov 3 participants 8 messages Mar 7, 2025 io_uring

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover vectored registered buffer support and tests

It should give good coverage, I plan to extend read-write.c to use it as well.

Pavel Begunkov (4):
Add vectored registered buffer req init helpers
test/sendzc: test registered vectored buffers
tests/helpers: add t_create_socketpair_ip
tests: targeted registered vector tests

 src/include/liburing.h |  31 +++
 test/Makefile          |   1 +
 test/helpers.c         | 111 ++++++++
 test/helpers.h         |   5 +
 test/regvec.c          | 604 +++++++++++++++++++++++++++++++++++++++++
 test/send-zerocopy.c   | 138 ++--------
 6 files changed, 776 insertions(+), 114 deletions(-)
 create mode 100644 test/regvec.c
Signature · 1 line
2.48.1
#1 lore raw

PATCH 1/4 Add vectored registered buffer req init helpers

---
 src/include/liburing.h | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
src/include/liburing.h +31 -0
index d162d0e6..e71551ed 100644
@@ -556,6 +556,16 @@ IOURINGINLINE void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
 	sqe->buf_index = (__u16) buf_index;
 }
 
+IOURINGINLINE void io_uring_prep_readv_fixed(struct io_uring_sqe *sqe, int fd,
+					     const struct iovec *iovecs,
+					     unsigned nr_vecs, __u64 offset,
+					     int flags, int buf_index)
+{
+	io_uring_prep_readv2(sqe, fd, iovecs, nr_vecs, offset, flags);
+	sqe->opcode = IORING_OP_WRITE_FIXED;
+	sqe->buf_index = (__u16)buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
 					const struct iovec *iovecs,
 					unsigned nr_vecs, __u64 offset)
@@ -580,6 +590,16 @@ IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd,
 	sqe->buf_index = (__u16) buf_index;
 }
 
+IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
+				       const struct iovec *iovecs,
+				       unsigned nr_vecs, __u64 offset,
+				       int flags, int buf_index)
+{
+	io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
+	sqe->opcode = IORING_OP_WRITE_FIXED;
+	sqe->buf_index = (__u16)buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
 					 struct msghdr *msg, unsigned flags)
 {
@@ -964,6 +984,17 @@ IOURINGINLINE void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd,
 	sqe->opcode = IORING_OP_SENDMSG_ZC;
 }
 
+IOURINGINLINE void io_uring_prep_sendmsg_zc_fixed(struct io_uring_sqe *sqe,
+						int fd,
+						const struct msghdr *msg,
+						unsigned flags,
+						unsigned buf_index)
+{
+	io_uring_prep_sendmsg_zc(sqe, fd, msg, flags);
+	sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+	sqe->buf_index = buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
 				      void *buf, size_t len, int flags)
 {
Signature · 1 line
2.48.1
#2 lore raw

Re: [PATCH liburing 1/4] Add vectored registered buffer req init helpers

Quoted 11 lines · Pavel Begunkov
---
 src/include/liburing.h | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
src/include/liburing.h +0 -0
index d162d0e6..e71551ed 100644

src/include/liburing.h IOURINGINLINE void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd, · line 565

Quoted 10 lines
src/include/liburing.h +7 -0
@@ -556,6 +556,16 @@ IOURINGINLINE void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
        sqe->buf_index = (__u16) buf_index;
 }
 
+IOURINGINLINE void io_uring_prep_readv_fixed(struct io_uring_sqe *sqe, int fd,
+                                            const struct iovec *iovecs,
+                                            unsigned nr_vecs, __u64 offset,
+                                            int flags, int buf_index)
+{
+       io_uring_prep_readv2(sqe, fd, iovecs, nr_vecs, offset, flags);
+       sqe->opcode = IORING_OP_WRITE_FIXED;

Presumably should be IORING_OP_READV_FIXED? You'll probably need to copy the UAPI header changes to liburing.

Quoted 7 lines
src/include/liburing.h +3 -0
+       sqe->buf_index = (__u16)buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
                                        const struct iovec *iovecs,
                                        unsigned nr_vecs, __u64 offset)

src/include/liburing.h IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, · line 599

Quoted 10 lines
src/include/liburing.h +7 -0
@@ -580,6 +590,16 @@ IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd,
        sqe->buf_index = (__u16) buf_index;
 }
 
+IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
+                                      const struct iovec *iovecs,
+                                      unsigned nr_vecs, __u64 offset,
+                                      int flags, int buf_index)
+{
+       io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
+       sqe->opcode = IORING_OP_WRITE_FIXED;

IORING_OP_WRITEV_FIXED?

Best,
Caleb

Quoted 28 lines
src/include/liburing.h +14 -1
+       sqe->buf_index = (__u16)buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
                                         struct msghdr *msg, unsigned flags)
 {
@@ -964,6 +984,17 @@ IOURINGINLINE void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd,
        sqe->opcode = IORING_OP_SENDMSG_ZC;
 }
 
+IOURINGINLINE void io_uring_prep_sendmsg_zc_fixed(struct io_uring_sqe *sqe,
+                                               int fd,
+                                               const struct msghdr *msg,
+                                               unsigned flags,
+                                               unsigned buf_index)
+{
+       io_uring_prep_sendmsg_zc(sqe, fd, msg, flags);
+       sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+       sqe->buf_index = buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
                                      void *buf, size_t len, int flags)
 {
--

2.48.1

#3 lore raw

Re: [PATCH liburing 1/4] Add vectored registered buffer req init helpers

Quoted 1 line · Caleb Sander Mateos

On Fri, Mar 7, 2025 at 8:22 AM Pavel Begunkov <[email protected]> wrote:

...

Quoted 9 lines

src/include/liburing.h IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, · line 599

Quoted 7 lines
src/include/liburing.h +7 -0
+IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
+                                      const struct iovec *iovecs,
+                                      unsigned nr_vecs, __u64 offset,
+                                      int flags, int buf_index)
+{
+       io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
+       sqe->opcode = IORING_OP_WRITE_FIXED;

IORING_OP_WRITEV_FIXED?

Good catch, that came from a prototype where it was based on that opcode, I should just use the helper in the test.

Signature · 1 line
Pavel Begunkov
maintainer
#4 lore raw

Re: [PATCH liburing 1/4] Add vectored registered buffer req init helpers

Quoted 13 lines · Pavel Begunkov
Quoted 1 line · Caleb Sander Mateos

On Fri, Mar 7, 2025 at 8:22 AM Pavel Begunkov <[email protected]> wrote:

...

Quoted 9 lines

src/include/liburing.h IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, · line 599

Quoted 7 lines
src/include/liburing.h +7 -0
+IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
+                                      const struct iovec *iovecs,
+                                      unsigned nr_vecs, __u64 offset,
+                                      int flags, int buf_index)
+{
+       io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
+       sqe->opcode = IORING_OP_WRITE_FIXED;

IORING_OP_WRITEV_FIXED?

Quoted 2 lines

Good catch, that came from a prototype where it was based on that opcode, I should just use the helper in the test.

Both also need an ffi addition in the map.

Signature · 1 line
Jens Axboe
#5 lore raw

PATCH 2/4 test/sendzc: test registered vectored buffers

In reply to Pavel Begunkov
---
 test/send-zerocopy.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
test/send-zerocopy.c +13 -2
index c8eafe28..680481a0 100644
@@ -69,6 +69,7 @@ static size_t page_sz;
 static char *tx_buffer, *rx_buffer;
 static struct iovec buffers_iov[__BUF_NR];
 
+static bool has_regvec;
 static bool has_sendzc;
 static bool has_sendmsg;
 static bool hit_enomem;
@@ -96,6 +97,7 @@ static int probe_zc_support(void)
 
 	has_sendzc = p->ops_len > IORING_OP_SEND_ZC;
 	has_sendmsg = p->ops_len > IORING_OP_SENDMSG_ZC;
+	has_regvec = p->ops_len > IORING_OP_READV_FIXED;
 	io_uring_queue_exit(&ring);
 	free(p);
 	return 0;
@@ -448,6 +450,11 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 			else
 				io_uring_prep_sendmsg(sqe, sock_client, &msghdr[i], msg_flags);
 
+			if (real_fixed_buf) {
+				sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+				sqe->buf_index = conf->buf_index;
+			}
+
 			if (!conf->iovec) {
 				io = &iov[i];
 				iov_len = 1;
@@ -619,7 +626,11 @@ static int test_inet_send(struct io_uring *ring)
 			conf.tcp = tcp;
 			regbuf = conf.mix_register || conf.fixed_buf;
 
-			if (conf.iovec && (!conf.use_sendmsg || regbuf || conf.cork))
+			if (!tcp && conf.long_iovec)
+				continue;
+			if (conf.use_sendmsg && regbuf && !has_regvec)
+				continue;
+			if (conf.iovec && (!conf.use_sendmsg || conf.cork))
 				continue;
 			if (!conf.zc) {
 				if (regbuf)
@@ -637,7 +648,7 @@ static int test_inet_send(struct io_uring *ring)
 				continue;
 			if (!client_connect && conf.addr == NULL)
 				continue;
-			if (conf.use_sendmsg && (regbuf || !has_sendmsg))
+			if (conf.use_sendmsg && !has_sendmsg)
 				continue;
 			if (msg_zc_set && !conf.zc)
 				continue;
Signature · 1 line
2.48.1
#6 lore raw

PATCH 3/4 tests/helpers: add t_create_socketpair_ip

In reply to Pavel Begunkov

create_socketpair_ip() is useful for zerocopy tx testing as it needs TCP sockets and not just AF_UNIX. There will be tx zc tests in more files, so move the function to helpers.

---
 test/helpers.c       | 111 ++++++++++++++++++++++++++++++++++++++
 test/helpers.h       |   5 ++
 test/send-zerocopy.c | 123 ++++---------------------------------------
 3 files changed, 127 insertions(+), 112 deletions(-)
test/helpers.c +111 -0
index 07186911..b2d386f4 100644
@@ -373,3 +373,114 @@ void *aligned_alloc(size_t alignment, size_t size)
 
 	return ret;
 }
+
+int t_create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp, const char *name)
+{
+	socklen_t addr_size;
+	int family, sock, listen_sock = -1;
+	int ret;
+
+	memset(addr, 0, sizeof(*addr));
+	if (ipv6) {
+		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
+
+		family = AF_INET6;
+		saddr->sin6_family = family;
+		saddr->sin6_port = htons(0);
+		addr_size = sizeof(*saddr);
+	} else {
+		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
+
+		family = AF_INET;
+		saddr->sin_family = family;
+		saddr->sin_port = htons(0);
+		saddr->sin_addr.s_addr = htonl(INADDR_ANY);
+		addr_size = sizeof(*saddr);
+	}
+
+	/* server sock setup */
+	if (tcp) {
+		sock = listen_sock = socket(family, SOCK_STREAM, IPPROTO_TCP);
+	} else {
+		sock = *sock_server = socket(family, SOCK_DGRAM, 0);
+	}
+	if (sock < 0) {
+		perror("socket");
+		return 1;
+	}
+
+	ret = bind(sock, (struct sockaddr *)addr, addr_size);
+	if (ret < 0) {
+		perror("bind");
+		return 1;
+	}
+
+	ret = getsockname(sock, (struct sockaddr *)addr, &addr_size);
+	if (ret < 0) {
+		fprintf(stderr, "getsockname failed %i\n", errno);
+		return 1;
+	}
+
+	if (tcp) {
+		ret = listen(sock, 128);
+		assert(ret != -1);
+	}
+
+	if (ipv6) {
+		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
+
+		inet_pton(AF_INET6, name, &(saddr->sin6_addr));
+	} else {
+		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
+
+		inet_pton(AF_INET, name, &saddr->sin_addr);
+	}
+
+	/* client sock setup */
+	if (tcp) {
+		*sock_client = socket(family, SOCK_STREAM, IPPROTO_TCP);
+		assert(client_connect);
+	} else {
+		*sock_client = socket(family, SOCK_DGRAM, 0);
+	}
+	if (*sock_client < 0) {
+		perror("socket");
+		return 1;
+	}
+	if (client_connect) {
+		ret = connect(*sock_client, (struct sockaddr *)addr, addr_size);
+		if (ret < 0) {
+			perror("connect");
+			return 1;
+		}
+	}
+	if (msg_zc) {
+#ifdef SO_ZEROCOPY
+		int val = 1;
+
+		/*
+		 * NOTE: apps must not set SO_ZEROCOPY when using io_uring zc.
+		 * It's only here to test interactions with MSG_ZEROCOPY.
+		 */
+		if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) {
+			perror("setsockopt zc");
+			return 1;
+		}
+#else
+		fprintf(stderr, "no SO_ZEROCOPY\n");
+		return 1;
+#endif
+	}
+	if (tcp) {
+		*sock_server = accept(listen_sock, NULL, NULL);
+		if (!*sock_server) {
+			fprintf(stderr, "can't accept\n");
+			return 1;
+		}
+		close(listen_sock);
+	}
+	return 0;
+}
test/helpers.h +5 -0
index d0294eba..f8a5c7f2 100644
@@ -81,6 +81,11 @@ struct iovec *t_create_buffers(size_t buf_num, size_t buf_size);
  */
 int t_create_socket_pair(int fd[2], bool stream);
 
+int t_create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp, const char *name);
+
 /*
  * Helper for setting up a ring and checking for user privs
  */
test/send-zerocopy.c +11 -112

141 lines not on this page · raw message

Signature · 1 line
2.48.1
#7 lore raw

PATCH 4/4 tests: targeted registered vector tests

In reply to Pavel Begunkov

The main part here is to exercise various iovec->bvec reallocation scenarios, but it also checks edge cases and validates final data.

---
 test/Makefile |   1 +
 test/regvec.c | 604 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 605 insertions(+)
 create mode 100644 test/regvec.c
test/Makefile +1 -0
index 0367ef72..e937c852 100644
@@ -241,6 +241,7 @@ test_srcs := \
 	wq-aff.c \
 	xattr.c \
 	zcrx.c \
+	regvec.c \
 	# EOL
 
 # Please keep this list sorted alphabetically.
test/regvec.c new file +604 -0

604 lines not on this page · raw message

Signature · 1 line
2.48.1