Skip to contentSkip to conversation

test sends with huge pages

PATCH Applied Pavel Begunkov 2 participants 9 messages Feb 21, 2023 – Feb 22, 2023 io_uring

lore
Share & tools
lore
#0 lore raw

cover test sends with huge pages

Add huge pages support for zc send benchmark and huge pages tests in send-zerocopy.c.

Pavel Begunkov (7):
tests/send: don't use a constant for page size
send: improve buffer iteration
send: test send with hugetlb
examples/zc: add a hugetlb option test/send: don't use SO_ZEROCOPY if not available tests/send: improve error reporting tests/send: sends with offsets

 examples/send-zerocopy.c |  24 ++++++-
 test/send-zerocopy.c     | 133 +++++++++++++++++++++++++++------------
 2 files changed, 114 insertions(+), 43 deletions(-)
Signature · 1 line
2.39.1
#1 lore raw

PATCH 1/7 tests/send: don't use a constant for page size

---
 test/send-zerocopy.c | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)
test/send-zerocopy.c +20 -9
index 86a31cd..2e30e49 100644
@@ -58,6 +58,10 @@ enum {
 	BUF_T_LARGE,
 };
 
+/* 32MB, should be enough to trigger a short send */
+#define LARGE_BUF_SIZE		(1U << 25)
+
+static size_t page_sz;
 static char *tx_buffer, *rx_buffer;
 static struct iovec buffers_iov[4];
 static bool has_sendmsg;
@@ -706,6 +710,8 @@ int main(int argc, char *argv[])
 	if (argc > 1)
 		return T_EXIT_SKIP;
 
+	page_sz = sysconf(_SC_PAGESIZE);
+
 	/* create TCP IPv6 pair */
 	ret = create_socketpair_ip(&addr, &sp[0], &sp[1], true, true, false, true);
 	if (ret) {
@@ -713,30 +719,35 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
-	len = 1U << 25; /* 32MB, should be enough to trigger a short send */
-	tx_buffer = aligned_alloc(4096, len);
-	rx_buffer = aligned_alloc(4096, len);
+	len = LARGE_BUF_SIZE;
+	tx_buffer = aligned_alloc(page_sz, len);
+	rx_buffer = aligned_alloc(page_sz, len);
 	if (tx_buffer && rx_buffer) {
 		buffers_iov[BUF_T_LARGE].iov_base = tx_buffer;
 		buffers_iov[BUF_T_LARGE].iov_len = len;
 	} else {
+		if (tx_buffer)
+			free(tx_buffer);
+		if (rx_buffer)
+			free(rx_buffer);
+
 		printf("skip large buffer tests, can't alloc\n");
 
-		len = 8192;
-		tx_buffer = aligned_alloc(4096, len);
-		rx_buffer = aligned_alloc(4096, len);
+		len = 2 * page_sz;
+		tx_buffer = aligned_alloc(page_sz, len);
+		rx_buffer = aligned_alloc(page_sz, len);
 	}
 	if (!tx_buffer || !rx_buffer) {
 		fprintf(stderr, "can't allocate buffers\n");
 		return T_EXIT_FAIL;
 	}
 
-	buffers_iov[BUF_T_NORMAL].iov_base = tx_buffer + 4096;
-	buffers_iov[BUF_T_NORMAL].iov_len = 4096;
+	buffers_iov[BUF_T_NORMAL].iov_base = tx_buffer + page_sz;
+	buffers_iov[BUF_T_NORMAL].iov_len = page_sz;
 	buffers_iov[BUF_T_SMALL].iov_base = tx_buffer;
 	buffers_iov[BUF_T_SMALL].iov_len = 137;
 	buffers_iov[BUF_T_NONALIGNED].iov_base = tx_buffer + BUFFER_OFFSET;
-	buffers_iov[BUF_T_NONALIGNED].iov_len = 8192 - BUFFER_OFFSET - 13;
+	buffers_iov[BUF_T_NONALIGNED].iov_len = 2 * page_sz - BUFFER_OFFSET - 13;
 
 	ret = io_uring_queue_init(32, &ring, 0);
 	if (ret) {
Signature · 1 line
2.39.1
#4 lore raw

PATCH 4/7 examples/zc: add a hugetlb option

In reply to Pavel Begunkov

Benchmark hugetlb sends

---
 examples/send-zerocopy.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)
examples/send-zerocopy.c +21 -3
index 6092af9..3a80d3d 100644
@@ -35,6 +35,8 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <sys/mman.h>
+#include <linux/mman.h>
 
 #include "liburing.h"
 
@@ -46,6 +48,7 @@ static bool cfg_fixed_files = 1;
 static bool cfg_zc = 1;
 static int  cfg_nr_reqs = 8;
 static bool cfg_fixed_buf = 1;
+static bool cfg_hugetlb = 0;
 
 static int  cfg_family		= PF_UNSPEC;
 static int  cfg_payload_len;
@@ -55,7 +58,8 @@ static int  cfg_runtime_ms	= 4200;
 static socklen_t cfg_alen;
 static struct sockaddr_storage cfg_dst_addr;
 
-static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));
+static char payload_buf[IP_MAXPACKET] __attribute__((aligned(4096)));
+static char *payload;
 
 /*
  * Implementation of error(3), prints an error message and exits.
@@ -277,7 +281,7 @@ static void usage(const char *filepath)
 
 static void parse_opts(int argc, char **argv)
 {
-	const int max_payload_len = sizeof(payload) -
+	const int max_payload_len = IP_MAXPACKET -
 				    sizeof(struct ipv6hdr) -
 				    sizeof(struct tcphdr) -
 				    40 /* max tcp options */;
@@ -289,7 +293,7 @@ static void parse_opts(int argc, char **argv)
 
 	cfg_payload_len = max_payload_len;
 
-	while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:k")) != -1) {
+	while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:l:")) != -1) {
 		switch (c) {
 		case '4':
 			if (cfg_family != PF_UNSPEC)
@@ -324,6 +328,9 @@ static void parse_opts(int argc, char **argv)
 		case 'b':
 			cfg_fixed_buf = strtoul(optarg, NULL, 0);
 			break;
+		case 'l':
+			cfg_hugetlb = strtoul(optarg, NULL, 0);
+			break;
 		}
 	}
 
@@ -344,6 +351,17 @@ int main(int argc, char **argv)
 
 	parse_opts(argc, argv);
 
+	payload = payload_buf;
+	if (cfg_hugetlb) {
+		payload = mmap(NULL, 2*1024*1024, PROT_READ | PROT_WRITE,
+				MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS,
+				-1, 0);
+		if (payload == MAP_FAILED) {
+			fprintf(stderr, "hugetlb alloc failed\n");
+			return 1;
+		}
+	}
+
 	cfg_test = argv[argc - 1];
 	if (!strcmp(cfg_test, "tcp"))
 		do_test(cfg_family, SOCK_STREAM, 0);
Signature · 1 line
2.39.1
#2 lore raw

PATCH 2/7 send: improve buffer iteration

In reply to Pavel Begunkov
---
 test/send-zerocopy.c | 42 ++++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)
test/send-zerocopy.c +24 -18
index 2e30e49..e663be7 100644
@@ -484,6 +484,7 @@ static int test_inet_send(struct io_uring *ring)
 	struct sockaddr_storage addr;
 	int sock_client = -1, sock_server = -1;
 	int ret, j, i;
+	int buf_index;
 
 	for (j = 0; j < 32; j++) {
 		bool ipv6 = j & 1;
@@ -510,20 +511,19 @@ static int test_inet_send(struct io_uring *ring)
 			sock_server = tmp_sock;
 		}
 
-		for (i = 0; i < 4096; i++) {
+		for (i = 0; i < 1024; i++) {
 			bool regbuf;
 
-			conf.buf_index = i & 3;
+			conf.use_sendmsg = i & 1;
+			conf.poll_first = i & 2;
 			conf.fixed_buf = i & 4;
 			conf.addr = (i & 8) ? &addr : NULL;
 			conf.cork = i & 16;
 			conf.mix_register = i & 32;
 			conf.force_async = i & 64;
-			conf.use_sendmsg = i & 128;
-			conf.zc = i & 256;
-			conf.iovec = i & 512;
-			conf.long_iovec = i & 1024;
-			conf.poll_first = i & 2048;
+			conf.zc = i & 128;
+			conf.iovec = i & 256;
+			conf.long_iovec = i & 512;
 			conf.tcp = tcp;
 			regbuf = conf.mix_register || conf.fixed_buf;
 
@@ -539,10 +539,6 @@ static int test_inet_send(struct io_uring *ring)
 				if (conf.addr && !has_sendmsg)
 					continue;
 			}
-			if (conf.buf_index == BUF_T_LARGE && !tcp)
-				continue;
-			if (!buffers_iov[conf.buf_index].iov_base)
-				continue;
 			if (tcp && (conf.cork || conf.addr))
 				continue;
 			if (conf.mix_register && (!conf.cork || conf.fixed_buf))
@@ -554,13 +550,23 @@ static int test_inet_send(struct io_uring *ring)
 			if (msg_zc_set && !conf.zc)
 				continue;
 
-			ret = do_test_inet_send(ring, sock_client, sock_server, &conf);
-			if (ret) {
-				fprintf(stderr, "send failed fixed buf %i, conn %i, addr %i, "
-					"cork %i\n",
-					conf.fixed_buf, client_connect, !!conf.addr,
-					conf.cork);
-				return 1;
+			for (buf_index = 0; buf_index < ARRAY_SIZE(buffers_iov); buf_index++) {
+				size_t len = buffers_iov[buf_index].iov_len;
+
+				if (!buffers_iov[buf_index].iov_base)
+					continue;
+				if (!tcp && len > 4 * page_sz)
+					continue;
+
+				conf.buf_index = buf_index;
+				ret = do_test_inet_send(ring, sock_client, sock_server, &conf);
+				if (ret) {
+					fprintf(stderr, "send failed fixed buf %i, "
+							"conn %i, addr %i, cork %i\n",
+						conf.fixed_buf, client_connect,
+						!!conf.addr, conf.cork);
+					return 1;
+				}
 			}
 		}
 
Signature · 1 line
2.39.1
#3 lore raw

PATCH 3/7 send: test send with hugetlb

In reply to Pavel Begunkov
---
 test/send-zerocopy.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)
test/send-zerocopy.c +25 -6
index e663be7..f1277fa 100644
@@ -33,6 +33,8 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <sys/mman.h>
+#include <linux/mman.h>
 
 #include "liburing.h"
 #include "helpers.h"
@@ -56,6 +58,9 @@ enum {
 	BUF_T_SMALL,
 	BUF_T_NONALIGNED,
 	BUF_T_LARGE,
+	BUF_T_HUGETLB,
+
+	__BUF_NR,
 };
 
 /* 32MB, should be enough to trigger a short send */
@@ -63,7 +68,7 @@ enum {
 
 static size_t page_sz;
 static char *tx_buffer, *rx_buffer;
-static struct iovec buffers_iov[4];
+static struct iovec buffers_iov[__BUF_NR];
 static bool has_sendmsg;
 
 static bool check_cq_empty(struct io_uring *ring)
@@ -748,6 +753,11 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
+	srand((unsigned)time(NULL));
+	for (i = 0; i < len; i++)
+		tx_buffer[i] = i;
+	memset(rx_buffer, 0, len);
+
 	buffers_iov[BUF_T_NORMAL].iov_base = tx_buffer + page_sz;
 	buffers_iov[BUF_T_NORMAL].iov_len = page_sz;
 	buffers_iov[BUF_T_SMALL].iov_base = tx_buffer;
@@ -755,17 +765,26 @@ int main(int argc, char *argv[])
 	buffers_iov[BUF_T_NONALIGNED].iov_base = tx_buffer + BUFFER_OFFSET;
 	buffers_iov[BUF_T_NONALIGNED].iov_len = 2 * page_sz - BUFFER_OFFSET - 13;
 
+	if (len == LARGE_BUF_SIZE) {
+		void *huge_page;
+		int off = page_sz + 27;
+
+		len = 1U << 22;
+		huge_page = mmap(NULL, len, PROT_READ|PROT_WRITE,
+				 MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS,
+				 -1, 0);
+		if (huge_page != MAP_FAILED) {
+			buffers_iov[BUF_T_HUGETLB].iov_base = huge_page + off;
+			buffers_iov[BUF_T_HUGETLB].iov_len = len - off;
+		}
+	}
+
 	ret = io_uring_queue_init(32, &ring, 0);
 	if (ret) {
 		fprintf(stderr, "queue init failed: %d\n", ret);
 		return T_EXIT_FAIL;
 	}
 
-	srand((unsigned)time(NULL));
-	for (i = 0; i < len; i++)
-		tx_buffer[i] = i;
-	memset(rx_buffer, 0, len);
-
 	ret = test_basic_send(&ring, sp[0], sp[1]);
 	if (ret == T_EXIT_SKIP)
 		return ret;
Signature · 1 line
2.39.1
#5 lore raw

PATCH 5/7 test/send: don't use SO_ZEROCOPY if not available

In reply to Pavel Begunkov
---
 test/send-zerocopy.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)
test/send-zerocopy.c +13 -6
index f1277fa..481aa28 100644
@@ -197,11 +197,9 @@ static int create_socketpair_ip(struct sockaddr_storage *addr,
 				bool ipv6, bool client_connect,
 				bool msg_zc, bool tcp)
 {
-	int family;
 	socklen_t addr_size;
-	int ret, val;
-	int listen_sock = -1;
-	int sock;
+	int family, sock, listen_sock = -1;
+	int ret;
 
 	memset(addr, 0, sizeof(*addr));
 	if (ipv6) {
@@ -278,11 +276,17 @@ static int create_socketpair_ip(struct sockaddr_storage *addr,
 		}
 	}
 	if (msg_zc) {
-		val = 1;
+#ifdef SO_ZEROCOPY
+		int val = 1;
+
 		if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) {
 			perror("setsockopt zc");
 			return 1;
 		}
+#else
+		fprintf(stderr, "no SO_ZEROCOPY\n");
+		return 1;
+#endif
 	}
 	if (tcp) {
 		*sock_server = accept(listen_sock, NULL, NULL);
@@ -502,7 +506,10 @@ static int test_inet_send(struct io_uring *ring)
 			continue;
 		if (swap_sockets && !tcp)
 			continue;
-
+#ifndef SO_ZEROCOPY
+		if (msg_zc_set)
+			continue;
+#endif
 		ret = create_socketpair_ip(&addr, &sock_client, &sock_server, ipv6,
 				 client_connect, msg_zc_set, tcp);
 		if (ret) {
Signature · 1 line
2.39.1
#7 lore raw

PATCH 7/7 tests/send: sends with offsets

In reply to Pavel Begunkov

Apart from arbitrary shifting buffers before registration so they're not page aligned, also add offsets to send requests.

---
 test/send-zerocopy.c | 9 +++++++++
 1 file changed, 9 insertions(+)
test/send-zerocopy.c +9 -0
index 8ddec39..57894aa 100644
@@ -833,6 +833,15 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
+	if (buffers_iov[BUF_T_HUGETLB].iov_base) {
+		buffers_iov[BUF_T_HUGETLB].iov_base += 13;
+		buffers_iov[BUF_T_HUGETLB].iov_len -= 26;
+	}
+	if (buffers_iov[BUF_T_LARGE].iov_base) {
+		buffers_iov[BUF_T_LARGE].iov_base += 13;
+		buffers_iov[BUF_T_LARGE].iov_len -= 26;
+	}
+
 	ret = test_inet_send(&ring);
 	if (ret) {
 		fprintf(stderr, "test_inet_send() failed\n");
Signature · 1 line
2.39.1
#6 lore raw

PATCH 6/7 tests/send: improve error reporting

In reply to Pavel Begunkov
---
 test/send-zerocopy.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
test/send-zerocopy.c +2 -1
index 481aa28..8ddec39 100644
@@ -447,7 +447,8 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 		}
 		if (cqe->user_data == RX_TAG) {
 			if (cqe->res != send_size) {
-				fprintf(stderr, "rx failed %i\n", cqe->res);
+				fprintf(stderr, "rx failed res: %i, expected %i\n",
+						cqe->res, (int)send_size);
 				return 1;
 			}
 			io_uring_cqe_seen(ring, cqe);
Signature · 1 line
2.39.1
maintainer Applied
#8 lore raw
In reply to Pavel Begunkov
Quoted 12 lines · Pavel Begunkov

Add huge pages support for zc send benchmark and huge pages tests in send-zerocopy.c.

Pavel Begunkov (7):
tests/send: don't use a constant for page size
send: improve buffer iteration
send: test send with hugetlb
examples/zc: add a hugetlb option test/send: don't use SO_ZEROCOPY if not available tests/send: improve error reporting tests/send: sends with offsets

Quoted 1 line

[...]

Applied, thanks!

[1/7] tests/send: don't use a constant for page size
(no commit info)
[2/7] send: improve buffer iteration
(no commit info)
[3/7] send: test send with hugetlb
(no commit info)
[4/7] examples/zc: add a hugetlb option
(no commit info)
[5/7] test/send: don't use SO_ZEROCOPY if not available
(no commit info)
[6/7] tests/send: improve error reporting
(no commit info)
[7/7] tests/send: sends with offsets
(no commit info)

Best regards,

Signature · 1 line
Jens Axboe