test sends with huge pages
cover test sends with huge pages
Add huge pages support for zc send benchmark and huge pages tests in send-zerocopy.c.
Pavel Begunkov (7):
tests/send: don't use a constant for page size
send: improve buffer iteration
send: test send with hugetlb
examples/zc: add a hugetlb option test/send: don't use SO_ZEROCOPY if not available tests/send: improve error reporting tests/send: sends with offsets
examples/send-zerocopy.c | 24 ++++++- test/send-zerocopy.c | 133 +++++++++++++++++++++++++++------------ 2 files changed, 114 insertions(+), 43 deletions(-)
Signature · 1 line
2.39.1
PATCH 1/7 tests/send: don't use a constant for page size
--- test/send-zerocopy.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-)
test/send-zerocopy.c +20 -9
@@ -58,6 +58,10 @@ enum { BUF_T_LARGE, }; +/* 32MB, should be enough to trigger a short send */ +#define LARGE_BUF_SIZE (1U << 25) + +static size_t page_sz;static char *tx_buffer, *rx_buffer;static struct iovec buffers_iov[4 ];static bool has_sendmsg;
@@ -706,6 +710,8 @@ int main(int argc, char *argv[])if (argc >1 )return T_EXIT_SKIP ; + page_sz =sysconf (_SC_PAGESIZE); +/* create TCP IPv6 pair */ ret =create_socketpair_ip (&addr, &sp[0 ], &sp[1 ], true, true, false, true);if (ret) {
@@ -713,30 +719,35 @@ int main(int argc, char *argv[])return T_EXIT_FAIL ; } - len =1U <<25 ;/* 32MB, should be enough to trigger a short send */ - tx_buffer =aligned_alloc (4096 , len); - rx_buffer =aligned_alloc (4096 , len); + len =LARGE_BUF_SIZE ; + tx_buffer =aligned_alloc (page_sz, len); + rx_buffer =aligned_alloc (page_sz, len);if (tx_buffer && rx_buffer) { buffers_iov[BUF_T_LARGE ].iov_base = tx_buffer; buffers_iov[BUF_T_LARGE ].iov_len = len; }else { +if (tx_buffer) +free (tx_buffer); +if (rx_buffer) +free (rx_buffer); +printf ("skip large buffer tests, can't alloc\n" ); - len =8192 ; - tx_buffer =aligned_alloc (4096 , len); - rx_buffer =aligned_alloc (4096 , len); + len =2 * page_sz; + tx_buffer =aligned_alloc (page_sz, len); + rx_buffer =aligned_alloc (page_sz, len); }if (!tx_buffer || !rx_buffer) {fprintf (stderr,"can't allocate buffers\n" );return T_EXIT_FAIL ; } - buffers_iov[BUF_T_NORMAL ].iov_base = tx_buffer +4096 ; - buffers_iov[BUF_T_NORMAL ].iov_len =4096 ; + buffers_iov[BUF_T_NORMAL ].iov_base = tx_buffer + page_sz; + buffers_iov[BUF_T_NORMAL ].iov_len = page_sz; buffers_iov[BUF_T_SMALL ].iov_base = tx_buffer; buffers_iov[BUF_T_SMALL ].iov_len =137 ; buffers_iov[BUF_T_NONALIGNED ].iov_base = tx_buffer +BUFFER_OFFSET ; - buffers_iov[BUF_T_NONALIGNED ].iov_len =8192 -BUFFER_OFFSET -13 ; + buffers_iov[BUF_T_NONALIGNED ].iov_len =2 * page_sz -BUFFER_OFFSET -13 ; ret =io_uring_queue_init (32 , &ring,0 );if (ret) {
Signature · 1 line
2.39.1
PATCH 2/7 send: improve buffer iteration
In reply to Pavel Begunkov--- test/send-zerocopy.c | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-)
test/send-zerocopy.c +24 -18
@@ -484,6 +484,7 @@ static int test_inet_send(struct io_uring *ring)struct sockaddr_storage addr;int sock_client =-1 , sock_server =-1 ;int ret, j, i; +int buf_index;for (j =0 ; j <32 ; j++) {bool ipv6 = j &1 ;
@@ -510,20 +511,19 @@ static int test_inet_send(struct io_uring *ring) sock_server = tmp_sock; } -for (i =0 ; i <4096 ; i++) { +for (i =0 ; i <1024 ; i++) {bool regbuf; - conf.buf_index = i &3 ; + conf.use_sendmsg = i &1 ; + conf.poll_first = i &2 ; conf.fixed_buf = i &4 ; conf.addr = (i &8 ) ? &addr :NULL ; conf.cork = i &16 ; conf.mix_register = i &32 ; conf.force_async = i &64 ; - conf.use_sendmsg = i &128 ; - conf.zc = i &256 ; - conf.iovec = i &512 ; - conf.long_iovec = i &1024 ; - conf.poll_first = i &2048 ; + conf.zc = i &128 ; + conf.iovec = i &256 ; + conf.long_iovec = i &512 ; conf.tcp = tcp; regbuf = conf.mix_register || conf.fixed_buf;
@@ -539,10 +539,6 @@ static int test_inet_send(struct io_uring *ring)if (conf.addr && !has_sendmsg)continue ; } -if (conf.buf_index ==BUF_T_LARGE && !tcp) -continue ; -if (!buffers_iov[conf.buf_index].iov_base) -continue ;if (tcp && (conf.cork || conf.addr))continue ;if (conf.mix_register && (!conf.cork || conf.fixed_buf))
@@ -554,13 +550,23 @@ static int test_inet_send(struct io_uring *ring)if (msg_zc_set && !conf.zc)continue ; - ret =do_test_inet_send (ring, sock_client, sock_server, &conf); -if (ret) { -fprintf (stderr,"send failed fixed buf %i, conn %i, addr %i, " -"cork %i\n" , - conf.fixed_buf, client_connect, !!conf.addr, - conf.cork); -return 1 ; +for (buf_index =0 ; buf_index <ARRAY_SIZE (buffers_iov); buf_index++) { +size_t len = buffers_iov[buf_index].iov_len; + +if (!buffers_iov[buf_index].iov_base) +continue ; +if (!tcp && len >4 * page_sz) +continue ; + + conf.buf_index = buf_index; + ret =do_test_inet_send (ring, sock_client, sock_server, &conf); +if (ret) { +fprintf (stderr,"send failed fixed buf %i, " +"conn %i, addr %i, cork %i\n" , + conf.fixed_buf, client_connect, + !!conf.addr, conf.cork); +return 1 ; + } } }
Signature · 1 line
2.39.1
PATCH 3/7 send: test send with hugetlb
In reply to Pavel Begunkov--- test/send-zerocopy.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-)
test/send-zerocopy.c +25 -6
@@ -33,6 +33,8 @@#include <sys/time.h> #include <sys/types.h> #include <sys/wait.h> +#include <sys/mman.h> +#include <linux/mman.h> #include "liburing.h" #include "helpers.h"
@@ -56,6 +58,9 @@ enum {BUF_T_SMALL ,BUF_T_NONALIGNED ,BUF_T_LARGE , +BUF_T_HUGETLB , + + __BUF_NR, };/* 32MB, should be enough to trigger a short send */
@@ -63,7 +68,7 @@ enum {static size_t page_sz;static char *tx_buffer, *rx_buffer; -static struct iovec buffers_iov[4 ]; +static struct iovec buffers_iov[__BUF_NR];static bool has_sendmsg;static bool check_cq_empty (struct io_uring *ring)
@@ -748,6 +753,11 @@ int main(int argc, char *argv[])return T_EXIT_FAIL ; } +srand ((unsigned )time (NULL )); +for (i =0 ; i < len; i++) + tx_buffer[i] = i; +memset (rx_buffer,0 , len); + buffers_iov[BUF_T_NORMAL ].iov_base = tx_buffer + page_sz; buffers_iov[BUF_T_NORMAL ].iov_len = page_sz; buffers_iov[BUF_T_SMALL ].iov_base = tx_buffer;
@@ -755,17 +765,26 @@ int main(int argc, char *argv[]) buffers_iov[BUF_T_NONALIGNED ].iov_base = tx_buffer +BUFFER_OFFSET ; buffers_iov[BUF_T_NONALIGNED ].iov_len =2 * page_sz -BUFFER_OFFSET -13 ; +if (len ==LARGE_BUF_SIZE ) { +void *huge_page; +int off = page_sz +27 ; + + len =1U <<22 ; + huge_page =mmap (NULL , len,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_HUGETLB |MAP_HUGE_2MB |MAP_ANONYMOUS , +-1 ,0 ); +if (huge_page !=MAP_FAILED ) { + buffers_iov[BUF_T_HUGETLB ].iov_base = huge_page + off; + buffers_iov[BUF_T_HUGETLB ].iov_len = len - off; + } + } + ret =io_uring_queue_init (32 , &ring,0 );if (ret) {fprintf (stderr,"queue init failed: %d\n" , ret);return T_EXIT_FAIL ; } -srand ((unsigned )time (NULL )); -for (i =0 ; i < len; i++) - tx_buffer[i] = i; -memset (rx_buffer,0 , len); - ret =test_basic_send (&ring, sp[0 ], sp[1 ]);if (ret ==T_EXIT_SKIP )return ret;
Signature · 1 line
2.39.1
PATCH 4/7 examples/zc: add a hugetlb option
In reply to Pavel BegunkovBenchmark hugetlb sends
--- examples/send-zerocopy.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-)
examples/send-zerocopy.c +21 -3
@@ -35,6 +35,8 @@#include <sys/time.h> #include <sys/types.h> #include <sys/wait.h> +#include <sys/mman.h> +#include <linux/mman.h> #include "liburing.h"
@@ -46,6 +48,7 @@ static bool cfg_fixed_files = 1;static bool cfg_zc =1 ;static int cfg_nr_reqs =8 ;static bool cfg_fixed_buf =1 ; +static bool cfg_hugetlb =0 ;static int cfg_family =PF_UNSPEC ;static int cfg_payload_len;
@@ -55,7 +58,8 @@ static int cfg_runtime_ms = 4200;static socklen_t cfg_alen;static struct sockaddr_storage cfg_dst_addr; -static char payload[IP_MAXPACKET] __attribute__((aligned(4096))); +static char payload_buf[IP_MAXPACKET] __attribute__((aligned(4096))); +static char *payload;/* * Implementation of error(3), prints an error message and exits.
@@ -277,7 +281,7 @@ static void usage(const char *filepath)static void parse_opts (int argc,char **argv) { -const int max_payload_len =sizeof (payload) - +const int max_payload_len =IP_MAXPACKET -sizeof (struct ipv6hdr ) -sizeof (struct tcphdr ) -40 /* max tcp options */ ;
@@ -289,7 +293,7 @@ static void parse_opts(int argc, char **argv) cfg_payload_len = max_payload_len; -while ((c =getopt (argc, argv,"46D:p:s:t:n:z:b:k" )) !=-1 ) { +while ((c =getopt (argc, argv,"46D:p:s:t:n:z:b:l:" )) !=-1 ) {switch (c) {case '4' :if (cfg_family !=PF_UNSPEC )
@@ -324,6 +328,9 @@ static void parse_opts(int argc, char **argv)case 'b' : cfg_fixed_buf =strtoul (optarg,NULL ,0 );break ; +case 'l' : + cfg_hugetlb =strtoul (optarg,NULL ,0 ); +break ; } }
@@ -344,6 +351,17 @@ int main(int argc, char **argv)parse_opts (argc, argv); + payload = payload_buf; +if (cfg_hugetlb) { + payload =mmap (NULL ,2 *1024 *1024 ,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_HUGETLB |MAP_HUGE_2MB |MAP_ANONYMOUS , +-1 ,0 ); +if (payload ==MAP_FAILED ) { +fprintf (stderr,"hugetlb alloc failed\n" ); +return 1 ; + } + } + cfg_test = argv[argc -1 ];if (!strcmp (cfg_test,"tcp" ))do_test (cfg_family,SOCK_STREAM ,0 );
Signature · 1 line
2.39.1
PATCH 5/7 test/send: don't use SO_ZEROCOPY if not available
In reply to Pavel Begunkov--- test/send-zerocopy.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-)
test/send-zerocopy.c +13 -6
@@ -197,11 +197,9 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, bool ipv6, bool client_connect, bool msg_zc, bool tcp) { - int family; socklen_t addr_size; -int ret, val; -int listen_sock =-1 ; -int sock; +int family, sock, listen_sock =-1 ; +int ret;memset (addr,0 ,sizeof (*addr));if (ipv6) {
@@ -278,11 +276,17 @@ static int create_socketpair_ip(struct sockaddr_storage *addr, } }if (msg_zc) { - val =1 ; +#ifdef SO_ZEROCOPY +int val =1 ; +if (setsockopt (*sock_client,SOL_SOCKET ,SO_ZEROCOPY , &val,sizeof (val))) {perror ("setsockopt zc" );return 1 ; } +#else +fprintf (stderr,"no SO_ZEROCOPY\n" ); +return 1 ; +#endif }if (tcp) { *sock_server =accept (listen_sock,NULL ,NULL );
@@ -502,7 +506,10 @@ static int test_inet_send(struct io_uring *ring)continue ;if (swap_sockets && !tcp)continue ; - +#ifndef SO_ZEROCOPY +if (msg_zc_set) +continue ; +#endif ret =create_socketpair_ip (&addr, &sock_client, &sock_server, ipv6, client_connect, msg_zc_set, tcp);if (ret) {
Signature · 1 line
2.39.1
PATCH 6/7 tests/send: improve error reporting
In reply to Pavel Begunkov--- test/send-zerocopy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
test/send-zerocopy.c +2 -1
@@ -447,7 +447,8 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se }if (cqe->user_data ==RX_TAG ) {if (cqe->res != send_size) { -fprintf (stderr,"rx failed %i\n" , cqe->res); +fprintf (stderr,"rx failed res: %i, expected %i\n" , + cqe->res, (int )send_size);return 1 ; }io_uring_cqe_seen (ring, cqe);
Signature · 1 line
2.39.1
PATCH 7/7 tests/send: sends with offsets
In reply to Pavel BegunkovApart from arbitrary shifting buffers before registration so they're not page aligned, also add offsets to send requests.
--- test/send-zerocopy.c | 9 +++++++++ 1 file changed, 9 insertions(+)
test/send-zerocopy.c +9 -0
@@ -833,6 +833,15 @@ int main(int argc, char *argv[])return T_EXIT_FAIL ; } +if (buffers_iov[BUF_T_HUGETLB ].iov_base) { + buffers_iov[BUF_T_HUGETLB ].iov_base +=13 ; + buffers_iov[BUF_T_HUGETLB ].iov_len -=26 ; + } +if (buffers_iov[BUF_T_LARGE ].iov_base) { + buffers_iov[BUF_T_LARGE ].iov_base +=13 ; + buffers_iov[BUF_T_LARGE ].iov_len -=26 ; + } + ret =test_inet_send (&ring);if (ret) {fprintf (stderr,"test_inet_send() failed\n" );
Signature · 1 line
2.39.1
Quoted 12 lines · Pavel Begunkov
Add huge pages support for zc send benchmark and huge pages tests in
send-zerocopy.c.Pavel Begunkov (7):
tests/send: don't use a constant for page size
send: improve buffer iteration
send: test send with hugetlb
examples/zc: add a hugetlb option test/send: don't useSO_ZEROCOPYif not available tests/send: improve error reporting tests/send: sends with offsets
Quoted 1 line
[...]
Applied, thanks!
[1/7] tests/send: don't use a constant for page size
(no commit info)
[2/7] send: improve buffer iteration
(no commit info)
[3/7] send: test send with hugetlb
(no commit info)
[4/7] examples/zc: add a hugetlb option
(no commit info)
[5/7] test/send: don't use SO_ZEROCOPY if not available
(no commit info)
[6/7] tests/send: improve error reporting
(no commit info)
[7/7] tests/send: sends with offsets
(no commit info)
Best regards,
Signature · 1 line
Jens Axboe