send-zc test/bench improvements
cover send-zc test/bench improvements
Patch 1 tries to resolve some misunderstandings from applications using the send zc test as an example.
Patches 2-5 are mostly quality of life improvements noticed while doing some tests.
Pavel Begunkov (5):
tests: comment on io_uring zc and SO_ZEROCOPY
examples/sendzc: remove get time overhead
examples/sendzc: use stdout for stats
examples/sendzc: try to print stats on SIGINT
examples/sendzc: improve help message
examples/send-zerocopy.c | 69 ++++++++++++++++++++++++++++++++-------- test/send-zerocopy.c | 4 +++ 2 files changed, 60 insertions(+), 13 deletions(-)
Signature · 1 line
2.43.0
PATCH 1/5 tests: comment on io_uring zc and SO_ZEROCOPY
Application writes can be using send-zerocopy.c as an example of io_uring zc enablement. However, it's a test, and we're testing all weird corner cases. One of them is setting SO_ZEROCOPY for io_uring zc, which is not needed, and applications should not do it. Add a comment in an attempt to limit misunderstanding.
--- test/send-zerocopy.c | 4 ++++ 1 file changed, 4 insertions(+)
test/send-zerocopy.c +4 -0
@@ -278,6 +278,10 @@ static int create_socketpair_ip(struct sockaddr_storage *addr,#ifdef SO_ZEROCOPY int val =1 ; +/* +* NOTE: apps must not set SO_ZEROCOPY when using io_uring zc. +* It's only here to test interactions with MSG_ZEROCOPY. +*/ if (setsockopt (*sock_client,SOL_SOCKET ,SO_ZEROCOPY , &val,sizeof (val))) {perror ("setsockopt zc" );return 1 ;
Signature · 1 line
2.43.0
PATCH 2/5 examples/sendzc: remove get time overhead
In reply to Pavel BegunkovWe call gettimeofday_ms() for each loop iteration, which annoyingly adds ~2% of overhead. We don't want to see it there, it should only be testing io_uring and the networking stack, so ammortise it to nothingness.
--- examples/send-zerocopy.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-)
examples/send-zerocopy.c +19 -9
@@ -52,6 +52,7 @@ struct thread_data {int idx;unsigned long long packets;unsigned long long bytes; +unsigned long long dt_ms;struct sockaddr_storage dst_addr;int fd; };
@@ -315,10 +316,11 @@ static void do_tx(struct thread_data *td, int domain, int type, int protocol)const int notif_slack =128 ;struct io_uring ring;struct iovec iov; -uint64_t tstop; +uint64_t tstart;int i, fd, ret;int compl_cqes =0 ;int ring_flags =IORING_SETUP_COOP_TASKRUN |IORING_SETUP_SINGLE_ISSUER ; +unsigned loop =0 ;if (cfg_defer_taskrun) ring_flags |=IORING_SETUP_DEFER_TASKRUN ;
@@ -357,7 +359,7 @@ static void do_tx(struct thread_data *td, int domain, int type, int protocol)pthread_barrier_wait (&barrier); - tstop =gettimeofday_ms () + cfg_runtime_ms; + tstart =gettimeofday_ms (); do { struct io_uring_sqe *sqe; struct io_uring_cqe *cqe;
@@ -419,7 +421,9 @@ static void do_tx(struct thread_data *td, int domain, int type, int protocol) }io_uring_cqe_seen (&ring, cqe); } - }while (gettimeofday_ms () < tstop); + }while ((++loop %16 !=0 ) ||gettimeofday_ms () < tstart + cfg_runtime_ms); + + td->dt_ms =gettimeofday_ms () - tstart; out_fail:shutdown (fd,SHUT_RDWR );
@@ -536,6 +540,7 @@ static void parse_opts(int argc, char **argv)int main (int argc,char **argv) { +unsigned long long tsum =0 ;unsigned long long packets =0 , bytes =0 ;struct thread_data *td;const char *cfg_test;
@@ -586,13 +591,18 @@ int main(int argc, char **argv)pthread_join (td->thread, &res); packets += td->packets; bytes += td->bytes; + tsum += td->dt_ms; + } + tsum = tsum / cfg_nr_threads; + +if (!tsum) { +fprintf (stderr,"The run is too short, can't gather stats\n" ); + }else { +fprintf (stderr,"packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n" , + packets, bytes >>20 , + packets *1000 / tsum, + (bytes >>20 ) *1000 / tsum); } - -fprintf (stderr,"packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n" , - packets, bytes >>20 , - packets / (cfg_runtime_ms /1000 ), - (bytes >>20 ) / (cfg_runtime_ms /1000 )); -pthread_barrier_destroy (&barrier);return 0 ; }
Signature · 1 line
2.43.0
PATCH 3/5 examples/sendzc: use stdout for stats
In reply to Pavel Begunkovstderr is not the right place to print a valid result of the program, use stdout.
--- examples/send-zerocopy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
examples/send-zerocopy.c +2 -2
@@ -596,9 +596,9 @@ int main(int argc, char **argv) tsum = tsum / cfg_nr_threads;if (!tsum) { -fprintf (stderr,"The run is too short, can't gather stats\n" ); +printf ("The run is too short, can't gather stats\n" ); }else { -fprintf (stderr,"packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n" , +printf ("packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n" , packets, bytes >>20 , packets *1000 / tsum, (bytes >>20 ) *1000 / tsum);
Signature · 1 line
2.43.0
PATCH 4/5 examples/sendzc: try to print stats on SIGINT
In reply to Pavel BegunkovIf interrupted in the middle of a long run, instead of silently crashing, as we currently do, we can try to print intermediate stats. That's a good use case, and it's always annoying loosing results when you forget about it.
--- examples/send-zerocopy.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
examples/send-zerocopy.c +16 -0
@@ -39,6 +39,7 @@#include <sys/wait.h> #include <sys/mman.h> #include <linux/mman.h> +#include <signal.h> #include "liburing.h"
@@ -82,6 +83,16 @@ static char *payload;static struct thread_data threads[MAX_THREADS ];static pthread_barrier_t barrier; +static bool should_stop = false; + +static void sigint_handler (int sig) +{ +/* kill if should_stop can't unblock threads fast enough */ +if (should_stop) +_exit (-1 ); + should_stop = true; +} +/* * Implementation of error(3), prints an error message and exits. */
@@ -421,6 +432,8 @@ static void do_tx(struct thread_data *td, int domain, int type, int protocol) }io_uring_cqe_seen (&ring, cqe); } +if (should_stop) +break ; }while ((++loop %16 !=0 ) ||gettimeofday_ms () < tstart + cfg_runtime_ms); td->dt_ms =gettimeofday_ms () - tstart;
@@ -582,6 +595,9 @@ int main(int argc, char **argv)if (cfg_rx)do_setup_rx (cfg_family, cfg_type,0 ); +if (!cfg_rx) +signal (SIGINT , sigint_handler); +for (i =0 ; i < cfg_nr_threads; i++)pthread_create (&threads[i].thread,NULL , !cfg_rx ? do_test : do_rx, &threads[i]);
Signature · 1 line
2.43.0
PATCH 5/5 examples/sendzc: improve help message
In reply to Pavel Begunkov--- examples/send-zerocopy.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-)
examples/send-zerocopy.c +21 -4
@@ -452,7 +452,6 @@ out_fail:io_uring_queue_exit (&ring); } -static void *do_test (void *arg) {struct thread_data *td = arg;
@@ -467,8 +466,24 @@ static void *do_test(void *arg)static void usage (const char *filepath) { -t_error (1 ,0 ,"Usage: %s [-n<N>] [-z<val>] [-s<payload size>] " -"(-4|-6) [-t<time s>] -D<dst_ip> udp" , filepath); +printf ("Usage:\t%s <protocol> <ip-version> -D<addr> [options]\n" , filepath); +printf ("\t%s <protocol> <ip-version> -R [options]\n\n" , filepath); + +printf (" -4\t\tUse IPv4\n" ); +printf (" -6\t\tUse IPv4\n" ); +printf (" -D <address>\tDestination address\n" ); +printf (" -p <port>\tServer port to listen on/connect to\n" ); +printf (" -s <size>\tBytes per request\n" ); +printf (" -s <size>\tBytes per request\n" ); +printf (" -n <nr>\tNumber of parallel requests\n" ); +printf (" -z <mode>\tZerocopy mode, 0 to disable, enabled otherwise\n" ); +printf (" -b <mode>\tUse registered buffers\n" ); +printf (" -l <mode>\tUse huge pages\n" ); +printf (" -d\t\tUse defer taskrun\n" ); +printf (" -C <cpu>\tPin to the specified CPU\n" ); +printf (" -T <nr>\tNumber of threads to use for sending\n" ); +printf (" -R\t\tPlay the server role\n" ); +printf (" -t <seconds>\tTime in seconds\n" ); }static void parse_opts (int argc,char **argv)
@@ -480,8 +495,10 @@ static void parse_opts(int argc, char **argv)int c;char *daddr =NULL ; -if (argc <=1 ) +if (argc <=1 ) {usage (argv[0 ]); +exit (0 ); + } cfg_payload_len = max_payload_len;
Signature · 1 line
2.43.0
Quoted 6 lines · Pavel Begunkov
Patch 1 tries to resolve some misunderstandings from applications using the send zc test as an example.
Patches 2-5 are mostly quality of life improvements noticed while doing some tests.
Quoted 8 lines
Pavel Begunkov (5):
tests: comment onio_uringzc andSO_ZEROCOPY
examples/sendzc: remove get time overhead
examples/sendzc: use stdout for stats
examples/sendzc: try to print stats on SIGINT
examples/sendzc: improve help message[...]
Applied, thanks!
[1/5] tests: comment on io_uring zc and SO_ZEROCOPY
commit: 88d99205fb4a6eb90dac88b0b44e04c88b74ef39
[2/5] examples/sendzc: remove get time overhead
commit: ec12f8c51af0afc3ebfa2db461c7737e5be012c4
[3/5] examples/sendzc: use stdout for stats
commit: 02866096e714c9e28616b88b5f79c509520999b2
[4/5] examples/sendzc: try to print stats on SIGINT
commit: 6bc6fe9266abe7054908cf10d6b3d0077f7d7465
[5/5] examples/sendzc: improve help message
commit: caa03326c5b6f6d5e013dd1c1a9adc64499e8cecBest regards,
Signature · 1 line
Jens Axboe