liburing: multishot receive
cover liburing: multishot receive
This adds tests and documentation for the multi shot receive functionality.
Patch 1 adds a helper t_create_socket_pair which provides two connected sockets without needing a hard coded port
Patch 2-4 adds multishot recv tests and docs
Dylan Yudaken (4):
add t_create_socket_pair
add IORING_RECV_MULTISHOT to io_uring.h
add recv-multishot test
add IORING_RECV_MULTISHOT docs
man/io_uring_prep_recv.3 | 15 ++ man/io_uring_prep_recvmsg.3 | 15 ++ src/include/liburing/io_uring.h | 53 ++++-- test/Makefile | 1 + test/helpers.c | 97 ++++++++++ test/helpers.h | 5 + test/recv-multishot.c | 308 ++++++++++++++++++++++++++++++++ 7 files changed, 480 insertions(+), 14 deletions(-) create mode 100644 test/recv-multishot.c
base-commit: 732bf609b670631731765a585a68d14ed3fdc9b7
Signature · 1 line
2.30.2
PATCH 1/4 add t_create_socket_pair
This is a useful tool for making networking tests, and does not require a hard coded port which is useful
--- test/helpers.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ test/helpers.h | 5 +++ 2 files changed, 102 insertions(+)
test/helpers.c +97 -0
@@ -10,6 +10,10 @@#include <unistd.h> #include <sys/types.h> +#include <arpa/inet.h> +#include <netinet/ip.h> +#include <netinet/tcp.h> +#include "helpers.h" #include "liburing.h"
@@ -143,3 +147,96 @@ enum t_setup_ret t_register_buffers(struct io_uring *ring,fprintf (stderr,"buffer register failed: %s\n" ,strerror (-ret));return ret; } + +int t_create_socket_pair (int fd[2 ],bool stream) +{ +int ret; +int type = stream ?SOCK_STREAM :SOCK_DGRAM ; +int val; +struct sockaddr_in serv_addr; +struct sockaddr *paddr; +size_t paddrlen; + + type |=SOCK_CLOEXEC ; + fd[0 ] =socket (AF_INET , type,0 ); +if (fd[0 ] <0 ) +return errno; + fd[1 ] =socket (AF_INET , type,0 ); +if (fd[1 ] <0 ) { + ret = errno; +close (fd[0 ]); +return ret; + } + + val =1 ; +if (setsockopt (fd[0 ],SOL_SOCKET ,SO_REUSEADDR , &val,sizeof (val))) + goto errno_cleanup; + +memset (&serv_addr,0 ,sizeof (serv_addr)); + serv_addr.sin_family =AF_INET ; + serv_addr.sin_port =0 ; +inet_pton (AF_INET ,"127.0.0.1" , &serv_addr.sin_addr); + + paddr = (struct sockaddr *)&serv_addr; + paddrlen =sizeof (serv_addr); + +if (bind (fd[0 ], paddr, paddrlen)) { +fprintf (stderr,"bind failed\n" ); + goto errno_cleanup; + } + +if (stream &&listen (fd[0 ],16 )) { +fprintf (stderr,"listen failed\n" ); + goto errno_cleanup; + } + +if (getsockname (fd[0 ], &serv_addr, (socklen_t *)&paddrlen)) { +fprintf (stderr,"getsockname failed\n" ); + goto errno_cleanup; + } +inet_pton (AF_INET ,"127.0.0.1" , &serv_addr.sin_addr); + +if (connect (fd[1 ], &serv_addr, paddrlen)) { +fprintf (stderr,"connect failed\n" ); + goto errno_cleanup; + } + +if (!stream) { +/* connect the other udp side */ +if (getsockname (fd[1 ], &serv_addr, (socklen_t *)&paddrlen)) { +fprintf (stderr,"getsockname failed\n" ); + goto errno_cleanup; + } +inet_pton (AF_INET ,"127.0.0.1" , &serv_addr.sin_addr); + +if (connect (fd[0 ], &serv_addr, paddrlen)) { +fprintf (stderr,"connect failed\n" ); + goto errno_cleanup; + } +return 0 ; + } + +/* for stream case we must accept and cleanup the listen socket */ + + ret =accept (fd[0 ],NULL ,NULL ); +if (ret <0 ) + goto errno_cleanup; + +close (fd[0 ]); + fd[0 ] = ret; + + val =1 ; +if (stream &&setsockopt (fd[0 ],SOL_TCP ,TCP_NODELAY , &val,sizeof (val))) + goto errno_cleanup; + val =1 ; +if (stream &&setsockopt (fd[1 ],SOL_TCP ,TCP_NODELAY , &val,sizeof (val))) + goto errno_cleanup; + +return 0 ; + +errno_cleanup: + ret = errno; +close (fd[0 ]); +close (fd[1 ]); +return ret; +}
test/helpers.h +5 -0
@@ -52,6 +52,11 @@ void t_create_file_pattern(const char *file, size_t size, char pattern);*/ struct iovec *t_create_buffers (size_t buf_num,size_t buf_size); +/* +* Helper for creating connected socket pairs +*/ +int t_create_socket_pair (int fd[2 ],bool stream); +/* * Helper for setting up a ring and checking for user privs */
Signature · 1 line
2.30.2
PATCH 2/4 add IORING_RECV_MULTISHOT to io_uring.h
In reply to Dylan Yudakencopy from include/uapi/linux/io_uring.h
--- src/include/liburing/io_uring.h | 53 ++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 14 deletions(-)
src/include/liburing/io_uring.h +39 -14
@@ -10,10 +10,7 @@#include <linux/fs.h> #include <linux/types.h> - -#ifdef __cplusplus -extern "C" { -#endif +#include <linux/time_types.h> /* * IO submission data structure (Submission Queue Entry)
@@ -26,6 +23,7 @@ struct io_uring_sqe {union {__u64 off;/* offset into file */ __u64 addr2; +__u32 cmd_op; };union {__u64 addr;/* pointer to buffer or iovecs */
@@ -65,8 +63,17 @@ struct io_uring_sqe {__s32 splice_fd_in;__u32 file_index; }; -__u64 addr3; -__u64 __pad2[1 ]; +union { +struct { +__u64 addr3; +__u64 __pad2[1 ]; + }; +/* +* If the ring is initialized with IORING_SETUP_SQE128, then +* this field is used for 80 bytes of arbitrary command data +*/ +__u8 cmd[0 ]; + }; };/*
@@ -131,9 +138,12 @@ enum {* IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN. */ #define IORING_SETUP_TASKRUN_FLAG (1U << 9) -#define IORING_SETUP_SQE128 (1U << 10)/* SQEs are 128 byte */ #define IORING_SETUP_CQE32 (1U << 11)/* CQEs are 32 byte */ +/* +* Only one task is allowed to submit requests +*/ +#define IORING_SETUP_SINGLE_ISSUER (1U << 12)enum io_uring_op {IORING_OP_NOP ,
@@ -220,10 +230,13 @@ enum io_uring_op {* * IORING_POLL_UPDATE Update existing poll request, matching * sqe->addr as the old user_data field. +* +* IORING_POLL_LEVEL Level triggered poll. */ #define IORING_POLL_ADD_MULTI (1U << 0)#define IORING_POLL_UPDATE_EVENTS (1U << 1)#define IORING_POLL_UPDATE_USER_DATA (1U << 2) +#define IORING_POLL_ADD_LEVEL (1U << 3)/* * ASYNC_CANCEL flags.
@@ -232,10 +245,12 @@ enum io_uring_op {* IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the * request 'user_data' * IORING_ASYNC_CANCEL_ANY Match any request +* IORING_ASYNC_CANCEL_FD_FIXED 'fd' passed in is a fixed descriptor */ #define IORING_ASYNC_CANCEL_ALL (1U << 0)#define IORING_ASYNC_CANCEL_FD (1U << 1) -#define IORING_ASYNC_CANCEL_ANY (1U << 2) +#define IORING_ASYNC_CANCEL_ANY (1U << 2) +#define IORING_ASYNC_CANCEL_FD_FIXED (1U << 3)/* * send/sendmsg and recv/recvmsg flags (sqe->addr2)
@@ -244,8 +259,13 @@ enum io_uring_op {* or receive and arm poll if that yields an * -EAGAIN result, arm poll upfront and skip * the initial transfer attempt. +* +* IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if +* the handler will continue to report +* CQEs on behalf of the same SQE. */ #define IORING_RECVSEND_POLL_FIRST (1U << 0) +#define IORING_RECV_MULTISHOT (1U << 1)/* * accept flags stored in sqe->ioprio
@@ -411,6 +431,9 @@ enum {IORING_REGISTER_PBUF_RING =22 ,IORING_UNREGISTER_PBUF_RING =23 , +/* sync cancelation API */ +IORING_REGISTER_SYNC_CANCEL =24 , +/* this goes last */ IORING_REGISTER_LAST };
@@ -547,12 +570,14 @@ struct io_uring_getevents_arg { };/* -* accept flags stored in sqe->ioprio +* Argument for IORING_REGISTER_SYNC_CANCEL */ -#define IORING_ACCEPT_MULTISHOT (1U << 0) - -#ifdef __cplusplus -} -#endif +struct io_uring_sync_cancel_reg { +__u64 addr; +__s32 fd; +__u32 flags; +struct __kernel_timespec timeout; +__u64 pad[4 ]; +};#endif
Signature · 1 line
2.30.2
Re: [PATCH liburing 2/4] add IORING_RECV_MULTISHOT to io_uring.h
Quoted 12 lines · Dylan Yudaken
copy from
include/uapi/linux/io_uring.hSigned-off-by Dylan Yudaken <[email protected]>--- src/include/liburing/io_uring.h | 53 ++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 14 deletions(-)src/include/liburing/io_uring.h +0
-0
src/include/liburing/io_uring.h line 16
Quoted 7 lines
src/include/liburing/io_uring.h +0
-4@@ -10,10 +10,7 @@#include <linux/fs.h> #include <linux/types.h> - -#ifdef __cplusplus -extern "C" { -#endif
Dylan,
That extern "C" thing is for C++, we shouldn't omit it.
Or better add that to the kernel tree as well, it won't break the kernel because we have a __cplusplus guard here.
Jens what do you think?
Just for reference, that line is introduced in commit:
commit 3d74c677c45eccf36b92f7ad4b3317adc1ed06bb
Author: Bart Van Assche <[email protected]>
Date: Sun Jun 28 12:58:19 2020 -0700
Make the liburing header files again compatible with C++
Include <atomic> instead of <stdatomic.h> if built with a C++ compiler.
src/include/liburing.h | 8 ++++---- src/include/liburing/barrier.h | 37 +++++++++++++++++++++++++++++++++++-- src/include/liburing/io_uring.h | 8 ++++++++ 3 files changed, 47 insertions(+), 6 deletions(-)
Adding the author to the CC list.
Signature · 1 line
Ammar Faizi
Jens Axboe
maintainerRe: [PATCH liburing 2/4] add IORING_RECV_MULTISHOT to io_uring.h
Quoted 22 lines · Ammar Faizi
Quoted 12 lines · Dylan Yudaken
copy from
include/uapi/linux/io_uring.hSigned-off-by Dylan Yudaken <[email protected]>--- src/include/liburing/io_uring.h | 53 ++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 14 deletions(-)src/include/liburing/io_uring.h +0
-0src/include/liburing/io_uring.h line 15
Quoted 6 lines
src/include/liburing/io_uring.h +0
-4@@ -10,10 +10,7 @@#include <linux/fs.h> #include <linux/types.h> - -#ifdef __cplusplus -extern "C" { -#endif Dylan,
Quoted 6 lines
That
extern "C"thing is for C++, we shouldn't omit it.Or better add that to the kernel tree as well, it won't break the kernel because we have a
__cplusplusguard here.Jens what do you think?
It'd be nice to keep them fully in sync. If I recall correctly, the only differences right now is that clause, and the change to not using a zero sized array at the end of a struct (which is slated for the kernel too).
Signature · 1 line
Jens Axboe
Re: [PATCH liburing 2/4] add IORING_RECV_MULTISHOT to io_uring.h
In reply to Ammar FaiziQuoted 48 lines · Ammar Faizi
Quoted 17 lines · Dylan Yudaken
copy from
include/uapi/linux/io_uring.hSigned-off-by Dylan Yudaken <[email protected]>---
src/include/liburing/io_uring.h| 53 ++++++++++++++++++++++++----
-----1 file changed, 39 insertions(+), 14 deletions(-)src/include/liburing/io_uring.h +0
-0@@ -10,10 +10,7 @@
#include <linux/fs.h>
#include <linux/types.h>src/include/liburing/io_uring.h line 16
Quoted 4 lines
src/include/liburing/io_uring.h +0
-4- -#ifdef __cplusplus -extern "C" { -#endif Dylan,
That
extern "C"thing is for C++, we shouldn't omit it.Or better add that to the kernel tree as well, it won't break the kernel because we have a
__cplusplusguard here.Jens what do you think?
Just for reference, that line is introduced in commit:
commit
3d74c677c45eccf36b92f7ad4b3317adc1ed06bb
Author: Bart Van Assche <[email protected]> Date: Sun Jun 28 12:58:19 2020 -0700Make the liburing header files again compatible with C++
Include <atomic> instead of <
stdatomic.h> if built with a C++ compiler.Fixes b9c0bf79aa87 ("src/include/liburing/barrier.h: Use C11atomics")
Signed-off-by Bart Van Assche <[email protected]>Signed-off-by Jens Axboe <[email protected]>
Quoted 8 lines
src/include/liburing.h | 8 ++++----
src/include/liburing/barrier.h | 37
+++++++++++++++++++++++++++++++++++--
src/include/liburing/io_uring.h| 8 ++++++++3 files changed, 47 insertions(+), 6 deletions(-)Adding the author to the CC list.
Yeah this makes sense. I think I just assumed the file was a manual copy of the latest kernel one which it clearly is not.
I'll fix it up in v2
PATCH 3/4 add recv-multishot test
In reply to Dylan Yudakenadd a test for multishot receive functionality
--- test/Makefile | 1 + test/recv-multishot.c | 308 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 test/recv-multishot.c
test/Makefile +1 -0
@@ -124,6 +124,7 @@ test_srcs := \ read-write.c \ recv-msgall.c \ recv-msgall-stream.c \ + recv-multishot.c \ register-restrictions.c \ rename.c \ ringbuf-read.c \
test/recv-multishot.c new file +308 -0
308 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH 4/4 add IORING_RECV_MULTISHOT docs
In reply to Dylan Yudakenadd appropriate docs to io_uring_prep_recvmsg/io_uring_prep_recv man pages
--- man/io_uring_prep_recv.3 | 15 +++++++++++++++ man/io_uring_prep_recvmsg.3 | 15 +++++++++++++++ 2 files changed, 30 insertions(+)
man/io_uring_prep_recv.3 +15 -0
@@ -60,6 +60,21 @@ operation. If set, the socket still had data to be read after the operation completed. Both these flags are available since 5.19. .P +.TP +.B IORING_RECV_MULTISHOT +If set, io_uring will post a new CQE whenever data is available to read. +The CQE will try and fill a provided buffer. +The +.B IORING_CQE_F_MORE +flag will be set if the receive is still active and more CQEs will be posted. + +It is required that the +.B IOSQE_BUFFER_SELECT +flag is set and that +.B MSG_WAITALL +is not set. +.P + .SH RETURN VALUE None .SH ERRORS
man/io_uring_prep_recvmsg.3 +15 -0
@@ -61,6 +61,21 @@ operation. If set, the socket still had data to be read after the operation completed. Both these flags are available since 5.19. .P +.TP +.B IORING_RECV_MULTISHOT +If set, io_uring will post a new CQE whenever data is available to read. +The CQE will try and fill a provided buffer. +The +.B IORING_CQE_F_MORE +flag will be set if the receive is still active and more CQEs will be posted. + +It is required that the +.B IOSQE_BUFFER_SELECT +flag is set and that +.B MSG_WAITALL +is not set. +.P + .SH RETURN VALUE None .SH ERRORS
Signature · 1 line
2.30.2