multishot recvmsg docs and example
cover multishot recvmsg docs and example
Some multishot recvmsg patches for liburing:
Patches 1-3 cleanup the API a little while we're doing this. Patch 4 adds docs for the new API
Patch 5 adds an example UDP echo server that uses the API
Dylan Yudaken (5):
more consistent multishot recvmsg API parameter names order like functions together in liburing.h change io_uring_recvmsg_payload_length return type add documentation for multishot recvmsg
add an example for a UDP server
.gitignore | 1 + examples/Makefile | 1 + examples/io_uring-udp.c | 388 ++++++++++++++++++++++++++ man/io_uring_prep_recvmsg.3 | 30 ++ man/io_uring_prep_recvmsg_multishot.3 | 1 + man/io_uring_recvmsg_cmsg_firsthdr.3 | 1 + man/io_uring_recvmsg_cmsg_nexthdr.3 | 1 + man/io_uring_recvmsg_name.3 | 1 + man/io_uring_recvmsg_out.3 | 78 ++++++ man/io_uring_recvmsg_payload.3 | 1 + man/io_uring_recvmsg_payload_length.3 | 1 + man/io_uring_recvmsg_validate.3 | 1 + src/include/liburing.h | 48 ++-- 13 files changed, 529 insertions(+), 24 deletions(-) create mode 100644 examples/io_uring-udp.c create mode 120000 man/io_uring_prep_recvmsg_multishot.3 create mode 120000 man/io_uring_recvmsg_cmsg_firsthdr.3 create mode 120000 man/io_uring_recvmsg_cmsg_nexthdr.3 create mode 120000 man/io_uring_recvmsg_name.3 create mode 100644 man/io_uring_recvmsg_out.3 create mode 120000 man/io_uring_recvmsg_payload.3 create mode 120000 man/io_uring_recvmsg_payload_length.3 create mode 120000 man/io_uring_recvmsg_validate.3
base-commit: 30a20795d7e4f300c606c6a2aa0a4c9492882d1d
Signature · 1 line
2.30.2
PATCH 1/5 more consistent multishot recvmsg API parameter names
Make the parameters more consistent with for example cmsg(3), and with itself (in the form of buf_len).
--- src/include/liburing.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-)
src/include/liburing.h +11 -11
@@ -734,9 +734,9 @@ static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe, }static inline struct io_uring_recvmsg_out * -io_uring_recvmsg_validate (void *buf,int buf_len,struct msghdr *m) +io_uring_recvmsg_validate (void *buf,int buf_len,struct msghdr *msgh) { -int header = m->msg_controllen + m->msg_namelen + +int header = msgh->msg_controllen + msgh->msg_namelen +sizeof (struct io_uring_recvmsg_out );if (buf_len < header)
@@ -751,42 +751,42 @@ static inline void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o)static inline struct cmsghdr *io_uring_recvmsg_cmsg_firsthdr (struct io_uring_recvmsg_out *o, -struct msghdr *m) +struct msghdr *msgh) {if (o->controllen <sizeof (struct cmsghdr ))return NULL ;return (struct cmsghdr *)((unsigned char *)io_uring_recvmsg_name (o) + - m->msg_namelen); + msgh->msg_namelen); }static inline void *io_uring_recvmsg_payload (struct io_uring_recvmsg_out *o, -struct msghdr *m) +struct msghdr *msgh) {return (void *)((unsigned char *)io_uring_recvmsg_name (o) + - m->msg_namelen + m->msg_controllen); + msgh->msg_namelen + msgh->msg_controllen); }static inline size_t io_uring_recvmsg_payload_length (struct io_uring_recvmsg_out *o, -int buf_length,struct msghdr *m) +int buf_len,struct msghdr *msgh) {unsigned long payload_start, payload_end; - payload_start = (unsigned long )io_uring_recvmsg_payload (o, m); - payload_end = (unsigned long ) o + buf_length; + payload_start = (unsigned long )io_uring_recvmsg_payload (o, msgh); + payload_end = (unsigned long ) o + buf_len;return payload_end - payload_start; }static inline struct cmsghdr * -io_uring_recvmsg_cmsg_nexthdr (struct io_uring_recvmsg_out *o,struct msghdr *m, +io_uring_recvmsg_cmsg_nexthdr (struct io_uring_recvmsg_out *o,struct msghdr *msgh,struct cmsghdr *cmsg) {unsigned char *end;if (cmsg->cmsg_len <sizeof (struct cmsghdr ))return NULL ; - end = (unsigned char *)io_uring_recvmsg_cmsg_firsthdr (o, m) + + end = (unsigned char *)io_uring_recvmsg_cmsg_firsthdr (o, msgh) + o->controllen; cmsg = (struct cmsghdr *)((unsigned char *) cmsg +CMSG_ALIGN (cmsg->cmsg_len));
Signature · 1 line
2.30.2
PATCH 2/5 order like functions together in liburing.h
In reply to Dylan Yudakenthe cmsg related functions were not together - which was confusing
--- src/include/liburing.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-)
src/include/liburing.h +18 -18
@@ -760,24 +760,6 @@ io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o, msgh->msg_namelen); } -static inline void *io_uring_recvmsg_payload (struct io_uring_recvmsg_out *o, -struct msghdr *msgh) -{ -return (void *)((unsigned char *)io_uring_recvmsg_name (o) + - msgh->msg_namelen + msgh->msg_controllen); -} - -static inline size_t -io_uring_recvmsg_payload_length (struct io_uring_recvmsg_out *o, -int buf_len,struct msghdr *msgh) -{ -unsigned long payload_start, payload_end; - - payload_start = (unsigned long )io_uring_recvmsg_payload (o, msgh); - payload_end = (unsigned long ) o + buf_len; -return payload_end - payload_start; -} -static inline struct cmsghdr *io_uring_recvmsg_cmsg_nexthdr (struct io_uring_recvmsg_out *o,struct msghdr *msgh,struct cmsghdr *cmsg)
@@ -799,6 +781,24 @@ io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgreturn cmsg; } +static inline void *io_uring_recvmsg_payload (struct io_uring_recvmsg_out *o, +struct msghdr *msgh) +{ +return (void *)((unsigned char *)io_uring_recvmsg_name (o) + + msgh->msg_namelen + msgh->msg_controllen); +} + +static inline size_t +io_uring_recvmsg_payload_length (struct io_uring_recvmsg_out *o, +int buf_len,struct msghdr *msgh) +{ +unsigned long payload_start, payload_end; + + payload_start = (unsigned long )io_uring_recvmsg_payload (o, msgh); + payload_end = (unsigned long ) o + buf_len; +return payload_end - payload_start; +} +static inline void io_uring_prep_openat2 (struct io_uring_sqe *sqe,int dfd,const char *path,struct open_how *how) {
Signature · 1 line
2.30.2
PATCH 3/5 change io_uring_recvmsg_payload_length return type
In reply to Dylan YudakenPayloads can not be bigger than UINT_MAX, so no need to force a bigger type on the caller.
--- src/include/liburing.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
src/include/liburing.h +2 -2
@@ -788,7 +788,7 @@ static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o, msgh->msg_namelen + msgh->msg_controllen); } -static inline size_t +static inline unsigned int io_uring_recvmsg_payload_length (struct io_uring_recvmsg_out *o,int buf_len,struct msghdr *msgh) {
@@ -796,7 +796,7 @@ io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o, payload_start = (unsigned long )io_uring_recvmsg_payload (o, msgh); payload_end = (unsigned long ) o + buf_len; -return payload_end - payload_start; +return (unsigned int ) (payload_end - payload_start); } static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
Signature · 1 line
2.30.2
PATCH 4/5 add documentation for multishot recvmsg
In reply to Dylan YudakenAdd missing documentation for new multishot recvmsg API
--- man/io_uring_prep_recvmsg.3 | 30 +++++++++++ man/io_uring_prep_recvmsg_multishot.3 | 1 + man/io_uring_recvmsg_cmsg_firsthdr.3 | 1 + man/io_uring_recvmsg_cmsg_nexthdr.3 | 1 + man/io_uring_recvmsg_name.3 | 1 + man/io_uring_recvmsg_out.3 | 78 +++++++++++++++++++++++++++ man/io_uring_recvmsg_payload.3 | 1 + man/io_uring_recvmsg_payload_length.3 | 1 + man/io_uring_recvmsg_validate.3 | 1 + 9 files changed, 115 insertions(+) create mode 120000 man/io_uring_prep_recvmsg_multishot.3 create mode 120000 man/io_uring_recvmsg_cmsg_firsthdr.3 create mode 120000 man/io_uring_recvmsg_cmsg_nexthdr.3 create mode 120000 man/io_uring_recvmsg_name.3 create mode 100644 man/io_uring_recvmsg_out.3 create mode 120000 man/io_uring_recvmsg_payload.3 create mode 120000 man/io_uring_recvmsg_payload_length.3 create mode 120000 man/io_uring_recvmsg_validate.3
man/io_uring_prep_recvmsg.3 +30 -0
@@ -15,6 +15,11 @@ io_uring_prep_recvmsg \- prepare a recvmsg request .BI " int " fd "," .BI " struct msghdr *" msg "," .BI " unsigned " flags ");" +.PP +.BI "void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *" sqe "," +.BI " int " fd "," +.BI " struct msghdr *" msg "," +.BI " unsigned " flags ");" .fi .SH DESCRIPTION .PP
@@ -37,6 +42,31 @@ This function prepares an async request. See that man page for details on the arguments specified to this prep helper. +The multishot version allows the application to issue a single receive request, +which repeatedly posts a CQE when data is available. It requires the +.B IOSQE_BUFFER_SELECT +flag to be set and no +.B MSG_WAITALL +flag to be set. +Therefore each CQE will take a buffer out of a provided buffer pool for receiving. +The application should check the flags of each CQE, regardless of it's result. +If a posted CQE does not have the +.B IORING_CQE_F_MORE +flag set then the multishot receive will be done and the application should issue a +new request. + +Unlike +.BR recvmsg (2) +, multishot recvmsg will prepend a +.I struct io_uring_recvmsg_out +which describes the layout of the rest of the buffer in combination with the initial +.I struct msghdr +submitted with the request. See +.B io_uring_recvmsg_out (3) +for more information on accessing the data. + +Multishot variants are available since kernel 5.20. + After calling this function, additional io_uring internal modifier flags may be set in the SQE .I ioprio
man/io_uring_prep_recvmsg_multishot.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_prep_recvmsg.3
man/io_uring_recvmsg_cmsg_firsthdr.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_recvmsg_out.3
man/io_uring_recvmsg_cmsg_nexthdr.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_recvmsg_out.3
man/io_uring_recvmsg_name.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_recvmsg_out.3
man/io_uring_recvmsg_out.3 new file +78 -0
78 lines not on this page · raw message
man/io_uring_recvmsg_payload.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_recvmsg_out.3
man/io_uring_recvmsg_payload_length.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_recvmsg_out.3
man/io_uring_recvmsg_validate.3 new file +1 -0
@@ -0,0 +1 @@ +io_uring_recvmsg_out.3
Signature · 1 line
2.30.2
PATCH 5/5 add an example for a UDP server
In reply to Dylan YudakenAdd an example showing off multishot recvmsg and provided buffer rings. The application is a UDP echo server that responds to all packets with the same packet.
--- .gitignore | 1 + examples/Makefile | 1 + examples/io_uring-udp.c | 388 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100644 examples/io_uring-udp.c
.gitignore +1 -0
@@ -13,6 +13,7 @@ /examples/io_uring-cp /examples/io_uring-test +/examples/io_uring-udp /examples/link-cp /examples/ucontext-cp
examples/Makefile +1 -0
@@ -13,6 +13,7 @@ endif example_srcs := \ io_uring-cp.c \ io_uring-test.c \ + io_uring-udp.c \ link-cp.c \ poll-bench.c \ send-zerocopy.c
examples/io_uring-udp.c new file +388 -0
388 lines not on this page · raw message
Signature · 1 line
2.30.2
Quoted 8 lines · Dylan Yudaken
Some multishot recvmsg patches for liburing:
Patches 1-3 cleanup the API a little while we're doing this. Patch 4 adds docs for the new API
Patch 5 adds an example UDP echo server that uses the API[...]
Applied, thanks!
[1/5] more consistent multishot recvmsg API parameter names
commit: c025f206a8d7337109f148a738bf5df75aeed494
[2/5] order like functions together in liburing.h
commit: b569b365c3c7d4ac78e8c2f482d5a227a980977f
[3/5] change io_uring_recvmsg_payload_length return type
commit: 464ed8e15b3dbec2b44e087c4620dcffdc28a753
[4/5] add documentation for multishot recvmsg
commit: 06f979baaae2c5a17f6b4233cb1f1d6720378149
[5/5] add an example for a UDP server
commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484Best regards,
Signature · 1 line
Jens Axboe
Quoted 2 lines · Jens Axboe
[5/5] add an example for a UDP server
commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484
This one breaks clang-13 build, I'll send a patch.
Signature · 1 line
Ammar Faizi
Jens Axboe
maintainerQuoted 7 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe
[5/5] add an example for a UDP server
commit:61d472b51e761e61cbf46caea40aaf40d8ed1484This one breaks clang-13 build, I'll send a patch.
Hmm, built fine with clang-13/14 here?
Signature · 1 line
Jens Axboe
Quoted 8 lines · Jens Axboe
Quoted 5 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe
[5/5] add an example for a UDP server
commit:61d472b51e761e61cbf46caea40aaf40d8ed1484This one breaks clang-13 build, I'll send a patch.
Hmm, built fine with clang-13/14 here?
Not sure what is going on, but clang-13 on my machine is not happy:
io_uring-udp.c:134:18: error: incompatible pointer types passing \
'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \
[-Werror,-Wincompatible-pointer-types
io_uring-udp.c:142:18: error: incompatible pointer types passing \
'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \
[-Werror,-Wincompatible-pointer-types]Changing the compiler to GCC builds just fine. I have fixed something like this more than once. Strange indeed.
Signature · 1 line
Ammar Faizi
Jens Axboe
maintainerQuoted 20 lines · Ammar Faizi
Quoted 8 lines · Jens Axboe
Quoted 5 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe
[5/5] add an example for a UDP server
commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484This one breaks clang-13 build, I'll send a patch.
Hmm, built fine with clang-13/14 here?
Not sure what is going on, but clang-13 on my machine is not happy:
io_uring-udp.c:134:18: error: incompatible pointer types passing \ 'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \ [-Werror,-Wincompatible-pointer-types io_uring-udp.c:142:18: error: incompatible pointer types passing \ 'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \ [-Werror,-Wincompatible-pointer-types]
Quoted 2 lines
Changing the compiler to GCC builds just fine. I have fixed something like this more than once. Strange indeed.
Regardless, the patch is sane!
Signature · 1 line
Jens Axboe
Quoted 26 lines · Jens Axboe
Quoted 24 lines · Ammar Faizi
Quoted 8 lines · Jens Axboe
Quoted 5 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe
[5/5] add an example for a UDP server
commit:61d472b51e761e61cbf46caea40aaf40d8ed1484This one breaks clang-13 build, I'll send a patch.
Hmm, built fine with clang-13/14 here?
Not sure what is going on, but clang-13 on my machine is not happy:
io_uring-udp.c:134:18: error: incompatible pointer types passing \
'structsockaddr_in6*' to parameter of type 'const struct sockaddr *' \
[-Werror,-Wincompatible-pointer-typesio_uring-udp.c:142:18: error: incompatible pointer types passing \
'structsockaddr_in*' to parameter of type 'const struct sockaddr *' \
[-Werror,-Wincompatible-pointer-types]
Quoted 3 lines
Quoted 3 lines
Changing the compiler to GCC builds just fine. I have fixed something like
this more than once. Strange indeed.
Interestingly it did not show up on the Github CI either. What flags are you setting for this? Maybe the CI can be expanded to include those flags.
As you say its not the first time you've fixed this, or that I've done this.
Dylan
Quoted 5 lines · Dylan Yudaken
Interestingly it did not show up on the Github CI either. What flags are you setting for this? Maybe the CI can be expanded to include those flags.
As you say its not the first time you've fixed this, or that I've done this.
I use the same flag with the GitHub CI. Just a small experiment here...
I compile this with default compilation flags:
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
int main(void)
{
struct sockaddr_in addr = { };
return bind(0, &addr, sizeof(addr));
}===============================================================================
ammarfaizi2@integral2:/tmp$ gcc test.c -o test
test.c: In function ‘main’:
test.c:9:24: warning: passing argument 2 of ‘bind’ from incompatible pointer type [-Wincompatible-pointer-types]
9 | return bind(0, &addr, sizeof(addr));
| ^~~~~
| |
| struct sockaddr_in *
In file included from /usr/include/netinet/in.h:23,
from /usr/include/arpa/inet.h:22,
from test.c:2:
/usr/include/x86_64-linux-gnu/sys/socket.h:112:49: note: expected ‘const struct sockaddr *’ but argument is of type ‘struct sockaddr_in *’
112 | extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
|===============================================================================
ammarfaizi2@integral2:/tmp$ clang test.c -o test
test.c:9:17: warning: incompatible pointer types passing 'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' [-Wincompatible-pointer-types]
return bind(0, &addr, sizeof(addr));
^~~~~
/usr/include/x86_64-linux-gnu/sys/socket.h:112:49: note: passing argument to parameter '__addr' here
extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
^
1 warning generated.===============================================================================
Interestingly GCC also complains here, but it doesn't complain when compiling your code. Your code only breaks my clang-13.
What is the magic behind this?
We never disable -Wincompatible-pointer-types in liburing either. It's enabled by default.
Signature · 1 line
Ammar Faizi
Quoted 4 lines · Ammar Faizi
Interestingly GCC also complains here, but it doesn't complain when compiling your code. Your code only breaks my clang-13.
What is the magic behind this?
OK, I figured it out.
This work:
gcc -D_GNU_SOURCE test.c -o test;
These 3 break:
clang -D_GNU_SOURCE test.c -o test;
clang test.c -o test;
gcc test.c -o test;
So -D_GNU_SOURCE is the culprit. It seems to be unavoidable as the warn seems to be compiler specific or something. Maybe that _GNU_SOURCE patches the definition of bind().
Signature · 1 line
Ammar Faizi
Quoted 3 lines · Ammar Faizi
So -D_GNU_SOURCE is the culprit. It seems to be unavoidable as the warn seems to be compiler specific or something. Maybe that
_GNU_SOURCEpatches the definition ofbind().
I did:
gcc -E -D_GNU_SOURCE test.c -o xtest.c
and examined the xtest.c output.
So basically when we use _GNU_SOURCE, sometimes the declaration of bind() is like this:
extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
__attribute__ ((__nothrow__ , __leaf__));
With __CONST_SOCKADDR_ARG being a union:
typedef union {
const struct sockaddr *__restrict __sockaddr__;
const struct sockaddr_at *__restrict __sockaddr_at__;
const struct sockaddr_ax25 *__restrict __sockaddr_ax25__;
const struct sockaddr_dl *__restrict __sockaddr_dl__;
const struct sockaddr_eon *__restrict __sockaddr_eon__;
const struct sockaddr_in *__restrict __sockaddr_in__;
const struct sockaddr_in6 *__restrict __sockaddr_in6__;
const struct sockaddr_inarp *__restrict __sockaddr_inarp__;
const struct sockaddr_ipx *__restrict __sockaddr_ipx__;
const struct sockaddr_iso *__restrict __sockaddr_iso__;
const struct sockaddr_ns *__restrict __sockaddr_ns__;
const struct sockaddr_un *__restrict __sockaddr_un__;
const struct sockaddr_x25 *__restrict __sockaddr_x25__;
} __CONST_SOCKADDR_ARG __attribute__ ((__transparent_union__));But not all header file included by the compiler has this union stuff. When it doesn't, it will throw a warning like that.
Signature · 1 line
Ammar Faizi