Skip to contentSkip to conversation

multishot recvmsg docs and example

PATCH Applied Dylan Yudaken 4 participants 15 messages Jul 26, 2022 – Jul 27, 2022 io_uring

lore
Share & tools
lore
#0 lore raw

cover multishot recvmsg docs and example

Some multishot recvmsg patches for liburing:

Patches 1-3 cleanup the API a little while we're doing this. Patch 4 adds docs for the new API
Patch 5 adds an example UDP echo server that uses the API

Dylan Yudaken (5):
more consistent multishot recvmsg API parameter names order like functions together in liburing.h change io_uring_recvmsg_payload_length return type add documentation for multishot recvmsg
add an example for a UDP server

 .gitignore                            |   1 +
 examples/Makefile                     |   1 +
 examples/io_uring-udp.c               | 388 ++++++++++++++++++++++++++
 man/io_uring_prep_recvmsg.3           |  30 ++
 man/io_uring_prep_recvmsg_multishot.3 |   1 +
 man/io_uring_recvmsg_cmsg_firsthdr.3  |   1 +
 man/io_uring_recvmsg_cmsg_nexthdr.3   |   1 +
 man/io_uring_recvmsg_name.3           |   1 +
 man/io_uring_recvmsg_out.3            |  78 ++++++
 man/io_uring_recvmsg_payload.3        |   1 +
 man/io_uring_recvmsg_payload_length.3 |   1 +
 man/io_uring_recvmsg_validate.3       |   1 +
 src/include/liburing.h                |  48 ++--
 13 files changed, 529 insertions(+), 24 deletions(-)
 create mode 100644 examples/io_uring-udp.c
 create mode 120000 man/io_uring_prep_recvmsg_multishot.3
 create mode 120000 man/io_uring_recvmsg_cmsg_firsthdr.3
 create mode 120000 man/io_uring_recvmsg_cmsg_nexthdr.3
 create mode 120000 man/io_uring_recvmsg_name.3
 create mode 100644 man/io_uring_recvmsg_out.3
 create mode 120000 man/io_uring_recvmsg_payload.3
 create mode 120000 man/io_uring_recvmsg_payload_length.3
 create mode 120000 man/io_uring_recvmsg_validate.3

base-commit: 30a20795d7e4f300c606c6a2aa0a4c9492882d1d

Signature · 1 line
2.30.2
#1 lore raw

PATCH 1/5 more consistent multishot recvmsg API parameter names

Make the parameters more consistent with for example cmsg(3), and with itself (in the form of buf_len).

---
 src/include/liburing.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)
src/include/liburing.h +11 -11
index cffdabd71144..d3cbdb037b32 100644
@@ -734,9 +734,9 @@ static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe,
 }
 
 static inline struct io_uring_recvmsg_out *
-io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *m)
+io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *msgh)
 {
-	int header = m->msg_controllen + m->msg_namelen +
+	int header = msgh->msg_controllen + msgh->msg_namelen +
 				sizeof(struct io_uring_recvmsg_out);
 
 	if (buf_len < header)
@@ -751,42 +751,42 @@ static inline void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o)
 
 static inline struct cmsghdr *
 io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o,
-			       struct msghdr *m)
+			       struct msghdr *msgh)
 {
 	if (o->controllen < sizeof(struct cmsghdr))
 		return NULL;
 
 	return (struct cmsghdr *)((unsigned char *) io_uring_recvmsg_name(o) +
-			m->msg_namelen);
+			msgh->msg_namelen);
 }
 
 static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
-					     struct msghdr *m)
+					     struct msghdr *msgh)
 {
 	return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
-			m->msg_namelen + m->msg_controllen);
+			msgh->msg_namelen + msgh->msg_controllen);
 }
 
 static inline size_t
 io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
-				int buf_length, struct msghdr *m)
+				int buf_len, struct msghdr *msgh)
 {
 	unsigned long payload_start, payload_end;
 
-	payload_start = (unsigned long) io_uring_recvmsg_payload(o, m);
-	payload_end = (unsigned long) o + buf_length;
+	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
+	payload_end = (unsigned long) o + buf_len;
 	return payload_end - payload_start;
 }
 
 static inline struct cmsghdr *
-io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *m,
+io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh,
 			      struct cmsghdr *cmsg)
 {
 	unsigned char *end;
 
 	if (cmsg->cmsg_len < sizeof(struct cmsghdr))
 		return NULL;
-	end = (unsigned char *) io_uring_recvmsg_cmsg_firsthdr(o, m) +
+	end = (unsigned char *) io_uring_recvmsg_cmsg_firsthdr(o, msgh) +
 		o->controllen;
 	cmsg = (struct cmsghdr *)((unsigned char *) cmsg +
 			CMSG_ALIGN(cmsg->cmsg_len));
Signature · 1 line
2.30.2
#2 lore raw

PATCH 2/5 order like functions together in liburing.h

In reply to Dylan Yudaken

the cmsg related functions were not together - which was confusing

---
 src/include/liburing.h | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)
src/include/liburing.h +18 -18
index d3cbdb037b32..9e7f06f390be 100644
@@ -760,24 +760,6 @@ io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o,
 			msgh->msg_namelen);
 }
 
-static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
-					     struct msghdr *msgh)
-{
-	return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
-			msgh->msg_namelen + msgh->msg_controllen);
-}
-
-static inline size_t
-io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
-				int buf_len, struct msghdr *msgh)
-{
-	unsigned long payload_start, payload_end;
-
-	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
-	payload_end = (unsigned long) o + buf_len;
-	return payload_end - payload_start;
-}
-
 static inline struct cmsghdr *
 io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh,
 			      struct cmsghdr *cmsg)
@@ -799,6 +781,24 @@ io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msg
 	return cmsg;
 }
 
+static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
+					     struct msghdr *msgh)
+{
+	return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
+			msgh->msg_namelen + msgh->msg_controllen);
+}
+
+static inline size_t
+io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
+				int buf_len, struct msghdr *msgh)
+{
+	unsigned long payload_start, payload_end;
+
+	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
+	payload_end = (unsigned long) o + buf_len;
+	return payload_end - payload_start;
+}
+
 static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
 					const char *path, struct open_how *how)
 {
Signature · 1 line
2.30.2
#3 lore raw

PATCH 3/5 change io_uring_recvmsg_payload_length return type

In reply to Dylan Yudaken

Payloads can not be bigger than UINT_MAX, so no need to force a bigger type on the caller.

---
 src/include/liburing.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
src/include/liburing.h +2 -2
index 9e7f06f390be..06f4a50bacb1 100644
@@ -788,7 +788,7 @@ static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
 			msgh->msg_namelen + msgh->msg_controllen);
 }
 
-static inline size_t
+static inline unsigned int
 io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
 				int buf_len, struct msghdr *msgh)
 {
@@ -796,7 +796,7 @@ io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
 
 	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
 	payload_end = (unsigned long) o + buf_len;
-	return payload_end - payload_start;
+	return (unsigned int) (payload_end - payload_start);
 }
 
 static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
Signature · 1 line
2.30.2
#4 lore raw

PATCH 4/5 add documentation for multishot recvmsg

In reply to Dylan Yudaken

Add missing documentation for new multishot recvmsg API

---
 man/io_uring_prep_recvmsg.3           | 30 +++++++++++
 man/io_uring_prep_recvmsg_multishot.3 |  1 +
 man/io_uring_recvmsg_cmsg_firsthdr.3  |  1 +
 man/io_uring_recvmsg_cmsg_nexthdr.3   |  1 +
 man/io_uring_recvmsg_name.3           |  1 +
 man/io_uring_recvmsg_out.3            | 78 +++++++++++++++++++++++++++
 man/io_uring_recvmsg_payload.3        |  1 +
 man/io_uring_recvmsg_payload_length.3 |  1 +
 man/io_uring_recvmsg_validate.3       |  1 +
 9 files changed, 115 insertions(+)
 create mode 120000 man/io_uring_prep_recvmsg_multishot.3
 create mode 120000 man/io_uring_recvmsg_cmsg_firsthdr.3
 create mode 120000 man/io_uring_recvmsg_cmsg_nexthdr.3
 create mode 120000 man/io_uring_recvmsg_name.3
 create mode 100644 man/io_uring_recvmsg_out.3
 create mode 120000 man/io_uring_recvmsg_payload.3
 create mode 120000 man/io_uring_recvmsg_payload_length.3
 create mode 120000 man/io_uring_recvmsg_validate.3
man/io_uring_prep_recvmsg.3 +30 -0
index f64326be0a99..07096ee4826c 100644
@@ -15,6 +15,11 @@ io_uring_prep_recvmsg \- prepare a recvmsg request
 .BI "                           int " fd ","
 .BI "                           struct msghdr *" msg ","
 .BI "                           unsigned " flags ");"
+.PP
+.BI "void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *" sqe ","
+.BI "                                     int " fd ","
+.BI "                                     struct msghdr *" msg ","
+.BI "                                     unsigned " flags ");"
 .fi
 .SH DESCRIPTION
 .PP
@@ -37,6 +42,31 @@ This function prepares an async
 request. See that man page for details on the arguments specified to this
 prep helper.
 
+The multishot version allows the application to issue a single receive request,
+which repeatedly posts a CQE when data is available. It requires the
+.B IOSQE_BUFFER_SELECT
+flag to be set and no
+.B MSG_WAITALL
+flag to be set.
+Therefore each CQE will take a buffer out of a provided buffer pool for receiving.
+The application should check the flags of each CQE, regardless of it's result.
+If a posted CQE does not have the
+.B IORING_CQE_F_MORE
+flag set then the multishot receive will be done and the application should issue a
+new request.
+
+Unlike
+.BR recvmsg (2)
+, multishot recvmsg will prepend a
+.I struct io_uring_recvmsg_out
+which describes the layout of the rest of the buffer in combination with the initial
+.I struct msghdr
+submitted with the request. See
+.B io_uring_recvmsg_out (3)
+for more information on accessing the data.
+
+Multishot variants are available since kernel 5.20.
+
 After calling this function, additional io_uring internal modifier flags
 may be set in the SQE
 .I ioprio
man/io_uring_prep_recvmsg_multishot.3 new file +1 -0
new file mode 120000
index 000000000000..cd9566f2c2be
@@ -0,0 +1 @@
+io_uring_prep_recvmsg.3
\ No newline at end of file
man/io_uring_recvmsg_cmsg_firsthdr.3 new file +1 -0
new file mode 120000
index 000000000000..8eb17436288d
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
man/io_uring_recvmsg_cmsg_nexthdr.3 new file +1 -0
new file mode 120000
index 000000000000..8eb17436288d
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
man/io_uring_recvmsg_name.3 new file +1 -0
new file mode 120000
index 000000000000..8eb17436288d
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
man/io_uring_recvmsg_out.3 new file +78 -0

78 lines not on this page · raw message

man/io_uring_recvmsg_payload.3 new file +1 -0
new file mode 120000
index 000000000000..8eb17436288d
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
man/io_uring_recvmsg_payload_length.3 new file +1 -0
new file mode 120000
index 000000000000..8eb17436288d
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
man/io_uring_recvmsg_validate.3 new file +1 -0
new file mode 120000
index 000000000000..8eb17436288d
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
Signature · 1 line
2.30.2
#5 lore raw

PATCH 5/5 add an example for a UDP server

In reply to Dylan Yudaken

Add an example showing off multishot recvmsg and provided buffer rings. The application is a UDP echo server that responds to all packets with the same packet.

---
 .gitignore              |   1 +
 examples/Makefile       |   1 +
 examples/io_uring-udp.c | 388 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 390 insertions(+)
 create mode 100644 examples/io_uring-udp.c
.gitignore +1 -0
index b5acffd68c95..7853dbdf1a97 100644
@@ -13,6 +13,7 @@
 
 /examples/io_uring-cp
 /examples/io_uring-test
+/examples/io_uring-udp
 /examples/link-cp
 /examples/ucontext-cp
 
examples/Makefile +1 -0
index 1997a31dcc08..e561e05c1785 100644
@@ -13,6 +13,7 @@ endif
 example_srcs := \
 	io_uring-cp.c \
 	io_uring-test.c \
+	io_uring-udp.c \
 	link-cp.c \
 	poll-bench.c \
 	send-zerocopy.c
examples/io_uring-udp.c new file +388 -0

388 lines not on this page · raw message

Signature · 1 line
2.30.2
maintainer Applied
#6 lore raw
In reply to Dylan Yudaken
Quoted 8 lines · Dylan Yudaken

Some multishot recvmsg patches for liburing:

Patches 1-3 cleanup the API a little while we're doing this. Patch 4 adds docs for the new API
Patch 5 adds an example UDP echo server that uses the API

[...]

Applied, thanks!

[1/5] more consistent multishot recvmsg API parameter names

      commit: c025f206a8d7337109f148a738bf5df75aeed494
[2/5] order like functions together in liburing.h
      commit: b569b365c3c7d4ac78e8c2f482d5a227a980977f
[3/5] change io_uring_recvmsg_payload_length return type
      commit: 464ed8e15b3dbec2b44e087c4620dcffdc28a753
[4/5] add documentation for multishot recvmsg
      commit: 06f979baaae2c5a17f6b4233cb1f1d6720378149
[5/5] add an example for a UDP server
      commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

Best regards,

Signature · 1 line
Jens Axboe
#7 lore raw
Quoted 2 lines · Jens Axboe

[5/5] add an example for a UDP server

       commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

This one breaks clang-13 build, I'll send a patch.

Signature · 1 line
Ammar Faizi
maintainer
#8 lore raw
Quoted 7 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe

[5/5] add an example for a UDP server
       commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

This one breaks clang-13 build, I'll send a patch.

Hmm, built fine with clang-13/14 here?

Signature · 1 line
Jens Axboe
#9 lore raw
Quoted 8 lines · Jens Axboe
Quoted 5 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe

[5/5] add an example for a UDP server
       commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

This one breaks clang-13 build, I'll send a patch.

Hmm, built fine with clang-13/14 here?

Not sure what is going on, but clang-13 on my machine is not happy:

    io_uring-udp.c:134:18: error: incompatible pointer types passing \
    'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \
    [-Werror,-Wincompatible-pointer-types

    io_uring-udp.c:142:18: error: incompatible pointer types passing \
    'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \
    [-Werror,-Wincompatible-pointer-types]

Changing the compiler to GCC builds just fine. I have fixed something like this more than once. Strange indeed.

Signature · 1 line
Ammar Faizi
maintainer
#10 lore raw
Quoted 20 lines · Ammar Faizi
Quoted 8 lines · Jens Axboe
Quoted 5 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe

[5/5] add an example for a UDP server

        commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

This one breaks clang-13 build, I'll send a patch.

Hmm, built fine with clang-13/14 here?

Not sure what is going on, but clang-13 on my machine is not happy:

    io_uring-udp.c:134:18: error: incompatible pointer types passing \
    'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \
    [-Werror,-Wincompatible-pointer-types

    io_uring-udp.c:142:18: error: incompatible pointer types passing \
    'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \
    [-Werror,-Wincompatible-pointer-types]
Quoted 2 lines

Changing the compiler to GCC builds just fine. I have fixed something like this more than once. Strange indeed.

Regardless, the patch is sane!

Signature · 1 line
Jens Axboe
#11 lore raw
Quoted 26 lines · Jens Axboe
Quoted 24 lines · Ammar Faizi
Quoted 8 lines · Jens Axboe
Quoted 5 lines · Ammar Faizi
Quoted 2 lines · Jens Axboe

[5/5] add an example for a UDP server
        commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

This one breaks clang-13 build, I'll send a patch.

Hmm, built fine with clang-13/14 here?

Not sure what is going on, but clang-13 on my machine is not happy:

    io_uring-udp.c:134:18: error: incompatible pointer types passing \
    'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \
    [-Werror,-Wincompatible-pointer-types

    io_uring-udp.c:142:18: error: incompatible pointer types passing \
    'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \
    [-Werror,-Wincompatible-pointer-types]

Quoted 3 lines
Quoted 3 lines

Changing the compiler to GCC builds just fine. I have fixed something like
this more than once. Strange indeed.

Interestingly it did not show up on the Github CI either. What flags are you setting for this? Maybe the CI can be expanded to include those flags.
As you say its not the first time you've fixed this, or that I've done this.

Dylan

#12 lore raw
Quoted 5 lines · Dylan Yudaken

Interestingly it did not show up on the Github CI either. What flags are you setting for this? Maybe the CI can be expanded to include those flags.
As you say its not the first time you've fixed this, or that I've done this.

I use the same flag with the GitHub CI. Just a small experiment here...

I compile this with default compilation flags:

#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>

int main(void)

{
        struct sockaddr_in addr = { };
        return bind(0, &addr, sizeof(addr));
}

===============================================================================

ammarfaizi2@integral2:/tmp$ gcc test.c -o test
test.c: In function ‘main’:
test.c:9:24: warning: passing argument 2 of ‘bind’ from incompatible pointer type [-Wincompatible-pointer-types]
    9 |         return bind(0, &addr, sizeof(addr));
      |                        ^~~~~
      |                        |
      |                        struct sockaddr_in *
In file included from /usr/include/netinet/in.h:23,
                 from /usr/include/arpa/inet.h:22,
                 from test.c:2:
/usr/include/x86_64-linux-gnu/sys/socket.h:112:49: note: expected ‘const struct sockaddr *’ but argument is of type ‘struct sockaddr_in *’
  112 | extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
      |

===============================================================================

ammarfaizi2@integral2:/tmp$ clang test.c -o test
test.c:9:17: warning: incompatible pointer types passing 'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' [-Wincompatible-pointer-types]
        return bind(0, &addr, sizeof(addr));
                       ^~~~~
/usr/include/x86_64-linux-gnu/sys/socket.h:112:49: note: passing argument to parameter '__addr' here
extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
                                                ^
1 warning generated.

===============================================================================

Interestingly GCC also complains here, but it doesn't complain when compiling your code. Your code only breaks my clang-13.

What is the magic behind this?

We never disable -Wincompatible-pointer-types in liburing either. It's enabled by default.

Signature · 1 line
Ammar Faizi
#13 lore raw
Quoted 4 lines · Ammar Faizi

Interestingly GCC also complains here, but it doesn't complain when compiling your code. Your code only breaks my clang-13.

What is the magic behind this?

OK, I figured it out.

This work:

gcc -D_GNU_SOURCE test.c -o test;

These 3 break:

clang -D_GNU_SOURCE test.c -o test;
clang test.c -o test;
gcc test.c -o test;

So -D_GNU_SOURCE is the culprit. It seems to be unavoidable as the warn seems to be compiler specific or something. Maybe that _GNU_SOURCE patches the definition of bind().

Signature · 1 line
Ammar Faizi
#14 lore raw
Quoted 3 lines · Ammar Faizi

So -D_GNU_SOURCE is the culprit. It seems to be unavoidable as the warn seems to be compiler specific or something. Maybe that _GNU_SOURCE patches the definition of bind().

I did:

gcc -E -D_GNU_SOURCE test.c -o xtest.c

and examined the xtest.c output.

So basically when we use _GNU_SOURCE, sometimes the declaration of bind() is like this:

extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)

     __attribute__ ((__nothrow__ , __leaf__));

With __CONST_SOCKADDR_ARG being a union:

typedef union {
    const struct sockaddr *__restrict __sockaddr__;
    const struct sockaddr_at *__restrict __sockaddr_at__;
    const struct sockaddr_ax25 *__restrict __sockaddr_ax25__;
    const struct sockaddr_dl *__restrict __sockaddr_dl__;
    const struct sockaddr_eon *__restrict __sockaddr_eon__;
    const struct sockaddr_in *__restrict __sockaddr_in__;
    const struct sockaddr_in6 *__restrict __sockaddr_in6__;
    const struct sockaddr_inarp *__restrict __sockaddr_inarp__;
    const struct sockaddr_ipx *__restrict __sockaddr_ipx__;
    const struct sockaddr_iso *__restrict __sockaddr_iso__;
    const struct sockaddr_ns *__restrict __sockaddr_ns__;
    const struct sockaddr_un *__restrict __sockaddr_un__;
    const struct sockaddr_x25 *__restrict __sockaddr_x25__;
} __CONST_SOCKADDR_ARG __attribute__ ((__transparent_union__));

But not all header file included by the compiler has this union stuff. When it doesn't, it will throw a warning like that.

Signature · 1 line
Ammar Faizi