Skip to contentSkip to conversation

zerocopy send headers and tests

PATCH Applied Pavel Begunkov 3 participants 10 messages Jul 25, 2022 io_uring

lore
Share & tools
lore
#0 lore raw

v2 cover zerocopy send headers and tests

Add zerocopy send headers, helpers and tests

v2:
use T_EXIT_*
fix ptr <-> int conversions for 32 bits arches slight renaming
get rid of error() in the test
add patch 5/5

Pavel Begunkov (5):
io_uring.h: sync with kernel for zc send and notifiers
liburing: add zc send and notif helpers
tests: add tests for zerocopy send and notifications
examples: add a zerocopy send example
liburing: improve fallocate typecasting

 examples/Makefile               |   3 +-
 examples/send-zerocopy.c        | 366 +++++++++++++
 src/include/liburing.h          |  47 +-
 src/include/liburing/io_uring.h |  37 +-
 src/liburing.map                |   2 +
 src/register.c                  |  20 +
 test/Makefile                   |   1 +
 test/send-zerocopy.c            | 888 ++++++++++++++++++++++++++++++++
 8 files changed, 1358 insertions(+), 6 deletions(-)
 create mode 100644 examples/send-zerocopy.c
 create mode 100644 test/send-zerocopy.c
Signature · 1 line
2.37.0
#1 lore raw

PATCH v2 1/5 io_uring.h: sync with kernel for zc send and notifiers

---
 src/include/liburing/io_uring.h | 37 +++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)
src/include/liburing/io_uring.h +35 -2
index 99e6963..3953807 100644
@@ -64,6 +64,10 @@ struct io_uring_sqe {
 	union {
 		__s32	splice_fd_in;
 		__u32	file_index;
+		struct {
+			__u16	notification_idx;
+			__u16	addr_len;
+		};
 	};
 	__u64	addr3;
 	__u64	__pad2[1];
@@ -160,7 +164,8 @@ enum io_uring_op {
 	IORING_OP_FALLOCATE,
 	IORING_OP_OPENAT,
 	IORING_OP_CLOSE,
-	IORING_OP_FILES_UPDATE,
+	IORING_OP_RSRC_UPDATE,
+	IORING_OP_FILES_UPDATE = IORING_OP_RSRC_UPDATE,
 	IORING_OP_STATX,
 	IORING_OP_READ,
 	IORING_OP_WRITE,
@@ -187,6 +192,7 @@ enum io_uring_op {
 	IORING_OP_GETXATTR,
 	IORING_OP_SOCKET,
 	IORING_OP_URING_CMD,
+	IORING_OP_SENDZC_NOTIF,
 
 	/* this goes last, obviously */
 	IORING_OP_LAST,
@@ -208,6 +214,7 @@ enum io_uring_op {
 #define IORING_TIMEOUT_ETIME_SUCCESS	(1U << 5)
 #define IORING_TIMEOUT_CLOCK_MASK	(IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
 #define IORING_TIMEOUT_UPDATE_MASK	(IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
+
 /*
  * sqe->splice_flags
  * extends splice(2) flags
@@ -254,13 +261,23 @@ enum io_uring_op {
  *				CQEs on behalf of the same SQE.
  */
 #define IORING_RECVSEND_POLL_FIRST	(1U << 0)
-#define IORING_RECV_MULTISHOT	(1U << 1)
+#define IORING_RECV_MULTISHOT		(1U << 1)
+#define IORING_RECVSEND_FIXED_BUF	(1U << 2)
+#define IORING_RECVSEND_NOTIF_FLUSH	(1U << 3)
 
 /*
  * accept flags stored in sqe->ioprio
  */
 #define IORING_ACCEPT_MULTISHOT	(1U << 0)
 
+/*
+ * IORING_OP_RSRC_UPDATE flags
+ */
+enum {
+	IORING_RSRC_UPDATE_FILES,
+	IORING_RSRC_UPDATE_NOTIF,
+};
+
 /*
  * IO completion data structure (Completion Queue Entry)
  */
@@ -426,6 +443,9 @@ enum {
 	/* register a range of fixed file slots for automatic slot allocation */
 	IORING_REGISTER_FILE_ALLOC_RANGE	= 25,
 
+	IORING_REGISTER_NOTIFIERS		= 26,
+	IORING_UNREGISTER_NOTIFIERS		= 27,
+
 	/* this goes last */
 	IORING_REGISTER_LAST
 };
@@ -472,6 +492,19 @@ struct io_uring_rsrc_update2 {
 	__u32 resv2;
 };
 
+struct io_uring_notification_slot {
+	__u64 tag;
+	__u64 resv[3];
+};
+
+struct io_uring_notification_register {
+	__u32 nr_slots;
+	__u32 resv;
+	__u64 resv2;
+	__u64 data;
+	__u64 resv3;
+};
+
 /* Skip updating fd indexes set to this value in the fd table */
 #define IORING_REGISTER_FILES_SKIP	(-2)
 
Signature · 1 line
2.37.0
#2 lore raw

PATCH v2 2/5 liburing: add zc send and notif helpers

In reply to Pavel Begunkov

Add helpers for notification registration and preparing zerocopy send requests.

---
 src/include/liburing.h | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/liburing.map       |  2 ++
 src/register.c         | 20 ++++++++++++++++++++
 3 files changed, 64 insertions(+)
src/include/liburing.h +42 -0
index fc7613d..20cd308 100644
@@ -189,6 +189,10 @@ int io_uring_register_sync_cancel(struct io_uring *ring,
 int io_uring_register_file_alloc_range(struct io_uring *ring,
 					unsigned off, unsigned len);
 
+int io_uring_register_notifications(struct io_uring *ring, unsigned nr,
+				    struct io_uring_notification_slot *slots);
+int io_uring_unregister_notifications(struct io_uring *ring);
+
 /*
  * Helper for the peek/wait single cqe functions. Exported because of that,
  * but probably shouldn't be used directly in an application.
@@ -677,6 +681,44 @@ static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
 	sqe->msg_flags = (__u32) flags;
 }
 
+static inline void io_uring_prep_sendzc(struct io_uring_sqe *sqe, int sockfd,
+				        const void *buf, size_t len, int flags,
+				        unsigned slot_idx, unsigned zc_flags)
+{
+	io_uring_prep_rw(IORING_OP_SENDZC_NOTIF, sqe, sockfd, buf, (__u32) len, 0);
+	sqe->msg_flags = (__u32) flags;
+	sqe->notification_idx = slot_idx;
+	sqe->ioprio = zc_flags;
+}
+
+static inline void io_uring_prep_sendzc_fixed(struct io_uring_sqe *sqe, int sockfd,
+					      const void *buf, size_t len,
+					      int flags, unsigned slot_idx,
+					      unsigned zc_flags, unsigned buf_idx)
+{
+	io_uring_prep_sendzc(sqe, sockfd, buf, len, flags, slot_idx, zc_flags);
+	sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+	sqe->buf_index = buf_idx;
+}
+
+static inline void io_uring_prep_sendzc_set_addr(struct io_uring_sqe *sqe,
+						 const struct sockaddr *dest_addr,
+						 __u16 addr_len)
+{
+	sqe->addr2 = (unsigned long)(void *)dest_addr;
+	sqe->addr_len = addr_len;
+}
+
+static inline void io_uring_prep_notif_update(struct io_uring_sqe *sqe,
+					      __u64 new_tag, /* 0 to ignore */
+					      unsigned offset, unsigned nr)
+{
+	io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, 0, nr,
+			 (__u64)offset);
+	sqe->addr = new_tag;
+	sqe->ioprio = IORING_RSRC_UPDATE_NOTIF;
+}
+
 static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
 				      void *buf, size_t len, int flags)
 {
src/liburing.map +2 -0
index 318d3d7..7d8f143 100644
@@ -60,4 +60,6 @@ LIBURING_2.3 {
 	global:
 		io_uring_register_sync_cancel;
 		io_uring_register_file_alloc_range;
+		io_uring_register_notifications;
+		io_uring_unregister_notifications;
 } LIBURING_2.2;
src/register.c +20 -0
index 2b37e5f..7482112 100644
@@ -364,3 +364,23 @@ int io_uring_register_file_alloc_range(struct io_uring *ring,
 				       IORING_REGISTER_FILE_ALLOC_RANGE, &range,
 				       0);
 }
+
+int io_uring_register_notifications(struct io_uring *ring, unsigned nr,
+				    struct io_uring_notification_slot *slots)
+{
+	struct io_uring_notification_register r = {
+		.nr_slots = nr,
+		.data = (unsigned long)slots,
+	};
+
+	return __sys_io_uring_register(ring->ring_fd,
+				       IORING_REGISTER_NOTIFIERS,
+				       &r, sizeof(r));
+}
+
+int io_uring_unregister_notifications(struct io_uring *ring)
+{
+	return __sys_io_uring_register(ring->ring_fd,
+				       IORING_UNREGISTER_NOTIFIERS,
+				       NULL, 0);
+}
Signature · 1 line
2.37.0
Reviewed-by on v2 2/5
#3 lore raw

Re: [PATCH liburing v2 2/5] liburing: add zc send and notif helpers

Quoted 9 lines · Pavel Begunkov

Add helpers for notification registration and preparing zerocopy send requests.

---
  src/include/liburing.h | 42 ++++++++++++++++++++++++++++++++++++++++++
  src/liburing.map       |  2 ++
  src/register.c         | 20 ++++++++++++++++++++
  3 files changed, 64 insertions(+)

[...]

src/include/liburing.h static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd, · line 720

Quoted 9 lines
src/include/liburing.h +9 -0
+static inline void io_uring_prep_notif_update(struct io_uring_sqe *sqe,
+					      __u64 new_tag, /* 0 to ignore */
+					      unsigned offset, unsigned nr)
+{
+	io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, 0, nr,
+			 (__u64)offset);
+	sqe->addr = new_tag;
+	sqe->ioprio = IORING_RSRC_UPDATE_NOTIF;
+}

The same nit on there. But overall it looks good.

Signature · 1 line
Ammar Faizi
#4 lore raw

PATCH v2 3/5 tests: add tests for zerocopy send and notifications

In reply to Pavel Begunkov
---
 test/Makefile        |   1 +
 test/send-zerocopy.c | 888 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 889 insertions(+)
 create mode 100644 test/send-zerocopy.c
test/Makefile +1 -0
index 8945368..a36ddb3 100644
@@ -175,6 +175,7 @@ test_srcs := \
 	xattr.c \
 	skip-cqe.c \
 	single-issuer.c \
+	send-zerocopy.c \
 	# EOL
 
 all_targets :=
test/send-zerocopy.c new file +888 -0

888 lines not on this page · raw message

Signature · 1 line
2.37.0
#5 lore raw

PATCH v2 4/5 examples: add a zerocopy send example

In reply to Pavel Begunkov
---
 examples/Makefile        |   3 +-
 examples/send-zerocopy.c | 366 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 368 insertions(+), 1 deletion(-)
 create mode 100644 examples/send-zerocopy.c
examples/Makefile +2 -1
index 8e7067f..1997a31 100644
@@ -14,7 +14,8 @@ example_srcs := \
 	io_uring-cp.c \
 	io_uring-test.c \
 	link-cp.c \
-	poll-bench.c
+	poll-bench.c \
+	send-zerocopy.c
 
 all_targets :=
 
examples/send-zerocopy.c new file +366 -0

366 lines not on this page · raw message

Signature · 1 line
2.37.0
#6 lore raw

PATCH v2 5/5 liburing: improve fallocate typecasting

In reply to Pavel Begunkov

Don't double cast int -> ptr -> int in io_uring_prep_fallocate(), assign len directly.

---
 src/include/liburing.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
src/include/liburing.h +2 -3
index 20cd308..cffdabd 100644
@@ -601,10 +601,9 @@ static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe,
 static inline void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd,
 					   int mode, off_t offset, off_t len)
 {
-
 	io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd,
-			(const uintptr_t *) (unsigned long) len,
-			(unsigned int) mode, (__u64) offset);
+			0, (unsigned int) mode, (__u64) offset);
+	sqe->addr = (__u64) len;
 }
 
 static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd,
Signature · 1 line
2.37.0
Reviewed-by on v2 5/5
#7 lore raw

Re: [PATCH liburing v2 5/5] liburing: improve fallocate typecasting

Quoted 7 lines · Pavel Begunkov

Don't double cast int -> ptr -> int in io_uring_prep_fallocate(), assign len directly.

---
  src/include/liburing.h | 5 ++---
  1 file changed, 2 insertions(+), 3 deletions(-)

[...]

src/include/liburing.h static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe, · line 605

Quoted 4 lines

io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd,

src/include/liburing.h +1 -2
-			(const uintptr_t *) (unsigned long) len,
-			(unsigned int) mode, (__u64) offset);
+			0, (unsigned int) mode, (__u64) offset);

nit: instead of 0 it's better to use NULL. No?

src/include/liburing.h static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe, · line 606

Quoted 1 line
src/include/liburing.h +1 -0
+	sqe->addr = (__u64) len;

This one looks simpler than a double cast.

Signature · 1 line
Ammar Faizi
#8 lore raw
In reply to Pavel Begunkov
Quoted 9 lines · Pavel Begunkov

Add zerocopy send headers, helpers and tests

v2:
use T_EXIT_*
fix ptr <-> int conversions for 32 bits arches slight renaming
get rid of error() in the test
add patch 5/5

Quoted 6 lines

Pavel Begunkov (5):
io_uring.h: sync with kernel for zc send and notifiers
liburing: add zc send and notif helpers
tests: add tests for zerocopy send and notifications
examples: add a zerocopy send example
liburing: improve fallocate typecasting

I can confirm it compiles fine on all architectures now. Nice.

Signature · 1 line
Ammar Faizi
maintainer Applied
#9 lore raw
In reply to Pavel Begunkov
Quoted 10 lines · Pavel Begunkov

Add zerocopy send headers, helpers and tests

v2:
use T_EXIT_*
fix ptr <-> int conversions for 32 bits arches slight renaming
get rid of error() in the test
add patch 5/5

[...]

Applied, thanks!

[1/5] io_uring.h: sync with kernel for zc send and notifiers
(no commit info)
[2/5] liburing: add zc send and notif helpers
(no commit info)
[3/5] tests: add tests for zerocopy send and notifications
(no commit info)
[4/5] examples: add a zerocopy send example
(no commit info)
[5/5] liburing: improve fallocate typecasting
(no commit info)

Best regards,

Signature · 1 line
Jens Axboe