Skip to contentSkip to conversation

io_uring: implement multishot mode for accept

PATCH In review Hao Xu 2 participants 2 messages May 7, 2022

Subsystems (2)
lore
Share & tools
lore

Hao Xu

author
#0 lore raw

PATCH 4/4 io_uring: implement multishot mode for accept

From: Hao Xu <[email protected]>

Refactor io_accept() to support multishot mode.

theoretical analysis:
1) when connections come in fast

    - singleshot:
              add accept sqe(userpsace) --> accept inline
                              ^                 |
                              |-----------------|
    - multishot:
             add accept sqe(userspace) --> accept inline
                                              ^     |
                                              |--*--|

    we do accept repeatedly in * place until get EAGAIN

2) when connections come in at a low pressure
similar thing like 1), we reduce a lot of userspace-kernel context switch and useless vfs_poll()

tests:
Did some tests, which goes in this way:

  server    client(multiple)
  accept    connect
  read      write
  write     read
  close     close

Basically, raise up a number of clients(on same machine with server) to connect to the server, and then write some data to it, the server will write those data back to the client after it receives them, and then close the connection after write return. Then the client will read the data and then close the connection. Here I test 10000 clients connect one server, data size 128 bytes. And each client has a go routine for it, so they come to the server in short time.
test 20 times before/after this patchset, time spent:(unit cycle, which is the return value of clock())
before:
1930136+1940725+1907981+1947601+1923812+1928226+1911087+1905897+1941075 +1934374+1906614+1912504+1949110+1908790+1909951+1941672+1969525+1934984 +1934226+1914385)/20.0 = 1927633.75
after:
1858905+1917104+1895455+1963963+1892706+1889208+1874175+1904753+1874112 +1874985+1882706+1884642+1864694+1906508+1916150+1924250+1869060+1889506 +1871324+1940803)/20.0 = 1894750.45

(1927633.75 - 1894750.45) / 1927633.75 = 1.65%

[fold in setting io_op_def->ioprio for accept and other cleaning up]
---
 fs/io_uring.c                 | 42 ++++++++++++++++++++++++++++++-----
 include/uapi/linux/io_uring.h |  4 ++--
 2 files changed, 39 insertions(+), 7 deletions(-)
fs/io_uring.c +37 -5
index e0d12af04cd1..f21172913336 100644
@@ -1146,6 +1146,7 @@ static const struct io_op_def io_op_defs[] = {
 		.unbound_nonreg_file	= 1,
 		.pollin			= 1,
 		.poll_exclusive		= 1,
+		.ioprio			= 1,	/* used for flags */
 	},
 	[IORING_OP_ASYNC_CANCEL] = {
 		.audit_skip		= 1,
@@ -5706,6 +5707,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_accept *accept = &req->accept;
+	unsigned flags;
 
 	if (sqe->len || sqe->buf_index)
 		return -EINVAL;
@@ -5714,19 +5716,26 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
 	accept->flags = READ_ONCE(sqe->accept_flags);
 	accept->nofile = rlimit(RLIMIT_NOFILE);
+	flags = READ_ONCE(sqe->ioprio);
+	if (flags & ~IORING_ACCEPT_MULTISHOT)
+		return -EINVAL;
 
 	accept->file_slot = READ_ONCE(sqe->file_index);
-	if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
+	if (accept->file_slot && ((accept->flags & SOCK_CLOEXEC) ||
+	   flags & IORING_ACCEPT_MULTISHOT))
 		return -EINVAL;
 	if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
 		return -EINVAL;
 	if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
 		accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
+	if (flags & IORING_ACCEPT_MULTISHOT)
+		req->flags |= REQ_F_APOLL_MULTISHOT;
 	return 0;
 }
 
 static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
 {
+	struct io_ring_ctx *ctx = req->ctx;
 	struct io_accept *accept = &req->accept;
 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
 	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
@@ -5734,6 +5743,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
 	struct file *file;
 	int ret, fd;
 
+retry:
 	if (!fixed) {
 		fd = __get_unused_fd_flags(accept->flags, accept->nofile);
 		if (unlikely(fd < 0))
@@ -5745,8 +5755,12 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
 		if (!fixed)
 			put_unused_fd(fd);
 		ret = PTR_ERR(file);
-		if (ret == -EAGAIN && force_nonblock)
-			return -EAGAIN;
+		if (ret == -EAGAIN && force_nonblock) {
+			if ((req->flags & IO_APOLL_MULTI_POLLED) ==
+			    IO_APOLL_MULTI_POLLED)
+				ret = 0;
+			return ret;
+		}
 		if (ret == -ERESTARTSYS)
 			ret = -EINTR;
 		req_set_fail(req);
@@ -5757,8 +5771,26 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
 		ret = io_install_fixed_file(req, file, issue_flags,
 					    accept->file_slot - 1);
 	}
-	__io_req_complete(req, issue_flags, ret, 0);
-	return 0;
+
+	if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
+		__io_req_complete(req, issue_flags, ret, 0);
+		return 0;
+	}
+	if (ret >= 0) {
+		bool filled;
+
+		spin_lock(&ctx->completion_lock);
+		filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
+					 IORING_CQE_F_MORE);
+		io_commit_cqring(ctx);
+		spin_unlock(&ctx->completion_lock);
+		if (!filled)
+			return -ECANCELED;
+		io_cqring_ev_posted(ctx);
+		goto retry;
+	}
+
+	return ret;
 }
 
 static int io_connect_prep_async(struct io_kiocb *req)
include/uapi/linux/io_uring.h +2 -2
index f4d9ca62a5a6..7c3d70d12428 100644
@@ -224,9 +224,9 @@ enum {
 #define IORING_RECVSEND_POLL_FIRST	(1U << 0)
 
 /*
- * accept flags stored in accept_flags
+ * accept flags stored in sqe->ioprio
  */
-#define IORING_ACCEPT_MULTISHOT	(1U << 15)
+#define IORING_ACCEPT_MULTISHOT	(1U << 0)
 
 /*
  * IO completion data structure (Completion Queue Entry)
Signature · 1 line
2.36.0
maintainer
#1 lore raw
Quoted 16 lines · Hao Xu
fs/io_uring.c +10 -2
@@ -5757,8 +5771,26 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
 		ret = io_install_fixed_file(req, file, issue_flags,
 					    accept->file_slot - 1);
 	}
-	__io_req_complete(req, issue_flags, ret, 0);
-	return 0;
+
+	if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
+		__io_req_complete(req, issue_flags, ret, 0);
+		return 0;
+	}
+	if (ret >= 0) {
+		bool filled;
+
+		spin_lock(&ctx->completion_lock);
+		filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,

fs/io_uring.c static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · line 5793

Quoted 10 lines
fs/io_uring.c +10 -0
+					 IORING_CQE_F_MORE);
+		io_commit_cqring(ctx);
+		spin_unlock(&ctx->completion_lock);
+		if (!filled)
+			return -ECANCELED;
+		io_cqring_ev_posted(ctx);
+		goto retry;
+	}
+
+	return ret;

I'd still make that:

	if (filled) {
		io_cqring_ev_posted(ctx);
		goto retry;
	}
	ret = -ECANCELED;

as it flows better and shows what the likely outcome is.

Quoted 7 lines
fs/io_uring.c +0 -0
 static int io_connect_prep_async(struct io_kiocb *req)
include/uapi/linux/io_uring.h +0 -0
index f4d9ca62a5a6..7c3d70d12428 100644
@@ -224,9 +224,9 @@ enum {
 #define IORING_RECVSEND_POLL_FIRST	(1U << 0)

include/uapi/linux/io_uring.h enum { · line 232

Quoted 10 lines
include/uapi/linux/io_uring.h +2 -2
 
 /*
- * accept flags stored in accept_flags
+ * accept flags stored in sqe->ioprio
  */
-#define IORING_ACCEPT_MULTISHOT	(1U << 15)
+#define IORING_ACCEPT_MULTISHOT	(1U << 0)
 
 /*
  * IO completion data structure (Completion Queue Entry)

This hunk needs to get folded in to the patch that adds the flag in the first place.

Signature · 1 line
Jens Axboe