io_uring: implement multishot mode for accept
PATCH 4/4 io_uring: implement multishot mode for accept
From: Hao Xu <[email protected]>
Refactor io_accept() to support multishot mode.
theoretical analysis:
1) when connections come in fast
- singleshot:
add accept sqe(userpsace) --> accept inline
^ |
|-----------------|
- multishot:
add accept sqe(userspace) --> accept inline
^ |
|--*--|
we do accept repeatedly in * place until get EAGAIN 2) when connections come in at a low pressure
similar thing like 1), we reduce a lot of userspace-kernel context switch and useless vfs_poll()
tests:
Did some tests, which goes in this way:
server client(multiple) accept connect read write write read close close
Basically, raise up a number of clients(on same machine with server) to connect to the server, and then write some data to it, the server will write those data back to the client after it receives them, and then close the connection after write return. Then the client will read the data and then close the connection. Here I test 10000 clients connect one server, data size 128 bytes. And each client has a go routine for it, so they come to the server in short time.
test 20 times before/after this patchset, time spent:(unit cycle, which is the return value of clock())
before:
1930136+1940725+1907981+1947601+1923812+1928226+1911087+1905897+1941075 +1934374+1906614+1912504+1949110+1908790+1909951+1941672+1969525+1934984 +1934226+1914385)/20.0 = 1927633.75
after:
1858905+1917104+1895455+1963963+1892706+1889208+1874175+1904753+1874112 +1874985+1882706+1884642+1864694+1906508+1916150+1924250+1869060+1889506 +1871324+1940803)/20.0 = 1894750.45
(1927633.75 - 1894750.45) / 1927633.75 = 1.65%
--- fs/io_uring.c | 42 ++++++++++++++++++++++++++++++----- include/uapi/linux/io_uring.h | 4 ++-- 2 files changed, 39 insertions(+), 7 deletions(-)
fs/io_uring.c +37 -5
@@ -1146,6 +1146,7 @@ static const struct io_op_def io_op_defs[] = { .unbound_nonreg_file = 1, .pollin = 1, .poll_exclusive = 1, + .ioprio = 1, /* used for flags */ }, [IORING_OP_ASYNC_CANCEL] = { .audit_skip = 1,
@@ -5706,6 +5707,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags)static int io_accept_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) {struct io_accept *accept = &req->accept; +unsigned flags;if (sqe->len || sqe->buf_index)return -EINVAL ;
@@ -5714,19 +5716,26 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) accept->addr_len =u64_to_user_ptr (READ_ONCE (sqe->addr2)); accept->flags =READ_ONCE (sqe->accept_flags); accept->nofile =rlimit (RLIMIT_NOFILE ); + flags =READ_ONCE (sqe->ioprio); +if (flags & ~IORING_ACCEPT_MULTISHOT ) +return -EINVAL ; accept->file_slot =READ_ONCE (sqe->file_index); -if (accept->file_slot && (accept->flags &SOCK_CLOEXEC )) +if (accept->file_slot && ((accept->flags &SOCK_CLOEXEC ) || + flags &IORING_ACCEPT_MULTISHOT ))return -EINVAL ;if (accept->flags & ~(SOCK_CLOEXEC |SOCK_NONBLOCK ))return -EINVAL ;if (SOCK_NONBLOCK !=O_NONBLOCK && (accept->flags &SOCK_NONBLOCK )) accept->flags = (accept->flags & ~SOCK_NONBLOCK ) |O_NONBLOCK ; +if (flags &IORING_ACCEPT_MULTISHOT ) + req->flags |=REQ_F_APOLL_MULTISHOT ;return 0 ; }static int io_accept (struct io_kiocb *req,unsigned int issue_flags) { +struct io_ring_ctx *ctx = req->ctx;struct io_accept *accept = &req->accept;bool force_nonblock = issue_flags &IO_URING_F_NONBLOCK ;unsigned int file_flags = force_nonblock ?O_NONBLOCK :0 ;
@@ -5734,6 +5743,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags)struct file *file;int ret, fd; +retry:if (!fixed) { fd =__get_unused_fd_flags (accept->flags, accept->nofile);if (unlikely (fd <0 ))
@@ -5745,8 +5755,12 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags)if (!fixed)put_unused_fd (fd); ret =PTR_ERR (file); -if (ret == -EAGAIN && force_nonblock) -return -EAGAIN ; +if (ret == -EAGAIN && force_nonblock) { +if ((req->flags &IO_APOLL_MULTI_POLLED ) == +IO_APOLL_MULTI_POLLED ) + ret =0 ; +return ret; + }if (ret == -ERESTARTSYS ) ret = -EINTR ;req_set_fail (req);
@@ -5757,8 +5771,26 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) ret =io_install_fixed_file (req, file, issue_flags, accept->file_slot -1 ); } -__io_req_complete (req, issue_flags, ret,0 ); -return 0 ; + +if (!(req->flags &REQ_F_APOLL_MULTISHOT )) { +__io_req_complete (req, issue_flags, ret,0 ); +return 0 ; + } +if (ret >=0 ) { +bool filled; + +spin_lock (&ctx->completion_lock); + filled =io_fill_cqe_aux (ctx, req->cqe.user_data, ret, +IORING_CQE_F_MORE ); +io_commit_cqring (ctx); +spin_unlock (&ctx->completion_lock); +if (!filled) +return -ECANCELED ; +io_cqring_ev_posted (ctx); + goto retry; + } + +return ret; }static int io_connect_prep_async (struct io_kiocb *req)
include/uapi/linux/io_uring.h +2 -2
@@ -224,9 +224,9 @@ enum {#define IORING_RECVSEND_POLL_FIRST (1U << 0)/* -* accept flags stored in accept_flags +* accept flags stored in sqe->ioprio */ -#define IORING_ACCEPT_MULTISHOT (1U << 15) +#define IORING_ACCEPT_MULTISHOT (1U << 0)/* * IO completion data structure (Completion Queue Entry)
Signature · 1 line
2.36.0
Jens Axboe
maintainerQuoted 16 lines · Hao Xu
fs/io_uring.c +10
-2@@ -5757,8 +5771,26 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) ret =io_install_fixed_file (req, file, issue_flags, accept->file_slot -1 ); } -__io_req_complete (req, issue_flags, ret,0 ); -return 0 ; + +if (!(req->flags &REQ_F_APOLL_MULTISHOT )) { +__io_req_complete (req, issue_flags, ret,0 ); +return 0 ; + } +if (ret >=0 ) { +bool filled; + +spin_lock (&ctx->completion_lock); + filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
fs/io_uring.c static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · line 5793
Quoted 10 lines
fs/io_uring.c +10
-0+ IORING_CQE_F_MORE); +io_commit_cqring (ctx); +spin_unlock (&ctx->completion_lock); +if (!filled) +return -ECANCELED ; +io_cqring_ev_posted (ctx); + goto retry; + } + +return ret;
I'd still make that:
if (filled) {
io_cqring_ev_posted(ctx);
goto retry;
}
ret = -ECANCELED;as it flows better and shows what the likely outcome is.
Quoted 7 lines
fs/io_uring.c +0
-0static int io_connect_prep_async (struct io_kiocb *req)include/uapi/linux/io_uring.h +0
-0@@ -224,9 +224,9 @@ enum {#define IORING_RECVSEND_POLL_FIRST (1U << 0)
include/uapi/linux/io_uring.h enum { · line 232
Quoted 10 lines
include/uapi/linux/io_uring.h +2
-2/* -* accept flags stored in accept_flags +* accept flags stored in sqe->ioprio */ -#define IORING_ACCEPT_MULTISHOT (1U << 15) +#define IORING_ACCEPT_MULTISHOT (1U << 0)/* * IO completion data structure (Completion Queue Entry)
This hunk needs to get folded in to the patch that adds the flag in the first place.
Signature · 1 line
Jens Axboe