leverage completion cache for poll requests
cover leverage completion cache for poll requests
1st is a non-related code clean
2nd ~ 4th are fixes.
5th is the the main logic, I tested it with an echo-server, both in single-shot mode and multishot mode poll, not much difference compared to code before with regard to req/s. But that may be resulted from my poor network knowledge, feel free to test it and leave comments.
Hao Xu (5):
io_uring: return boolean value for io_alloc_async_data
io_uring: code clean for io_poll_complete()
io_uring: fix race between poll completion and cancel_hash insertion
io_uring: fix lacking of EPOLLONESHOT
io_uring: leverage completion cache for poll requests
fs/io_uring.c | 70 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 17 deletions(-)
Signature · 1 line
2.24.4
PATCH 1/5 io_uring: return boolean value for io_alloc_async_data
boolean value is good enough for io_alloc_async_data.
--- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/io_uring.c +1 -1
@@ -3309,7 +3309,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, } } -static inline int io_alloc_async_data (struct io_kiocb *req) +static inline bool io_alloc_async_data (struct io_kiocb *req) {WARN_ON_ONCE (!io_op_defs[req->opcode].async_size); req->async_data =kmalloc (io_op_defs[req->opcode].async_size,GFP_KERNEL );
Signature · 1 line
2.24.4
PATCH 2/5 io_uring: code clean for io_poll_complete()
In reply to Hao XuNo need to return boolean value, since no callers consume it.
--- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/io_uring.c +2 -2
@@ -5350,14 +5350,14 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask)return !(flags &IORING_CQE_F_MORE ); } -static inline bool io_poll_complete (struct io_kiocb *req,__poll_t mask) +static inline void io_poll_complete (struct io_kiocb *req,__poll_t mask) __must_hold(&req->ctx->completion_lock) {bool done; done =__io_poll_complete (req, mask);io_commit_cqring (req->ctx); -return done; +return ; }static void io_poll_task_func (struct io_kiocb *req,bool *locked)
Signature · 1 line
2.24.4
PATCH 3/5 io_uring: fix race between poll completion and cancel_hash insertion
In reply to Hao XuIf poll arming and poll completion runs parallelly, there maybe races. For instance, run io_poll_add in iowq and io_poll_task_func in original context, then:
iowq original context
io_poll_add
vfs_poll
(interruption happens
tw queued to original
context) io_poll_task_func
generate cqe
del from cancel_hash[]
if !poll.done
insert to cancel_hash[]The entry left in cancel_hash[], similar case for fast poll. Fix it by set poll.done = true when del from cancel_hash[].
--- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
fs/io_uring.c +3 -3
@@ -5340,10 +5340,8 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { - req->poll.done = true; +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) flags =0 ; - }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;
@@ -5374,6 +5372,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (done) {io_poll_remove_double (req);hash_del (&req->hash_node); + req->poll.done = true; }else { req->result =0 ;add_wait_queue (req->poll.head, &req->poll.wait);
@@ -5511,6 +5510,7 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)hash_del (&req->hash_node);io_poll_remove_double (req); + apoll->poll.done = true;spin_unlock (&ctx->completion_lock);if (!READ_ONCE (apoll->poll.canceled))
Signature · 1 line
2.24.4
PATCH 4/5 io_uring: fix lacking of EPOLLONESHOT
In reply to Hao XuWe should set EPOLLONESHOT if cqring_fill_event() returns false since io_poll_add() decides to put req or not by it.
--- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
fs/io_uring.c +3 -1
@@ -5340,8 +5340,10 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { + req->poll.events |=EPOLLONESHOT ; flags =0 ; + }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;
Signature · 1 line
2.24.4
PATCH 5/5 io_uring: leverage completion cache for poll requests
In reply to Hao XuLeverage completion cache to handle completions of poll requests in a batch.
Good thing is we save compl_nr - 1 completion_lock and io_cqring_ev_posted.
Bad thing is compl_nr extra ifs in flush_completion.
--- fs/io_uring.c | 64 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 15 deletions(-)
fs/io_uring.c +49 -15
@@ -1099,6 +1099,8 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, unsigned int issue_flags, u32 slot_index);static enum hrtimer_restart io_link_timeout_fn (struct hrtimer *timer); +static bool io_complete_poll (struct io_kiocb *req); +static struct kmem_cache *req_cachep;static const struct file_operations io_uring_fops;
@@ -2333,6 +2335,11 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)for (i =0 ; i < nr; i++) {struct io_kiocb *req = state->compl_reqs[i]; +if (req->opcode ==IORING_OP_POLL_ADD ) { +if (!io_complete_poll (req)) + state->compl_reqs[i] =NULL ; +continue ; + }__io_cqring_fill_event (ctx, req->user_data, req->result, req->compl.cflags); }
@@ -2344,7 +2351,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)for (i =0 ; i < nr; i++) {struct io_kiocb *req = state->compl_reqs[i]; -if (req_ref_put_and_test (req)) +if (req &&req_ref_put_and_test (req))io_req_free_batch (&rb, req, &ctx->submit_state); }
@@ -5360,6 +5367,23 @@ static inline void io_poll_complete(struct io_kiocb *req, __poll_t mask)return ; } +static bool io_complete_poll (struct io_kiocb *req) +{ +bool done; + + done =__io_poll_complete (req, req->result); +if (done) { +io_poll_remove_double (req); +hash_del (&req->hash_node); + req->poll.done = true; + }else { + req->result =0 ; +add_wait_queue (req->poll.head, &req->poll.wait); + } + +return done; +} +static void io_poll_task_func (struct io_kiocb *req,bool *locked) {struct io_ring_ctx *ctx = req->ctx;
@@ -5367,18 +5391,10 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (io_poll_rewait (req, &req->poll)) {spin_unlock (&ctx->completion_lock); - }else { + }else if (!*locked) {bool done; - done =__io_poll_complete (req, req->result); -if (done) { -io_poll_remove_double (req); -hash_del (&req->hash_node); - req->poll.done = true; - }else { - req->result =0 ; -add_wait_queue (req->poll.head, &req->poll.wait); - } + done =io_complete_poll (req);io_commit_cqring (ctx);spin_unlock (&ctx->completion_lock);io_cqring_ev_posted (ctx);
@@ -5388,6 +5404,13 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (nxt)io_req_task_submit (nxt, locked); } + }else { +struct io_submit_state *state = &ctx->submit_state; + +spin_unlock (&ctx->completion_lock); + state->compl_reqs[state->compl_nr++] = req; +if (state->compl_nr ==ARRAY_SIZE (state->compl_reqs)) +io_submit_flush_completions (ctx); } }
@@ -5833,6 +5856,7 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)struct io_ring_ctx *ctx = req->ctx;struct io_poll_table ipt;__poll_t mask; +bool locked = current == req->task; ipt.pt._qproc = io_poll_queue_proc;
@@ -5841,14 +5865,24 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)if (mask) {/* no async, we'd stolen it */ ipt.error =0 ; -io_poll_complete (req, mask); +if (!locked) +io_poll_complete (req, mask); }spin_unlock (&ctx->completion_lock);if (mask) { -io_cqring_ev_posted (ctx); -if (poll->events &EPOLLONESHOT ) -io_put_req (req); +if (!locked) { +io_cqring_ev_posted (ctx); +if (poll->events &EPOLLONESHOT ) +io_put_req (req); + }else { +struct io_submit_state *state = &ctx->submit_state; + + req->result = mask; + state->compl_reqs[state->compl_nr++] = req; +if (state->compl_nr ==ARRAY_SIZE (state->compl_reqs)) +io_submit_flush_completions (ctx); + } }return ipt.error; }
Signature · 1 line
2.24.4
Re: [PATCH 5/5] io_uring: leverage completion cache for poll requests
Quoted 7 lines · Hao Xu
Leverage completion cache to handle completions of poll requests in a batch.
Good thing is we savecompl_nr- 1completion_lockandio_cqring_ev_posted.
Bad thing iscompl_nrextra ifs inflush_completion.Signed-off-by Hao Xu <[email protected]>
Sorry, please ignore this one, didn't notice it failed to pass poll-mshot-update test
Quoted 135 lines
--- fs/io_uring.c | 64 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 15 deletions(-)fs/io_uring.c +49
-15@@ -1099,6 +1099,8 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, unsigned int issue_flags, u32 slot_index);static enum hrtimer_restart io_link_timeout_fn (struct hrtimer *timer); +static bool io_complete_poll (struct io_kiocb *req); +static struct kmem_cache *req_cachep;static const struct file_operations io_uring_fops;@@ -2333,6 +2335,11 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)for (i =0 ; i < nr; i++) {struct io_kiocb *req = state->compl_reqs[i]; +if (req->opcode ==IORING_OP_POLL_ADD ) { +if (!io_complete_poll (req)) + state->compl_reqs[i] =NULL ; +continue ; + }__io_cqring_fill_event (ctx, req->user_data, req->result, req->compl.cflags); }@@ -2344,7 +2351,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)for (i =0 ; i < nr; i++) {struct io_kiocb *req = state->compl_reqs[i]; -if (req_ref_put_and_test (req)) +if (req &&req_ref_put_and_test (req))io_req_free_batch (&rb, req, &ctx->submit_state); }@@ -5360,6 +5367,23 @@ static inline void io_poll_complete(struct io_kiocb *req, __poll_t mask)return ; } +static bool io_complete_poll (struct io_kiocb *req) +{ +bool done; + + done =__io_poll_complete (req, req->result); +if (done) { +io_poll_remove_double (req); +hash_del (&req->hash_node); + req->poll.done = true; + }else { + req->result =0 ; +add_wait_queue (req->poll.head, &req->poll.wait); + } + +return done; +} +static void io_poll_task_func (struct io_kiocb *req,bool *locked) {struct io_ring_ctx *ctx = req->ctx;@@ -5367,18 +5391,10 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (io_poll_rewait (req, &req->poll)) {spin_unlock (&ctx->completion_lock); - }else { + }else if (!*locked) {bool done; - done =__io_poll_complete (req, req->result); -if (done) { -io_poll_remove_double (req); -hash_del (&req->hash_node); - req->poll.done = true; - }else { - req->result =0 ; -add_wait_queue (req->poll.head, &req->poll.wait); - } + done =io_complete_poll (req);io_commit_cqring (ctx);spin_unlock (&ctx->completion_lock);io_cqring_ev_posted (ctx);@@ -5388,6 +5404,13 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (nxt)io_req_task_submit (nxt, locked); } + }else { +struct io_submit_state *state = &ctx->submit_state; + +spin_unlock (&ctx->completion_lock); + state->compl_reqs[state->compl_nr++] = req; +if (state->compl_nr ==ARRAY_SIZE (state->compl_reqs)) +io_submit_flush_completions (ctx); } }@@ -5833,6 +5856,7 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)struct io_ring_ctx *ctx = req->ctx;struct io_poll_table ipt;__poll_t mask; +bool locked = current == req->task; ipt.pt._qproc = io_poll_queue_proc;@@ -5841,14 +5865,24 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)if (mask) {/* no async, we'd stolen it */ ipt.error =0 ; -io_poll_complete (req, mask); +if (!locked) +io_poll_complete (req, mask); }spin_unlock (&ctx->completion_lock);if (mask) { -io_cqring_ev_posted (ctx); -if (poll->events &EPOLLONESHOT ) -io_put_req (req); +if (!locked) { +io_cqring_ev_posted (ctx); +if (poll->events &EPOLLONESHOT ) +io_put_req (req); + }else { +struct io_submit_state *state = &ctx->submit_state; + + req->result = mask; + state->compl_reqs[state->compl_nr++] = req; +if (state->compl_nr ==ARRAY_SIZE (state->compl_reqs)) +io_submit_flush_completions (ctx); + } }return ipt.error; }
Re: [PATCH 5/5] io_uring: leverage completion cache for poll requests
In reply to Hao XuQuoted 5 lines · Hao Xu
Leverage completion cache to handle completions of poll requests in a batch.
Good thing is we savecompl_nr- 1completion_lockandio_cqring_ev_posted.
Bad thing iscompl_nrextra ifs inflush_completion.
It does something really wrong breaking all abstractions, we can't go with this. Can we have one of those below?
- __io_req_complete(issue_flags), forwarding issue_flags from above
- or maybe io_req_task_complete(), if we're in tw
In any case, I'd recommend sending it separately from fixes.
Signature · 1 line
Pavel Begunkov
Re: [PATCH 5/5] io_uring: leverage completion cache for poll requests
Quoted 11 lines · Pavel Begunkov
Quoted 5 lines · Hao Xu
Leverage completion cache to handle completions of poll requests in a batch.
Good thing is we savecompl_nr- 1completion_lockandio_cqring_ev_posted.
Bad thing iscompl_nrextra ifs inflush_completion.It does something really wrong breaking all abstractions, we can't go with this. Can we have one of those below?
-__io_req_complete(issue_flags), forwardingissue_flagsfrom above
- or maybeio_req_task_complete(), if we're in tw
Make sense. we may need to remove io_clean_op logic in io_req_complete_state(), multishot reqs shouldn't do it, and it's ok for normal reqs since we do it later in __io_submit_flush_completions-> io_req_free_batch->io_dismantle_req->io_clean_op, correct me if I'm wrong.
I'll send another version later.
Quoted 3 lines
In any case, I'd recommend sending it separately from fixes.
Re: [PATCH 5/5] io_uring: leverage completion cache for poll requests
Quoted 12 lines · Hao Xu
Quoted 11 lines · Pavel Begunkov
Quoted 5 lines · Hao Xu
Leverage completion cache to handle completions of poll requests in a batch.
Good thing is we savecompl_nr- 1completion_lockandio_cqring_ev_posted.
Bad thing iscompl_nrextra ifs inflush_completion.It does something really wrong breaking all abstractions, we can't go with this. Can we have one of those below?
-__io_req_complete(issue_flags), forwardingissue_flagsfrom above
- or maybeio_req_task_complete(), if we're in tw
Quoted 6 lines
Make sense. we may need to remove
io_clean_oplogic in
io_req_complete_state(), multishot reqs shouldn't do it, and it's ok for normal reqs since we do it later in __io_submit_flush_completions->io_req_free_batch->io_dismantle_req->io_clean_op, correct me if I'm wrong.
req->compl.cflags is in the first 64B, i.e. aliased with req->rw and others. We need to clean everything left in there before using the space, that's what io_clean_op() there is for
Signature · 1 line
Pavel Begunkov
Re: [PATCH 5/5] io_uring: leverage completion cache for poll requests
Quoted 15 lines · Pavel Begunkov
Quoted 13 lines · Hao Xu
Quoted 11 lines · Pavel Begunkov
Quoted 5 lines · Hao Xu
Leverage completion cache to handle completions of poll requests in a batch.
Good thing is we savecompl_nr- 1completion_lockandio_cqring_ev_posted.
Bad thing iscompl_nrextra ifs inflush_completion.It does something really wrong breaking all abstractions, we can't go with this. Can we have one of those below?
-__io_req_complete(issue_flags), forwardingissue_flagsfrom above
- or maybeio_req_task_complete(), if we're in twMake sense. we may need to remove
io_clean_oplogic in
Quoted 8 lines
Quoted 4 lines
io_req_complete_state(), multishot reqs shouldn't do it, and it's ok for normal reqs since we do it later in __io_submit_flush_completions->io_req_free_batch->io_dismantle_req->io_clean_op, correct me if I'm wrong.
req->compl.cflagsis in the first 64B, i.e. aliased withreq->rwand others. We need to clean everything left in there before using the space, that's whatio_clean_op()there is for
True, and that's why we souldn't do it for multishot reqs, since they are not completed yet, and we won't reuse the req resource until its final completion.
Quoted 1 line
Re: [PATCH 5/5] io_uring: leverage completion cache for poll requests
Quoted 21 lines · Hao Xu
Quoted 20 lines · Pavel Begunkov
Quoted 18 lines · Hao Xu
Quoted 11 lines · Pavel Begunkov
Quoted 5 lines · Hao Xu
Leverage completion cache to handle completions of poll requests in a batch.
Good thing is we savecompl_nr- 1completion_lockandio_cqring_ev_posted.
Bad thing iscompl_nrextra ifs inflush_completion.It does something really wrong breaking all abstractions, we can't go with this. Can we have one of those below?
-__io_req_complete(issue_flags), forwardingissue_flagsfrom above
- or maybeio_req_task_complete(), if we're in twMake sense. we may need to remove
io_clean_oplogic in
io_req_complete_state(), multishot reqs shouldn't do it, and it's ok for normal reqs since we do it later in __io_submit_flush_completions->io_req_free_batch->io_dismantle_req->io_clean_op, correct me if I'm wrong.
Quoted 6 lines
Quoted 3 lines
req->compl.cflagsis in the first 64B, i.e. aliased withreq->rwand others. We need to clean everything left in there before using the space, that's whatio_clean_op()there is forTrue, and that's why we souldn't do it for multishot reqs, since they are not completed yet, and we won't reuse the req resource until its final completion.
I see what you are saying now..yes, we shouldn't remove that io_clean_op there.
Quoted 1 line
Quoted 1 line
Quoted 2 lines · Hao Xu
1st is a non-related code clean
2nd ~ 4th are fixes.
Looks only 3 and 4 are fixes. It's usually a good idea to send fixes separately or place them first in the series, so can be picked for-current / for-next. However, 1-2 are very simple, maybe easier to leave as is.
Quoted 15 lines
5th is the the main logic, I tested it with an echo-server, both in single-shot mode and multishot mode poll, not much difference compared to code before with regard to req/s. But that may be resulted from my poor network knowledge, feel free to test it and leave comments.
Hao Xu (5):
io_uring: return boolean value forio_alloc_async_data
io_uring: code clean forio_poll_complete()
io_uring: fix race between poll completion andcancel_hashinsertion
io_uring: fix lacking of EPOLLONESHOT
io_uring: leverage completion cache for poll requestsfs/io_uring.c | 70 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 17 deletions(-)
Signature · 1 line
Pavel Begunkov