cancel_hash per entry lock
cover cancel_hash per entry lock
From: Hao Xu <[email protected]>
Make per entry lock for cancel_hash array, this reduces usage of completion_lock and contension between cancel_hash entries.
Hao Xu (2):
io_uring: add an argument for io_poll_disarm()
io_uring: switch cancel_hash to use per list spinlock
io_uring/cancel.c | 12 ++++++++++-- io_uring/cancel.h | 1 + io_uring/io_uring.c | 9 +++++++++ io_uring/io_uring_types.h | 1 + io_uring/poll.c | 38 +++++++++++++++++++++----------------- 5 files changed, 42 insertions(+), 19 deletions(-)
base-commit: cbbf9a4865526c55a6a984ef69578def9944f319
Signature · 1 line
2.25.1
PATCH 1/2 io_uring: add an argument for io_poll_disarm()
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Add an argument for io_poll_disarm() for later use.
--- io_uring/poll.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
io_uring/poll.c +5 -3
@@ -561,8 +561,9 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only, {struct hlist_head *list;struct io_kiocb *req; +u32 index =hash_long (cd->data, ctx->cancel_hash_bits); - list = &ctx->cancel_hash[hash_long (cd->data, ctx->cancel_hash_bits)]; + list = &ctx->cancel_hash[index];hlist_for_each_entry (req, list, hash_node) {if (cd->data != req->cqe.user_data)continue ;
@@ -573,6 +574,7 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,continue ; req->work.cancel_seq = cd->seq; } + cd->flags = index;return req; }return NULL ;
@@ -602,7 +604,7 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,return NULL ; } -static bool io_poll_disarm (struct io_kiocb *req) +static bool io_poll_disarm (struct io_kiocb *req,u32 index) __must_hold(&ctx->completion_lock) {if (!io_poll_get_ownership (req))
@@ -724,7 +726,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)spin_lock (&ctx->completion_lock); preq =io_poll_find (ctx, true, &cd); -if (!preq || !io_poll_disarm (preq)) { +if (!preq || !io_poll_disarm (preq, cd.flags)) {spin_unlock (&ctx->completion_lock); ret = preq ? -EALREADY : -ENOENT ; goto out;
Signature · 1 line
2.25.1
PATCH 2/2 io_uring: switch cancel_hash to use per list spinlock
In reply to Hao XuFrom: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for cancel_hash, this removes some completion lock invocation and remove contension between different cancel_hash entries
--- io_uring/cancel.c | 12 ++++++++++-- io_uring/cancel.h | 1 + io_uring/io_uring.c | 9 +++++++++ io_uring/io_uring_types.h | 1 + io_uring/poll.c | 30 ++++++++++++++++-------------- 5 files changed, 37 insertions(+), 16 deletions(-)
io_uring/cancel.c +10 -2
@@ -93,14 +93,14 @@ int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd)if (!ret)return 0 ; -spin_lock (&ctx->completion_lock); ret =io_poll_cancel (ctx, cd);if (ret != -ENOENT ) goto out; +spin_lock (&ctx->completion_lock);if (!(cd->flags &IORING_ASYNC_CANCEL_FD )) ret =io_timeout_cancel (ctx, cd); -out:spin_unlock (&ctx->completion_lock); +out:return ret; }
@@ -192,3 +192,11 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)io_req_set_res (req, ret,0 );return IOU_OK ; } + +inline void init_cancel_hash_locks (spinlock_t *cancel_hash_locks,unsigned size) +{ +int i; + +for (i =0 ; i < size; i++) +spin_lock_init (&cancel_hash_locks[i]); +}
io_uring/cancel.h +1 -0
@@ -4,3 +4,4 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);int io_async_cancel (struct io_kiocb *req,unsigned int issue_flags);int io_try_cancel (struct io_kiocb *req,struct io_cancel_data *cd); +inline void init_cancel_hash_locks (spinlock_t *cancel_hash_locks,unsigned size);
io_uring/io_uring.c +9 -0
@@ -706,7 +706,14 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) GFP_KERNEL);if (!ctx->cancel_hash) goto err; + ctx->cancel_hash_locks = +kmalloc ((1U << hash_bits) *sizeof (spinlock_t), +GFP_KERNEL ); +if (!ctx->cancel_hash_locks) + goto err; +__hash_init (ctx->cancel_hash,1U << hash_bits); +init_cancel_hash_locks (ctx->cancel_hash_locks,1U << hash_bits); ctx->dummy_ubuf =kzalloc (sizeof (*ctx->dummy_ubuf),GFP_KERNEL );if (!ctx->dummy_ubuf)
@@ -749,6 +756,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) err:kfree (ctx->dummy_ubuf);kfree (ctx->cancel_hash); +kfree (ctx->cancel_hash_locks);kfree (ctx->io_bl);xa_destroy (&ctx->io_bl_xa);kfree (ctx);
@@ -3045,6 +3053,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)if (ctx->hash_map)io_wq_put_hash (ctx->hash_map);kfree (ctx->cancel_hash); +kfree (ctx->cancel_hash_locks);kfree (ctx->dummy_ubuf);kfree (ctx->io_bl);xa_destroy (&ctx->io_bl_xa);
io_uring/io_uring_types.h +1 -0
@@ -231,6 +231,7 @@ struct io_ring_ctx {*/ struct io_wq_work_list iopoll_list;struct hlist_head *cancel_hash; +spinlock_t *cancel_hash_locks;unsigned cancel_hash_bits;bool poll_multi_queue;
io_uring/poll.c +16 -14
@@ -73,10 +73,11 @@ static struct io_poll *io_poll_get_single(struct io_kiocb *req)static void io_poll_req_insert (struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx; -struct hlist_head *list; +u32 index =hash_long (req->cqe.user_data, ctx->cancel_hash_bits); - list = &ctx->cancel_hash[hash_long (req->cqe.user_data, ctx->cancel_hash_bits)]; -hlist_add_head (&req->hash_node, list); +spin_lock (&ctx->cancel_hash_locks[index]); +hlist_add_head (&req->hash_node, &ctx->cancel_hash[index]); +spin_unlock (&ctx->cancel_hash_locks[index]); } static void io_init_poll_iocb(struct io_poll *poll, __poll_t events,
@@ -439,9 +440,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req,return 0 ; } -spin_lock (&ctx->completion_lock);io_poll_req_insert (req); -spin_unlock (&ctx->completion_lock);if (mask && (poll->events &EPOLLET )) {/* can't multishot if failed, just queue the event we've got */
@@ -538,10 +537,10 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,bool found = false;int i; -spin_lock (&ctx->completion_lock);for (i =0 ; i < (1U << ctx->cancel_hash_bits); i++) {struct hlist_head *list; +spin_lock (&ctx->cancel_hash_locks[i]); list = &ctx->cancel_hash[i];hlist_for_each_entry_safe (req, tmp, list, hash_node) {if (io_match_task_safe (req, tsk, cancel_all)) {
@@ -550,19 +549,19 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, found = true; } } +spin_unlock (&ctx->cancel_hash_locks[i]); } -spin_unlock (&ctx->completion_lock);return found; }static struct io_kiocb *io_poll_find (struct io_ring_ctx *ctx,bool poll_only,struct io_cancel_data *cd) - __must_hold(&ctx->completion_lock) {struct hlist_head *list;struct io_kiocb *req;u32 index =hash_long (cd->data, ctx->cancel_hash_bits); +spin_lock (&ctx->cancel_hash_locks[index]); list = &ctx->cancel_hash[index];hlist_for_each_entry (req, list, hash_node) {if (cd->data != req->cqe.user_data)
@@ -574,15 +573,16 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,continue ; req->work.cancel_seq = cd->seq; } +spin_unlock (&ctx->cancel_hash_locks[index]); cd->flags = index;return req; } +spin_unlock (&ctx->cancel_hash_locks[index]);return NULL ; }static struct io_kiocb *io_poll_file_find (struct io_ring_ctx *ctx,struct io_cancel_data *cd) - __must_hold(&ctx->completion_lock) {struct io_kiocb *req;int i;
@@ -590,6 +590,7 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,for (i =0 ; i < (1U << ctx->cancel_hash_bits); i++) {struct hlist_head *list; +spin_lock (&ctx->cancel_hash_locks[i]); list = &ctx->cancel_hash[i];hlist_for_each_entry (req, list, hash_node) { if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
@@ -598,24 +599,28 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,if (cd->seq == req->work.cancel_seq)continue ; req->work.cancel_seq = cd->seq; +spin_unlock (&ctx->cancel_hash_locks[i]);return req; } +spin_unlock (&ctx->cancel_hash_locks[i]); }return NULL ; }static bool io_poll_disarm (struct io_kiocb *req,u32 index) - __must_hold(&ctx->completion_lock) { +struct io_ring_ctx *ctx = req->ctx; +if (!io_poll_get_ownership (req))return false;io_poll_remove_entries (req); +spin_lock (&ctx->cancel_hash_locks[index]);hash_del (&req->hash_node); +spin_unlock (&ctx->cancel_hash_locks[index]);return true; }int io_poll_cancel (struct io_ring_ctx *ctx,struct io_cancel_data *cd) - __must_hold(&ctx->completion_lock) {struct io_kiocb *req;
@@ -724,14 +729,11 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)int ret2, ret =0 ;bool locked; -spin_lock (&ctx->completion_lock); preq =io_poll_find (ctx, true, &cd);if (!preq || !io_poll_disarm (preq, cd.flags)) { -spin_unlock (&ctx->completion_lock); ret = preq ? -EALREADY : -ENOENT ; goto out; } -spin_unlock (&ctx->completion_lock);if (poll_update->update_events || poll_update->update_user_data) {/* only mask one event flags, keep behavior flags */
Signature · 1 line
2.25.1
Jens Axboe
maintainerRe: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentries
Interesting, do you have any numbers on this?
Also, I'd make a hash bucket struct:
struct io_hash_bucket {
spinlock_t lock;
struct hlist_head list;
};rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Hmm?
Signature · 1 line
Jens Axboe
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct io_hash_bucket { spinlock_t lock; struct hlist_head list; };
Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Sure, if the test number make sense, I'll send v2. I'll test the hlist_bl list as well(the comment of it says it is much slower than normal spin_lock, but we may not care the efficiency of poll cancellation very much?).
Quoted 3 lines
Hmm?
Jens Axboe
maintainerRe: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct io_hash_bucket { spinlock_t lock; struct hlist_head list; };Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
Signature · 1 line
Jens Axboe
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct io_hash_bucket { spinlock_t lock; struct hlist_head list; };Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).
Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
We can put the cancel hashes under uring_lock and completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.
Even better would be to have two hash tables, and auto-magically apply the feature to SINGLE_SUBMITTER, SQPOLL (both will have uring_lock held) and apoll (need uring_lock after anyway).
fs/io_uring.c +29 -12
@@ -7120,12 +7120,20 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) }io_poll_remove_entries (req); -spin_lock (&ctx->completion_lock); -hash_del (&req->hash_node); -__io_req_complete_post (req, req->cqe.res,0 ); -io_commit_cqring (ctx); -spin_unlock (&ctx->completion_lock); -io_cqring_ev_posted (ctx); + +if (ctx->flags &IORING_MUTEX_HASH ) { +io_tw_lock (ctx, locked); +hash_del (&req->hash_node); +io_req_complete_state (req, req->cqe.res,0 ); +io_req_add_compl_list (req); + }else { +spin_lock (&ctx->completion_lock); +hash_del (&req->hash_node); +__io_req_complete_post (req, req->cqe.res,0 ); +io_commit_cqring (ctx); +spin_unlock (&ctx->completion_lock); +io_cqring_ev_posted (ctx); + } }static void io_apoll_task_func (struct io_kiocb *req,bool *locked)
@@ -7138,9 +7146,14 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked)return ;io_poll_remove_entries (req); -spin_lock (&ctx->completion_lock); -hash_del (&req->hash_node); -spin_unlock (&ctx->completion_lock); +if (ctx->flags &IORING_MUTEX_HASH ) { +io_tw_lock (ctx, locked); +hash_del (&req->hash_node); + }else { +spin_lock (&ctx->completion_lock); +hash_del (&req->hash_node); +spin_unlock (&ctx->completion_lock); + }if (!ret)io_req_task_submit (req, locked);
@@ -7332,9 +7345,13 @@ static int __io_arm_poll_handler(struct io_kiocb *req,return 0 ; } -spin_lock (&ctx->completion_lock); -io_poll_req_insert (req); -spin_unlock (&ctx->completion_lock); +if (ctx->flags &IORING_MUTEX_HASH ) { +io_poll_req_insert (req); + }else { +spin_lock (&ctx->completion_lock); +io_poll_req_insert (req); +spin_unlock (&ctx->completion_lock); + }if (mask) {/* can't multishot if failed, just queue the event we've got */
Jens Axboe
maintainerRe: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 40 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct io_hash_bucket { spinlock_t lock; struct hlist_head list; };Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
Quoted 8 lines
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).
My hope was that it'd take us closer to being able to use more granular locking for hashing in general. I don't care too much about the cancelation, but the normal hash locking would be useful to do.
However, for cancelations, under uring_lock would indeed be preferable to doing per-bucket locks there. Guess I'll wait and see what your final patch looks like, not sure why it'd be a ctx conditional?
What about io_poll_remove_all()?
Signature · 1 line
Jens Axboe
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 50 lines · Jens Axboe
Quoted 40 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct io_hash_bucket { spinlock_t lock; struct hlist_head list; };Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
Quoted 8 lines
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).
Quoted 7 lines
My hope was that it'd take us closer to being able to use more granular locking for hashing in general. I don't care too much about the cancelation, but the normal hash locking would be useful to do.
However, for cancelations, under
uring_lockwould indeed be preferable to doing per-bucket locks there. Guess I'll wait and see what your final patch looks like, not sure why it'd be a ctx conditional?
It replaces 2 spin lock/unlock with one io_tw_lock() in the completion path, which is done once per tw batch and grabbed anyway if there is no contention (see handle_tw_list()).
It could be unconditional, but I'd say for those 3 cases we have non-existing chance to regress perf/latency, but I can think of some cases where it might screw latencies, all share io_uring b/w threads.
Should benefit the cancellation path as well, but I don't care about it as well.
Quoted 1 line
What about
io_poll_remove_all()?
As mentioned, it's not handled in the diff, but easily doable, it should just traverse both hash tables.
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 68 lines · Pavel Begunkov
Quoted 50 lines · Jens Axboe
Quoted 40 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct
io_hash_bucket{
spinlock_tlock;
structhlist_headlist;
};Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
Quoted 8 lines
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).Quoted 7 lines
My hope was that it'd take us closer to being able to use more granular locking for hashing in general. I don't care too much about the cancelation, but the normal hash locking would be useful to do.
However, for cancelations, under
uring_lockwould indeed be preferable to doing per-bucket locks there. Guess I'll wait and see what your final patch looks like, not sure why it'd be a ctx conditional?It replaces 2 spin lock/unlock with one
io_tw_lock()in the completion path, which is done once per tw batch and grabbed anyway if there is no contention (seehandle_tw_list()).It could be unconditional, but I'd say for those 3 cases we have non-existing chance to regress perf/latency, but I can think of some cases where it might screw latencies, all share
io_uringb/w threads.
Quoted 8 lines
Should benefit the cancellation path as well, but I don't care about it as well.
Quoted 1 line
What about
io_poll_remove_all()?As mentioned, it's not handled in the diff, but easily doable, it should just traverse both hash tables.
Two hash tables looks good to me. If I don't get you wrong, one table under uring_lock, the other one for normal handling(like per bucket locking)?
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 74 lines · Hao Xu
Quoted 73 lines · Pavel Begunkov
Quoted 50 lines · Jens Axboe
Quoted 40 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct
io_hash_bucket{
spinlock_tlock;
structhlist_headlist;
};Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
Quoted 8 lines
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).Quoted 7 lines
My hope was that it'd take us closer to being able to use more granular locking for hashing in general. I don't care too much about the cancelation, but the normal hash locking would be useful to do.
However, for cancelations, under
uring_lockwould indeed be preferable to doing per-bucket locks there. Guess I'll wait and see what your final patch looks like, not sure why it'd be a ctx conditional?It replaces 2 spin lock/unlock with one
io_tw_lock()in the completion path, which is done once per tw batch and grabbed anyway if there is no contention (seehandle_tw_list()).It could be unconditional, but I'd say for those 3 cases we have non-existing chance to regress perf/latency, but I can think of some cases where it might screw latencies, all share
io_uringb/w threads.Should benefit the cancellation path as well, but I don't care about it as well.
Quoted 1 line
What about
io_poll_remove_all()?
Quoted 7 lines
Quoted 3 lines
As mentioned, it's not handled in the diff, but easily doable, it should just traverse both hash tables.
Two hash tables looks good to me. If I don't get you wrong, one table under
uring_lock, the other one for normal handling(like per bucket locking)?
Right, that's the plan
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
In reply to Pavel BegunkovQuoted 102 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct
io_hash_bucket{
spinlock_tlock;
structhlist_headlist;
};Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).fs/io_uring.c +14
-6@@ -7120,12 +7120,20 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) } io_poll_remove_entries(req); - spin_lock(&ctx->completion_lock); - hash_del(&req->hash_node); - __io_req_complete_post(req, req->cqe.res, 0); - io_commit_cqring(ctx); - spin_unlock(&ctx->completion_lock); - io_cqring_ev_posted(ctx); + + if (ctx->flags & IORING_MUTEX_HASH) { + io_tw_lock(ctx, locked); + hash_del(&req->hash_node); + io_req_complete_state(req, req->cqe.res, 0); + io_req_add_compl_list(req); + } else { + spin_lock(&ctx->completion_lock); + hash_del(&req->hash_node); + __io_req_complete_post(req, req->cqe.res, 0); + io_commit_cqring(ctx); + spin_unlock(&ctx->completion_lock); + io_cqring_ev_posted(ctx); + } }static void
io_apoll_task_func(struct io_kiocb *req, bool *locked)fs/io_uring.c +8
-3@@ -7138,9 +7146,14 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) return; io_poll_remove_entries(req); - spin_lock(&ctx->completion_lock); - hash_del(&req->hash_node); - spin_unlock(&ctx->completion_lock); + if (ctx->flags & IORING_MUTEX_HASH) { + io_tw_lock(ctx, locked); + hash_del(&req->hash_node); + } else { + spin_lock(&ctx->completion_lock); + hash_del(&req->hash_node); + spin_unlock(&ctx->completion_lock); + } if (!ret)
io_req_task_submit(req, locked);
fs/io_uring.c static int __io_arm_poll_handler(struct io_kiocb · line 7349
Quoted 8 lines
fs/io_uring.c +1
-3@@ -7332,9 +7345,13 @@ static int __io_arm_poll_handler(struct io_kiocb *req, return 0; } - spin_lock(&ctx->completion_lock); - io_poll_req_insert(req); - spin_unlock(&ctx->completion_lock); + if (ctx->flags & IORING_MUTEX_HASH) {
Does IORING_MUTEX_HASH exclude IOSQE_ASYNC as well? though IOSQE_ASYNC is uncommon but users may do that.
Quoted 9 lines
fs/io_uring.c +6
-0+ io_poll_req_insert(req); + } else { + spin_lock(&ctx->completion_lock); + io_poll_req_insert(req); + spin_unlock(&ctx->completion_lock); + } if (mask) {/* can't multishot if failed, just queue the event we've got */
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
Quoted 107 lines · Hao Xu
Quoted 106 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct
io_hash_bucket{
spinlock_tlock;
structhlist_headlist;
};Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).fs/io_uring.c +14
-6@@ -7120,12 +7120,20 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) } io_poll_remove_entries(req); - spin_lock(&ctx->completion_lock); - hash_del(&req->hash_node); - __io_req_complete_post(req, req->cqe.res, 0); - io_commit_cqring(ctx); - spin_unlock(&ctx->completion_lock); - io_cqring_ev_posted(ctx); + + if (ctx->flags & IORING_MUTEX_HASH) { + io_tw_lock(ctx, locked); + hash_del(&req->hash_node); + io_req_complete_state(req, req->cqe.res, 0); + io_req_add_compl_list(req); + } else { + spin_lock(&ctx->completion_lock); + hash_del(&req->hash_node); + __io_req_complete_post(req, req->cqe.res, 0); + io_commit_cqring(ctx); + spin_unlock(&ctx->completion_lock); + io_cqring_ev_posted(ctx); + } }static void
io_apoll_task_func(struct io_kiocb *req, bool *locked)fs/io_uring.c +8
-3@@ -7138,9 +7146,14 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) return; io_poll_remove_entries(req); - spin_lock(&ctx->completion_lock); - hash_del(&req->hash_node); - spin_unlock(&ctx->completion_lock); + if (ctx->flags & IORING_MUTEX_HASH) { + io_tw_lock(ctx, locked); + hash_del(&req->hash_node); + } else { + spin_lock(&ctx->completion_lock); + hash_del(&req->hash_node); + spin_unlock(&ctx->completion_lock); + } if (!ret)
io_req_task_submit(req, locked);fs/io_uring.c +0
-0@@ -7332,9 +7345,13 @@ static int __io_arm_poll_handler(struct io_kiocb *req, return 0; }
Quoted 7 lines
fs/io_uring.c static int __io_arm_poll_handler(struct io_kiocb *req, · line 7348
Quoted 4 lines
fs/io_uring.c +1
-3- spin_lock(&ctx->completion_lock); - io_poll_req_insert(req); - spin_unlock(&ctx->completion_lock); + if (ctx->flags & IORING_MUTEX_HASH) {Does
IORING_MUTEX_HASHexcludeIOSQE_ASYNCas well? thoughIOSQE_ASYNCis uncommon but users may do that.
I see, cases like IOSQE_ASYNC definitely goes into the else branch since it's in iowq context.
Quoted 11 lines
Quoted 10 lines
fs/io_uring.c +6
-0+ io_poll_req_insert(req); + } else { + spin_lock(&ctx->completion_lock); + io_poll_req_insert(req); + spin_unlock(&ctx->completion_lock); + } if (mask) {/* can't multishot if failed, just queue the event we've got */
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
In reply to Hao XuQuoted 54 lines · Hao Xu
Quoted 53 lines · Pavel Begunkov
Quoted 33 lines · Jens Axboe
Quoted 27 lines · Hao Xu
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Just Theoretically for now, I'll do some tests tomorrow. This is actually RFC, forgot to change the subject.
Quoted 8 lines
Also, I'd make a hash bucket struct:
struct
io_hash_bucket{
spinlock_tlock;
structhlist_headlist;
};Quoted 4 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Quoted 4 lines
Sure, if the test number make sense, I'll send v2. I'll test the
hlist_bllist as well(the comment of it says it is much slower than normalspin_lock, but we may not care the efficiency of poll cancellation very much?).Quoted 5 lines
I don't think the bit spinlocks are going to be useful, we should stick with a spinlock for this. They are indeed slower and generally not used for that reason. For a use case where you need a ton of locks and saving the 4 bytes for a spinlock would make sense (or maybe not changing some struct?), maybe they have a purpose. But not for this.
We can put the cancel hashes under
uring_lockand completely kill the hash spinlocking (2 lock/unlock pairs per single-shot). The code below won't even compile and missing cancellation bits, I'll pick it up in a week.Even better would be to have two hash tables, and auto-magically apply the feature to
SINGLE_SUBMITTER, SQPOLL (both will haveuring_lockheld) and apoll (needuring_lockafter anyway).fs/io_uring.c +0
-0@@ -7332,9 +7345,13 @@ static int __io_arm_poll_handler(struct io_kiocb *req, return 0; }
Quoted 7 lines
fs/io_uring.c static int __io_arm_poll_handler(struct io_kiocb *req, · line 7348
Quoted 4 lines
fs/io_uring.c +1
-3- spin_lock(&ctx->completion_lock); - io_poll_req_insert(req); - spin_unlock(&ctx->completion_lock); + if (ctx->flags & IORING_MUTEX_HASH) {Does
IORING_MUTEX_HASHexcludeIOSQE_ASYNCas well? thoughIOSQE_ASYNCis uncommon but users may do that.
It should, io-wq -> arm_poll() doesn't hold uring_lock, good point
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/2] io_uring: switch cancel_hash to use per list spinlock
In reply to Jens AxboeQuoted 17 lines · Jens Axboe
Quoted 6 lines · Hao Xu
From: Hao Xu <[email protected]>
From: Hao Xu <[email protected]>
Use per list lock for
cancel_hash, this removes some completion lock invocation and remove contension between differentcancel_hashentriesInteresting, do you have any numbers on this?
Also, I'd make a hash bucket struct:
struct io_hash_bucket { spinlock_t lock; struct hlist_head list; };
Quoted 7 lines
rather than two separate structs, that'll have nicer memory locality too and should further improve it. Could be done as a prep patch with the old locking in place, making the end patch doing the per-bucket lock simpler as well.
Hmm?
I've done a v2 here, also a test which issues async poll densely to make high frequency cancel_hash[] visits. But I won't have a real box with big number of cpu processors which is suitable for testing until tomorrow, so I'd test it tomorrow.
https://github.com/HowHsu/linux/commits/for-5.20/io_uring_hash_lock
https://github.com/HowHsu/liburing/commit/b9fb4d20a5dfe7c7bd62fe36c37aea3b261d4499