Poll cleanups and unlocked table removal
Jens Axboe
maintainercover Poll cleanups and unlocked table removal
Hi,
This patchset gets rid of the distinction between the locked and unlocked hashed cancel table, by simply ensuring that the unlocked issuer grabs the lock. That then enables us to drop a bunch of code and helpers (and even a request flag), which is always nice.
Ran this through the usual testing, no issues observed.
include/linux/io_uring_types.h | 7 +- io_uring/cancel.c | 10 -- io_uring/cancel.h | 1 - io_uring/fdinfo.c | 11 +-- io_uring/io_uring.c | 8 +- io_uring/poll.c | 173 +++++++-------------------------- 6 files changed, 40 insertions(+), 170 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/5 io_uring/poll: remove 'ctx' argument from io_poll_req_delete()
It's always req->ctx being used anyway, having this as a separate argument (that is then not even used) just makes it more confusing.
--- io_uring/poll.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/poll.c +2 -2
@@ -129,7 +129,7 @@ static void io_poll_req_insert(struct io_kiocb *req)spin_unlock (&hb->lock); } -static void io_poll_req_delete (struct io_kiocb *req,struct io_ring_ctx *ctx) +static void io_poll_req_delete (struct io_kiocb *req) {struct io_hash_table *table = &req->ctx->cancel_table;u32 index =hash_long (req->cqe.user_data, table->hash_bits);
@@ -165,7 +165,7 @@ static void io_poll_tw_hash_eject(struct io_kiocb *req, struct io_tw_state *ts)hash_del (&req->hash_node); req->flags &= ~REQ_F_HASH_LOCKED ; }else { -io_poll_req_delete (req, ctx); +io_poll_req_delete (req); } }
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 2/5 io_uring/poll: get rid of unlocked cancel hash
In reply to Jens Axboeio_uring maintains two hash lists of inflight requests:
1) ctx->cancel_table_locked. This is used when the caller has the ctx->uring_lock held already. This is only an issue side parameter, as removal or task_work will always have it held.
2) ctx->cancel_table. This is used when the issuer does NOT have the ctx->uring_lock held, and relies on the table spinlocks for access.
However, it's pretty trivial to simply grab the lock in the one spot where we care about it, for insertion. With that, we can kill the unlocked table (and get rid of the _locked postfix for the other one).
--- include/linux/io_uring_types.h | 6 +- io_uring/fdinfo.c | 11 +-- io_uring/io_uring.c | 4 - io_uring/poll.c | 139 ++++++++------------------------- 4 files changed, 36 insertions(+), 124 deletions(-)
include/linux/io_uring_types.h +1 -5
@@ -291,7 +291,7 @@ struct io_ring_ctx {struct xarray io_bl_xa; -struct io_hash_table cancel_table_locked; +struct io_hash_table cancel_table;struct io_alloc_cache apoll_cache;struct io_alloc_cache netmsg_cache;struct io_alloc_cache rw_cache;
@@ -342,7 +342,6 @@ struct io_ring_ctx {struct list_head io_buffers_comp;struct list_head cq_overflow_list; -struct io_hash_table cancel_table;struct hlist_head waitid_list;
@@ -459,7 +458,6 @@ enum {REQ_F_DOUBLE_POLL_BIT ,REQ_F_APOLL_MULTISHOT_BIT ,REQ_F_CLEAR_POLLIN_BIT , -REQ_F_HASH_LOCKED_BIT ,/* keep async read/write and isreg together and in order */ REQ_F_SUPPORT_NOWAIT_BIT , REQ_F_ISREG_BIT,
@@ -534,8 +532,6 @@ enum {REQ_F_APOLL_MULTISHOT =IO_REQ_FLAG (REQ_F_APOLL_MULTISHOT_BIT ),/* recvmsg special flag, clear EPOLLIN */ REQ_F_CLEAR_POLLIN =IO_REQ_FLAG (REQ_F_CLEAR_POLLIN_BIT ), -/* hashed into ->cancel_hash_locked, protected by ->uring_lock */ -REQ_F_HASH_LOCKED =IO_REQ_FLAG (REQ_F_HASH_LOCKED_BIT ),/* don't use lazy poll wake for this request */ REQ_F_POLL_NO_LAZY = IO_REQ_FLAG(REQ_F_POLL_NO_LAZY_BIT),/* file is pollable */
io_uring/fdinfo.c +1 -10
@@ -190,22 +190,13 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) }seq_puts (m,"PollList:\n" ); -for (i =0 ; i < (1U << ctx->cancel_table.hash_bits); i++) { +for (i =0 ; has_lock && i < (1U << ctx->cancel_table.hash_bits); i++) {struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; -struct io_hash_bucket *hbl = &ctx->cancel_table_locked.hbs[i];struct io_kiocb *req; -spin_lock (&hb->lock);hlist_for_each_entry (req, &hb->list, hash_node)seq_printf (m," op=%d, task_works=%d\n" , req->opcode,task_work_pending (req->task)); -spin_unlock (&hb->lock); - -if (!has_lock) -continue ; -hlist_for_each_entry (req, &hbl->list, hash_node) -seq_printf (m," op=%d, task_works=%d\n" , req->opcode, -task_work_pending (req->task)); }if (has_lock)
io_uring/io_uring.c +0 -4
@@ -294,8 +294,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) hash_bits =clamp (hash_bits,1 ,8 );if (io_alloc_hash_table (&ctx->cancel_table, hash_bits)) goto err; -if (io_alloc_hash_table (&ctx->cancel_table_locked, hash_bits)) - goto err;if (percpu_ref_init (&ctx->refs, io_ring_ctx_ref_free,0 ,GFP_KERNEL )) goto err;
@@ -358,7 +356,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)io_alloc_cache_free (&ctx->msg_cache, io_msg_cache_free);io_futex_cache_free (ctx);kfree (ctx->cancel_table.hbs); -kfree (ctx->cancel_table_locked.hbs);xa_destroy (&ctx->io_bl_xa);kfree (ctx);return NULL ;
@@ -2771,7 +2768,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_wq_put_hash (ctx->hash_map);io_napi_free (ctx);kfree (ctx->cancel_table.hbs); -kfree (ctx->cancel_table_locked.hbs);xa_destroy (&ctx->io_bl_xa);kfree (ctx); }
io_uring/poll.c +34 -105
261 lines not on this page · raw message
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 3/5 io_uring/poll: get rid of io_poll_tw_hash_eject()
In reply to Jens AxboeIt serves no purposes anymore, all it does is delete the hash list entry. task_work always has the ring locked.
--- io_uring/poll.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-)
io_uring/poll.c +2 -15
@@ -128,20 +128,6 @@ static void io_poll_req_insert(struct io_kiocb *req)hlist_add_head (&req->hash_node, &table->hbs[index].list); } -static void io_poll_tw_hash_eject (struct io_kiocb *req,struct io_tw_state *ts) -{ -struct io_ring_ctx *ctx = req->ctx; - -/* -* ->cancel_table_locked is protected by ->uring_lock in -* contrast to per bucket spinlocks. Likely, tctx_task_work() -* already grabbed the mutex for us, but there is a chance it -* failed. -*/ -io_tw_lock (ctx, ts); -hash_del (&req->hash_node); -} -static void io_init_poll_iocb (struct io_poll *poll,__poll_t events) { poll->head =NULL ;
@@ -336,7 +322,8 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)return ; }io_poll_remove_entries (req); -io_poll_tw_hash_eject (req, ts); +/* task_work always has ->uring_lock held */ +hash_del (&req->hash_node);if (req->opcode ==IORING_OP_POLL_ADD ) {if (ret ==IOU_POLL_DONE ) {
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 4/5 io_uring/poll: get rid of per-hashtable bucket locks
In reply to Jens AxboeAny access to the table is protected by ctx->uring_lock now anyway, the per-bucket locking doesn't buy us anything.
--- include/linux/io_uring_types.h | 1 - io_uring/cancel.c | 4 +--- io_uring/poll.c | 39 +++++++++------------------------- 3 files changed, 11 insertions(+), 33 deletions(-)
include/linux/io_uring_types.h +0 -1
@@ -67,7 +67,6 @@ struct io_file_table { };struct io_hash_bucket { -spinlock_t lock;struct hlist_head list; }____cacheline_aligned_in_smp ;
io_uring/cancel.c +1 -3
@@ -236,10 +236,8 @@ void init_hash_table(struct io_hash_table *table, unsigned size) {unsigned int i; -for (i =0 ; i < size; i++) { -spin_lock_init (&table->hbs[i].lock); +for (i =0 ; i < size; i++)INIT_HLIST_HEAD (&table->hbs[i].list); - } } static int __io_sync_cancel(struct io_uring_task *tctx,
io_uring/poll.c +10 -29
@@ -728,7 +728,6 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,for (i =0 ; i < nr_buckets; i++) {struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; -spin_lock (&hb->lock);hlist_for_each_entry_safe (req, tmp, &hb->list, hash_node) {if (io_match_task_safe (req, tsk, cancel_all)) {hlist_del_init (&req->hash_node);
@@ -736,22 +735,17 @@ __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, found = true; } } -spin_unlock (&hb->lock); }return found; }static struct io_kiocb *io_poll_find (struct io_ring_ctx *ctx,bool poll_only, -struct io_cancel_data *cd, -struct io_hash_bucket **out_bucket) +struct io_cancel_data *cd) {struct io_kiocb *req;u32 index =hash_long (cd->data, ctx->cancel_table.hash_bits);struct io_hash_bucket *hb = &ctx->cancel_table.hbs[index]; - *out_bucket =NULL ; - -spin_lock (&hb->lock);hlist_for_each_entry (req, &hb->list, hash_node) {if (cd->data != req->cqe.user_data)continue ;
@@ -761,34 +755,25 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,if (io_cancel_match_sequence (req, cd->seq))continue ; } - *out_bucket = hb;return req; } -spin_unlock (&hb->lock);return NULL ; }static struct io_kiocb *io_poll_file_find (struct io_ring_ctx *ctx, -struct io_cancel_data *cd, -struct io_hash_bucket **out_bucket) +struct io_cancel_data *cd) {unsigned nr_buckets =1U << ctx->cancel_table.hash_bits;struct io_kiocb *req;int i; - *out_bucket =NULL ; -for (i =0 ; i < nr_buckets; i++) {struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i]; -spin_lock (&hb->lock);hlist_for_each_entry (req, &hb->list, hash_node) { -if (io_cancel_req_match (req, cd)) { - *out_bucket = hb; +if (io_cancel_req_match (req, cd))return req; - } } -spin_unlock (&hb->lock); }return NULL ; }
@@ -806,20 +791,19 @@ static int io_poll_disarm(struct io_kiocb *req)static int __io_poll_cancel (struct io_ring_ctx *ctx,struct io_cancel_data *cd) { -struct io_hash_bucket *bucket;struct io_kiocb *req;if (cd->flags & (IORING_ASYNC_CANCEL_FD |IORING_ASYNC_CANCEL_OP |IORING_ASYNC_CANCEL_ANY )) - req =io_poll_file_find (ctx, cd, &bucket); + req =io_poll_file_find (ctx, cd);else - req =io_poll_find (ctx, false, cd, &bucket); + req =io_poll_find (ctx, false, cd); -if (req) +if (req) {io_poll_cancel_req (req); -if (bucket) -spin_unlock (&bucket->lock); -return req ?0 : -ENOENT ; +return 0 ; + } +return -ENOENT ; } int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
@@ -918,15 +902,12 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update);struct io_ring_ctx *ctx = req->ctx;struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, }; -struct io_hash_bucket *bucket;struct io_kiocb *preq;int ret2, ret =0 ;io_ring_submit_lock (ctx, issue_flags); - preq =io_poll_find (ctx, true, &cd, &bucket); + preq =io_poll_find (ctx, true, &cd); ret2 =io_poll_disarm (preq); -if (bucket) -spin_unlock (&bucket->lock);if (!ret2) goto found;if (ret2) {
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 5/5 io_uring/cancel: get rid of init_hash_table() helper
In reply to Jens AxboeAll it does is initialize the lists, just move the INIT_HLIST_HEAD() into the one caller.
--- io_uring/cancel.c | 8 -------- io_uring/cancel.h | 1 - io_uring/io_uring.c | 4 +++- 3 files changed, 3 insertions(+), 10 deletions(-)
io_uring/cancel.c +0 -8
@@ -232,14 +232,6 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)return IOU_OK ; } -void init_hash_table (struct io_hash_table *table,unsigned size) -{ -unsigned int i; - -for (i =0 ; i < size; i++) -INIT_HLIST_HEAD (&table->hbs[i].list); -} -static int __io_sync_cancel (struct io_uring_task *tctx,struct io_cancel_data *cd,int fd) {
io_uring/cancel.h +0 -1
@@ -20,7 +20,6 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags);int io_try_cancel (struct io_uring_task *tctx,struct io_cancel_data *cd,unsigned int issue_flags); -void init_hash_table (struct io_hash_table *table,unsigned size);int io_sync_cancel (struct io_ring_ctx *ctx,void __user *arg);bool io_cancel_req_match (struct io_kiocb *req,struct io_cancel_data *cd);
io_uring/io_uring.c +3 -1
@@ -263,13 +263,15 @@ static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits) {unsigned hash_buckets =1U << bits;size_t hash_size = hash_buckets *sizeof (table->hbs[0 ]); +int i; table->hbs =kmalloc (hash_size,GFP_KERNEL );if (!table->hbs)return -ENOMEM ; table->hash_bits = bits; -init_hash_table (table, hash_buckets); +for (i =0 ; i < hash_buckets; i++) +INIT_HLIST_HEAD (&table->hbs[i].list);return 0 ; }
Signature · 1 line
2.45.2