io_uring: drop the old style inflight file tracking
Jens Axboe
maintainerPATCH v2 1/1 io_uring: drop the old style inflight file tracking
io_uring tracks requests that are referencing an io_uring descriptor to be able to cancel without worrying about loops in the references. Since we now assign the file at execution time, the easier approach is to drop a potentially problematic reference before we punt the request. This eliminates the need to special case these types of files beyond just marking them as such, and simplifies cancelation quite a bit.
This also fixes a recent issue where an async punted tee operation would with the io_uring descriptor as the output file would crash when attempting to get a reference to the file from the io-wq worker. We could have worked around that, but this is the much cleaner fix.
---
v2: right after sending out v1 I realized that we can then also kill
the weird cancelation hacks we have for retaining a tracked file across punt. Kill it with fire, this drops even more dead code.
fs/io_uring.c +22 -57
@@ -113,8 +113,7 @@ IOSQE_IO_DRAIN | IOSQE_CQE_SKIP_SUCCESS)#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ - REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \ - REQ_F_ASYNC_DATA) + REQ_F_POLLED | REQ_F_CREDS | REQ_F_ASYNC_DATA)#define IO_TCTX_REFS_CACHE_NR (1U << 10)
@@ -501,7 +500,6 @@ struct io_uring_task {const struct io_ring_ctx *last;struct io_wq *io_wq;struct percpu_counter inflight; -atomic_t inflight_tracked;atomic_t in_idle;spinlock_t task_lock;
@@ -1190,6 +1188,7 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,static void io_clean_op (struct io_kiocb *req);static struct file *io_file_get (struct io_ring_ctx *ctx,struct io_kiocb *req,int fd,bool fixed); +static void io_drop_inflight_file (struct io_kiocb *req);static void __io_queue_sqe (struct io_kiocb *req);static void io_rsrc_put_work (struct work_struct *work);
@@ -1430,29 +1429,9 @@ static bool io_match_task(struct io_kiocb *head, struct task_struct *task, bool cancel_all) __must_hold(&req->ctx->timeout_lock) { -struct io_kiocb *req; -if (task && head->task != task)return false; -if (cancel_all) -return true; - -io_for_each_link (req, head) { -if (req->flags &REQ_F_INFLIGHT ) -return true; - } -return false; -} - -static bool io_match_linked (struct io_kiocb *head) -{ -struct io_kiocb *req; - -io_for_each_link (req, head) { -if (req->flags &REQ_F_INFLIGHT ) -return true; - } -return false; +return cancel_all; }/*
@@ -1462,24 +1441,9 @@ static bool io_match_linked(struct io_kiocb *head)static bool io_match_task_safe (struct io_kiocb *head,struct task_struct *task,bool cancel_all) { -bool matched; -if (task && head->task != task)return false; -if (cancel_all) -return true; - -if (head->flags &REQ_F_LINK_TIMEOUT ) { -struct io_ring_ctx *ctx = head->ctx; - -/* protect against races with linked timeouts */ -spin_lock_irq (&ctx->timeout_lock); - matched =io_match_linked (head); -spin_unlock_irq (&ctx->timeout_lock); - }else { - matched =io_match_linked (head); - } -return matched; +return cancel_all; }static inline bool req_has_async_data (struct io_kiocb *req)
@@ -1642,14 +1606,6 @@ static inline bool io_req_ffs_set(struct io_kiocb *req)return req->flags &REQ_F_FIXED_FILE ; } -static inline void io_req_track_inflight (struct io_kiocb *req) -{ -if (!(req->flags &REQ_F_INFLIGHT )) { - req->flags |=REQ_F_INFLIGHT ; -atomic_inc (¤t->io_uring->inflight_tracked); - } -} -static struct io_kiocb *__io_prep_linked_timeout (struct io_kiocb *req) {if (WARN_ON_ONCE (!req->link))
@@ -2560,6 +2516,8 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)WARN_ON_ONCE (!tctx); +io_drop_inflight_file (req); +spin_lock_irqsave (&tctx->task_lock, flags);if (priority)wq_list_add_tail (&req->io_task_work.node, &tctx->prior_task_list);
@@ -7198,11 +7156,6 @@ static void io_clean_op(struct io_kiocb *req)kfree (req->apoll); req->apoll =NULL ; } -if (req->flags &REQ_F_INFLIGHT ) { -struct io_uring_task *tctx = req->task->io_uring; - -atomic_dec (&tctx->inflight_tracked); - }if (req->flags &REQ_F_CREDS )put_cred (req->creds);if (req->flags &REQ_F_ASYNC_DATA ) {
@@ -7487,6 +7440,19 @@ static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,return file; } +/* +* Drop the file for requeue operations. Only used of req->file is the +* io_uring descriptor itself. +*/ +static void io_drop_inflight_file (struct io_kiocb *req) +{ +if (unlikely (req->flags &REQ_F_INFLIGHT )) { +fput (req->file); + req->file =NULL ; + req->flags &= ~REQ_F_INFLIGHT ; + } +} +static struct file *io_file_get_normal (struct io_ring_ctx *ctx,struct io_kiocb *req,int fd) {
@@ -7495,8 +7461,8 @@ static struct file *io_file_get_normal(struct io_ring_ctx *ctx,trace_io_uring_file_get (ctx, req, req->user_data, fd);/* we don't allow fixed io_uring files */ -if (file &&unlikely (file->f_op == &io_uring_fops)) -io_req_track_inflight (req); +if (unlikely (file &&unlikely (file->f_op == &io_uring_fops))) + req->flags |=REQ_F_INFLIGHT ;return file; }
@@ -9412,7 +9378,6 @@ static __cold int io_uring_alloc_task_context(struct task_struct *task,xa_init (&tctx->xa);init_waitqueue_head (&tctx->wait);atomic_set (&tctx->in_idle,0 ); -atomic_set (&tctx->inflight_tracked,0 ); task->io_uring = tctx;spin_lock_init (&tctx->task_lock);INIT_WQ_LIST (&tctx->task_list);
@@ -10605,7 +10570,7 @@ static __cold void io_uring_clean_tctx(struct io_uring_task *tctx)static s64 tctx_inflight (struct io_uring_task *tctx,bool tracked) {if (tracked) -return atomic_read (&tctx->inflight_tracked); +return 0 ;return percpu_counter_sum (&tctx->inflight); }
Signature · 1 line
Jens Axboe