Misc cleanups / optimizations
Jens Axboe
maintainerv2 cover Misc cleanups / optimizations
Hi,
Nothing major in here:
- Expand io_kiocb flags to 64-bits, so we can use two more bits for caching cancelation sequence and pollable state.
- Misc cleanups
Changes since v1:
- Drop nr_tw union with comp_list, that breaks iopoll with DEFER_TASKRUN usage.
- Rearrange io_kiocb again in patch 1, now just moving nr_tw up to fill the new hole, and shifting rsrc_node down to keep io_comp_list in the 2nd cacheline.
- Add cleanup patch for io_req_complete_post()
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v2 1/6 io_uring: expand main struct io_kiocb flags to 64-bits
We're out of space here, and none of the flags are easily reclaimable. Bump it to 64-bits and re-arrange the struct a bit to avoid gaps.
Add a specific bitwise type for the request flags, io_request_flags_t. This will help catch violations of casting this value to a smaller type on 32-bit archs, like unsigned int.
This creates a hole in the io_kiocb, so move nr_tw up and rsrc_node down to retain needing only cacheline 0 and 1 for non-polled opcodes.
No functional changes intended in this patch.
--- include/linux/io_uring_types.h | 77 ++++++++++++++++++--------------- include/trace/events/io_uring.h | 14 +++--- io_uring/filetable.h | 2 +- io_uring/io_uring.c | 9 ++-- 4 files changed, 55 insertions(+), 47 deletions(-)
include/linux/io_uring_types.h +42 -35
@@ -468,70 +468,73 @@ enum { __REQ_F_LAST_BIT, }; +typedef u64 __bitwise io_req_flags_t ; +#define IO_REQ_FLAG (bitno) ((__force io_req_flags_t) BIT_ULL((bitno))) +enum {/* ctx owns file */ -REQ_F_FIXED_FILE =BIT (REQ_F_FIXED_FILE_BIT ), +REQ_F_FIXED_FILE =IO_REQ_FLAG (REQ_F_FIXED_FILE_BIT ),/* drain existing IO first */ -REQ_F_IO_DRAIN =BIT (REQ_F_IO_DRAIN_BIT ), +REQ_F_IO_DRAIN =IO_REQ_FLAG (REQ_F_IO_DRAIN_BIT ),/* linked sqes */ -REQ_F_LINK =BIT (REQ_F_LINK_BIT ), +REQ_F_LINK =IO_REQ_FLAG (REQ_F_LINK_BIT ),/* doesn't sever on completion < 0 */ -REQ_F_HARDLINK =BIT (REQ_F_HARDLINK_BIT ), +REQ_F_HARDLINK =IO_REQ_FLAG (REQ_F_HARDLINK_BIT ),/* IOSQE_ASYNC */ -REQ_F_FORCE_ASYNC =BIT (REQ_F_FORCE_ASYNC_BIT ), +REQ_F_FORCE_ASYNC =IO_REQ_FLAG (REQ_F_FORCE_ASYNC_BIT ),/* IOSQE_BUFFER_SELECT */ -REQ_F_BUFFER_SELECT =BIT (REQ_F_BUFFER_SELECT_BIT ), +REQ_F_BUFFER_SELECT =IO_REQ_FLAG (REQ_F_BUFFER_SELECT_BIT ),/* IOSQE_CQE_SKIP_SUCCESS */ -REQ_F_CQE_SKIP =BIT (REQ_F_CQE_SKIP_BIT ), +REQ_F_CQE_SKIP =IO_REQ_FLAG (REQ_F_CQE_SKIP_BIT ),/* fail rest of links */ -REQ_F_FAIL =BIT (REQ_F_FAIL_BIT ), +REQ_F_FAIL =IO_REQ_FLAG (REQ_F_FAIL_BIT ),/* on inflight list, should be cancelled and waited on exit reliably */ -REQ_F_INFLIGHT =BIT (REQ_F_INFLIGHT_BIT ), +REQ_F_INFLIGHT =IO_REQ_FLAG (REQ_F_INFLIGHT_BIT ),/* read/write uses file position */ -REQ_F_CUR_POS =BIT (REQ_F_CUR_POS_BIT ), +REQ_F_CUR_POS =IO_REQ_FLAG (REQ_F_CUR_POS_BIT ),/* must not punt to workers */ -REQ_F_NOWAIT =BIT (REQ_F_NOWAIT_BIT ), +REQ_F_NOWAIT =IO_REQ_FLAG (REQ_F_NOWAIT_BIT ),/* has or had linked timeout */ -REQ_F_LINK_TIMEOUT =BIT (REQ_F_LINK_TIMEOUT_BIT ), +REQ_F_LINK_TIMEOUT =IO_REQ_FLAG (REQ_F_LINK_TIMEOUT_BIT ),/* needs cleanup */ -REQ_F_NEED_CLEANUP =BIT (REQ_F_NEED_CLEANUP_BIT ), +REQ_F_NEED_CLEANUP =IO_REQ_FLAG (REQ_F_NEED_CLEANUP_BIT ),/* already went through poll handler */ -REQ_F_POLLED =BIT (REQ_F_POLLED_BIT ), +REQ_F_POLLED =IO_REQ_FLAG (REQ_F_POLLED_BIT ),/* buffer already selected */ -REQ_F_BUFFER_SELECTED =BIT (REQ_F_BUFFER_SELECTED_BIT ), +REQ_F_BUFFER_SELECTED =IO_REQ_FLAG (REQ_F_BUFFER_SELECTED_BIT ),/* buffer selected from ring, needs commit */ -REQ_F_BUFFER_RING =BIT (REQ_F_BUFFER_RING_BIT ), +REQ_F_BUFFER_RING =IO_REQ_FLAG (REQ_F_BUFFER_RING_BIT ),/* caller should reissue async */ -REQ_F_REISSUE =BIT (REQ_F_REISSUE_BIT ), +REQ_F_REISSUE =IO_REQ_FLAG (REQ_F_REISSUE_BIT ),/* supports async reads/writes */ -REQ_F_SUPPORT_NOWAIT =BIT (REQ_F_SUPPORT_NOWAIT_BIT ), +REQ_F_SUPPORT_NOWAIT =IO_REQ_FLAG (REQ_F_SUPPORT_NOWAIT_BIT ),/* regular file */ -REQ_F_ISREG =BIT (REQ_F_ISREG_BIT ), +REQ_F_ISREG =IO_REQ_FLAG (REQ_F_ISREG_BIT ),/* has creds assigned */ -REQ_F_CREDS =BIT (REQ_F_CREDS_BIT ), +REQ_F_CREDS =IO_REQ_FLAG (REQ_F_CREDS_BIT ),/* skip refcounting if not set */ -REQ_F_REFCOUNT =BIT (REQ_F_REFCOUNT_BIT ), +REQ_F_REFCOUNT =IO_REQ_FLAG (REQ_F_REFCOUNT_BIT ),/* there is a linked timeout that has to be armed */ -REQ_F_ARM_LTIMEOUT =BIT (REQ_F_ARM_LTIMEOUT_BIT ), +REQ_F_ARM_LTIMEOUT =IO_REQ_FLAG (REQ_F_ARM_LTIMEOUT_BIT ),/* ->async_data allocated */ -REQ_F_ASYNC_DATA =BIT (REQ_F_ASYNC_DATA_BIT ), +REQ_F_ASYNC_DATA =IO_REQ_FLAG (REQ_F_ASYNC_DATA_BIT ),/* don't post CQEs while failing linked requests */ -REQ_F_SKIP_LINK_CQES =BIT (REQ_F_SKIP_LINK_CQES_BIT ), +REQ_F_SKIP_LINK_CQES =IO_REQ_FLAG (REQ_F_SKIP_LINK_CQES_BIT ),/* single poll may be active */ -REQ_F_SINGLE_POLL =BIT (REQ_F_SINGLE_POLL_BIT ), +REQ_F_SINGLE_POLL =IO_REQ_FLAG (REQ_F_SINGLE_POLL_BIT ),/* double poll may active */ -REQ_F_DOUBLE_POLL =BIT (REQ_F_DOUBLE_POLL_BIT ), +REQ_F_DOUBLE_POLL =IO_REQ_FLAG (REQ_F_DOUBLE_POLL_BIT ),/* request has already done partial IO */ -REQ_F_PARTIAL_IO =BIT (REQ_F_PARTIAL_IO_BIT ), +REQ_F_PARTIAL_IO =IO_REQ_FLAG (REQ_F_PARTIAL_IO_BIT ),/* fast poll multishot mode */ -REQ_F_APOLL_MULTISHOT =BIT (REQ_F_APOLL_MULTISHOT_BIT ), +REQ_F_APOLL_MULTISHOT =IO_REQ_FLAG (REQ_F_APOLL_MULTISHOT_BIT ),/* recvmsg special flag, clear EPOLLIN */ -REQ_F_CLEAR_POLLIN =BIT (REQ_F_CLEAR_POLLIN_BIT ), +REQ_F_CLEAR_POLLIN =IO_REQ_FLAG (REQ_F_CLEAR_POLLIN_BIT ),/* hashed into ->cancel_hash_locked, protected by ->uring_lock */ -REQ_F_HASH_LOCKED =BIT (REQ_F_HASH_LOCKED_BIT ), +REQ_F_HASH_LOCKED =IO_REQ_FLAG (REQ_F_HASH_LOCKED_BIT ),/* don't use lazy poll wake for this request */ -REQ_F_POLL_NO_LAZY =BIT (REQ_F_POLL_NO_LAZY_BIT ), +REQ_F_POLL_NO_LAZY =IO_REQ_FLAG (REQ_F_POLL_NO_LAZY_BIT ), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);
@@ -592,15 +595,17 @@ struct io_kiocb {* and after selection it points to the buffer ID itself. */ u16 buf_index; -unsigned int flags; + +unsigned nr_tw; + +/* REQ_F_* flags */ +io_req_flags_t flags;struct io_cqe cqe;struct io_ring_ctx *ctx;struct task_struct *task; -struct io_rsrc_node *rsrc_node; -union {/* store used ubuf, so we can prevent reloading */ struct io_mapped_ubuf *imu;
@@ -621,10 +626,12 @@ struct io_kiocb {/* cache ->apoll->events */ __poll_t apoll_events; }; + +struct io_rsrc_node *rsrc_node; +atomic_t refs;atomic_t poll_refs;struct io_task_work io_task_work; -unsigned nr_tw;/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node;/* internal polling, see IORING_FEAT_FAST_POLL */
include/trace/events/io_uring.h +7 -7
@@ -148,7 +148,7 @@ TRACE_EVENT(io_uring_queue_async_work, __field( void *, req ) __field( u64, user_data ) __field( u8, opcode ) - __field( unsigned int, flags ) + __field( io_req_flags_t, flags ) __field( struct io_wq_work *, work ) __field( int, rw )
@@ -167,10 +167,10 @@ TRACE_EVENT(io_uring_queue_async_work,__assign_str (op_str,io_uring_get_opcode (req->opcode)); ), -TP_printk ("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%x, %s queue, work %p" , +TP_printk ("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%lx, %s queue, work %p" , __entry->ctx, __entry->req, __entry->user_data, -__get_str (op_str), - __entry->flags, __entry->rw ?"hashed" :"normal" , __entry->work) +__get_str (op_str), (long ) __entry->flags, + __entry->rw ?"hashed" :"normal" , __entry->work) );/**
@@ -378,7 +378,7 @@ TRACE_EVENT(io_uring_submit_req, __field( void *, req ) __field( unsigned long long, user_data ) __field( u8, opcode ) - __field( u32, flags ) + __field( io_req_flags_t, flags ) __field( bool, sq_thread ) __string( op_str, io_uring_get_opcode(req->opcode) )
@@ -395,10 +395,10 @@ TRACE_EVENT(io_uring_submit_req,__assign_str (op_str,io_uring_get_opcode (req->opcode)); ), -TP_printk ("ring %p, req %p, user_data 0x%llx, opcode %s, flags 0x%x, " +TP_printk ("ring %p, req %p, user_data 0x%llx, opcode %s, flags 0x%lx, " "sq_thread %d" , __entry->ctx, __entry->req, __entry->user_data,__get_str (op_str), - __entry->flags, __entry->sq_thread) + (long ) __entry->flags, __entry->sq_thread) );/*
io_uring/filetable.h +1 -1
@@ -17,7 +17,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset);int io_register_file_alloc_range (struct io_ring_ctx *ctx,struct io_uring_file_index_range __user *arg); -unsigned int io_file_get_flags (struct file *file); +io_req_flags_t io_file_get_flags (struct file *file);static inline void io_file_bitmap_clear (struct io_file_table *table,int bit) {
io_uring/io_uring.c +5 -4
@@ -1768,9 +1768,9 @@ static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags) } } -unsigned int io_file_get_flags (struct file *file) +io_req_flags_t io_file_get_flags (struct file *file) { -unsigned int res =0 ; +io_req_flags_t res =0 ;if (S_ISREG (file_inode (file)->i_mode)) res |=REQ_F_ISREG ;
@@ -2171,7 +2171,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,/* req is partially pre-initialised, see io_preinit_req() */ req->opcode = opcode =READ_ONCE (sqe->opcode);/* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags = sqe_flags =READ_ONCE (sqe->flags); + sqe_flags =READ_ONCE (sqe->flags); + req->flags = (io_req_flags_t ) sqe_flags; req->cqe.user_data =READ_ONCE (sqe->user_data); req->file =NULL ; req->rsrc_node =NULL ;
@@ -4153,7 +4154,7 @@ static int __init io_uring_init(void)BUILD_BUG_ON (SQE_COMMON_FLAGS >= (1 <<8 ));BUILD_BUG_ON ((SQE_VALID_FLAGS |SQE_COMMON_FLAGS ) !=SQE_VALID_FLAGS ); -BUILD_BUG_ON (__REQ_F_LAST_BIT >8 *sizeof (int )); + BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof_field(struct io_kiocb, flags));BUILD_BUG_ON (sizeof (atomic_t) !=sizeof (u32));
Signature · 1 line
2.43.0
Re: [PATCH 1/6] io_uring: expand main struct io_kiocb flags to 64-bits
include/trace/events/io_uring.h TRACE_EVENT(io_uring_queue_async_work, · line 172
Quoted 6 lines · Jens Axboe
include/trace/events/io_uring.h +2
-3-TP_printk ("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%x, %s queue, work %p" , + TP_printk("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%lx, %s queue, work %p", __entry->ctx, __entry->req, __entry->user_data, -__get_str (op_str), - __entry->flags, __entry->rw ?"hashed" :"normal" , __entry->work) + __get_str(op_str), (long) __entry->flags,
Hi Jens,
Minor, but on 32-bit kernel the cast is wrong since sizeof(long)==4. Afaik, io_uring still builds on 32-bit archs.
If you use (unsigned long long), it will be 64 bit anywhere.
Quoted 14 lines
include/trace/events/io_uring.h +2
-1+ __entry->rw ? "hashed" : "normal", __entry->work) );/** @@ -378,7 +378,7 @@ TRACE_EVENT(io_uring_submit_req, __field( void *, req ) __field( unsigned long long, user_data ) __field( u8, opcode ) - __field( u32, flags ) + __field( io_req_flags_t, flags ) __field( bool, sq_thread ) __string( op_str, io_uring_get_opcode(req->opcode) )
include/trace/events/io_uring.h TRACE_EVENT(io_uring_submit_req, · line 401
Quoted 9 lines
include/trace/events/io_uring.h +2
-2@@ -395,10 +395,10 @@ TRACE_EVENT(io_uring_submit_req,__assign_str (op_str,io_uring_get_opcode (req->opcode)); ), -TP_printk ("ring %p, req %p, user_data 0x%llx, opcode %s, flags 0x%x, " +TP_printk ("ring %p, req %p, user_data 0x%llx, opcode %s, flags 0x%lx, " "sq_thread %d" , __entry->ctx, __entry->req, __entry->user_data,__get_str (op_str), - __entry->flags, __entry->sq_thread) + (long ) __entry->flags, __entry->sq_thread)
likewise.
Quoted 33 lines
(quoted diff) +0
-0); /*io_uring/filetable.h +1
-1@@ -17,7 +17,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset);int io_register_file_alloc_range (struct io_ring_ctx *ctx,struct io_uring_file_index_range __user *arg); -unsigned int io_file_get_flags (struct file *file); +io_req_flags_t io_file_get_flags (struct file *file);static inline void io_file_bitmap_clear (struct io_file_table *table,int bit) {io_uring/io_uring.c +2
-2@@ -1768,9 +1768,9 @@ static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags) } } -unsigned int io_file_get_flags (struct file *file) +io_req_flags_t io_file_get_flags (struct file *file) { -unsigned int res =0 ; +io_req_flags_t res =0 ;if (S_ISREG (file_inode (file)->i_mode)) res |=REQ_F_ISREG ;
io_uring/io_uring.c static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · line 2174
Quoted 5 lines
io_uring/io_uring.c +1
-1@@ -2171,7 +2171,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,/* req is partially pre-initialised, see io_preinit_req() */ req->opcode = opcode =READ_ONCE (sqe->opcode);/* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags = sqe_flags =READ_ONCE (sqe->flags); + sqe_flags =READ_ONCE (sqe->flags);
Did you consider that READ_ONCE won't protect from load tearing the userspace value in 32-bit architectures? It builds silently, though, and I suspect it is mostly fine in the current code, but might become a bug eventually.
Thanks,
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerRe: [PATCH 1/6] io_uring: expand main struct io_kiocb flags to 64-bits
Quoted 12 lines · Gabriel Krisman Bertazi
include/trace/events/io_uring.h TRACE_EVENT(io_uring_queue_async_work, · line 172
Quoted 6 lines · Jens Axboe
include/trace/events/io_uring.h +2
-3-TP_printk ("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%x, %s queue, work %p" , + TP_printk("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%lx, %s queue, work %p", __entry->ctx, __entry->req, __entry->user_data, -__get_str (op_str), - __entry->flags, __entry->rw ?"hashed" :"normal" , __entry->work) + __get_str(op_str), (long) __entry->flags,Hi Jens,
Quoted 4 lines
Minor, but on 32-bit kernel the cast is wrong since sizeof(long)==4. Afaik,
io_uringstill builds on 32-bit archs.If you use (unsigned long long), it will be 64 bit anywhere.
Ah thanks, I'll make that edit.
Quoted 11 lines
io_uring/io_uring.c static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · line 2174
Quoted 6 lines
io_uring/io_uring.c +1
-1@@ -2171,7 +2171,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,/* req is partially pre-initialised, see io_preinit_req() */ req->opcode = opcode =READ_ONCE (sqe->opcode);/* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags = sqe_flags =READ_ONCE (sqe->flags); + sqe_flags =READ_ONCE (sqe->flags);Did you consider that
READ_ONCEwon't protect from load tearing the userspace value in 32-bit architectures? It builds silently, though, and I suspect it is mostly fine in the current code, but might become a bug eventually.
sqe->flags is just a byte, so no tearing is possible here. The only thing that changed type is req->flags.
Signature · 1 line
Jens Axboe
Re: [PATCH 1/6] io_uring: expand main struct io_kiocb flags to 64-bits
Quoted 32 lines · Jens Axboe
Quoted 12 lines · Gabriel Krisman Bertazi
include/trace/events/io_uring.h TRACE_EVENT(io_uring_queue_async_work, · line 172
Quoted 6 lines · Jens Axboe
include/trace/events/io_uring.h +2
-3-TP_printk ("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%x, %s queue, work %p" , + TP_printk("ring %p, request %p, user_data 0x%llx, opcode %s, flags 0x%lx, %s queue, work %p", __entry->ctx, __entry->req, __entry->user_data, -__get_str (op_str), - __entry->flags, __entry->rw ?"hashed" :"normal" , __entry->work) + __get_str(op_str), (long) __entry->flags,Hi Jens,
Quoted 4 lines
Minor, but on 32-bit kernel the cast is wrong since sizeof(long)==4. Afaik,
io_uringstill builds on 32-bit archs.If you use (unsigned long long), it will be 64 bit anywhere.
Ah thanks, I'll make that edit.
Quoted 11 lines
io_uring/io_uring.c static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · line 2174
Quoted 6 lines
io_uring/io_uring.c +1
-1@@ -2171,7 +2171,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,/* req is partially pre-initialised, see io_preinit_req() */ req->opcode = opcode =READ_ONCE (sqe->opcode);/* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags = sqe_flags =READ_ONCE (sqe->flags); + sqe_flags =READ_ONCE (sqe->flags);Did you consider that
READ_ONCEwon't protect from load tearing the userspace value in 32-bit architectures? It builds silently, though, and I suspect it is mostly fine in the current code, but might become a bug eventually.
Quoted 2 lines
sqe->flagsis just a byte, so no tearing is possible here. The only thing that changed type isreq->flags.
You're right, of course. I confused the source of the read with struct io_kiocb.
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerPATCH v2 2/6 io_uring/cancel: don't default to setting req->work.cancel_seq
In reply to Jens AxboeJust leave it unset by default, avoiding dipping into the last cacheline (which is otherwise untouched) for the fast path of using poll to drive networked traffic. Add a flag that tells us if the sequence is valid or not, and then we can defer actually assigning the flag and sequence until someone runs cancelations.
--- include/linux/io_uring_types.h | 3 +++ io_uring/cancel.c | 3 +-- io_uring/cancel.h | 10 ++++++++++ io_uring/io_uring.c | 1 - io_uring/poll.c | 6 +----- 5 files changed, 15 insertions(+), 8 deletions(-)
include/linux/io_uring_types.h +3 -0
@@ -463,6 +463,7 @@ enum {REQ_F_SUPPORT_NOWAIT_BIT ,REQ_F_ISREG_BIT ,REQ_F_POLL_NO_LAZY_BIT , +REQ_F_CANCEL_SEQ_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -535,6 +536,8 @@ enum {REQ_F_HASH_LOCKED =IO_REQ_FLAG (REQ_F_HASH_LOCKED_BIT ),/* don't use lazy poll wake for this request */ REQ_F_POLL_NO_LAZY =IO_REQ_FLAG (REQ_F_POLL_NO_LAZY_BIT ), +/* cancel sequence is set and valid */ + REQ_F_CANCEL_SEQ = IO_REQ_FLAG(REQ_F_CANCEL_SEQ_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);
io_uring/cancel.c +1 -2
@@ -58,9 +58,8 @@ bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)return false;if (cd->flags &IORING_ASYNC_CANCEL_ALL ) { check_seq: -if (cd->seq == req->work.cancel_seq) +if (io_cancel_match_sequence (req, cd->seq))return false; - req->work.cancel_seq = cd->seq; }return true;
io_uring/cancel.h +10 -0
@@ -25,4 +25,14 @@ void init_hash_table(struct io_hash_table *table, unsigned size);int io_sync_cancel (struct io_ring_ctx *ctx,void __user *arg);bool io_cancel_req_match (struct io_kiocb *req,struct io_cancel_data *cd); +static inline bool io_cancel_match_sequence (struct io_kiocb *req,int sequence) +{ +if ((req->flags &REQ_F_CANCEL_SEQ ) && sequence == req->work.cancel_seq) +return true; + + req->flags |=REQ_F_CANCEL_SEQ ; + req->work.cancel_seq = sequence; +return false; +} +#endif
io_uring/io_uring.c +0 -1
@@ -463,7 +463,6 @@ static void io_prep_async_work(struct io_kiocb *req) req->work.list.next =NULL ; req->work.flags =0 ; - req->work.cancel_seq =atomic_read (&ctx->cancel_seq);if (req->flags &REQ_F_FORCE_ASYNC ) req->work.flags |=IO_WQ_WORK_CONCURRENT ;
io_uring/poll.c +1 -5
@@ -588,10 +588,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, struct io_poll_table *ipt, __poll_t mask, unsigned issue_flags) { - struct io_ring_ctx *ctx = req->ctx; - INIT_HLIST_NODE(&req->hash_node); - req->work.cancel_seq =atomic_read (&ctx->cancel_seq);io_init_poll_iocb (poll, mask); poll->file = req->file; req->apoll_events = poll->events;
@@ -818,9 +815,8 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,if (poll_only && req->opcode !=IORING_OP_POLL_ADD )continue ;if (cd->flags &IORING_ASYNC_CANCEL_ALL ) { -if (cd->seq == req->work.cancel_seq) +if (io_cancel_match_sequence (req, cd->seq))continue ; - req->work.cancel_seq = cd->seq; } *out_bucket = hb;return req;
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v2 3/6 io_uring: add io_file_can_poll() helper
In reply to Jens AxboeThis adds a flag to avoid dipping dereferencing file and then f_op to figure out if the file has a poll handler defined or not. We generally call this at least twice for networked workloads, and if using ring provided buffers, we do it on every buffer selection. Particularly the latter is troublesome, as it's otherwise a very fast operation.
--- include/linux/io_uring_types.h | 3 +++ io_uring/io_uring.c | 2 +- io_uring/io_uring.h | 12 ++++++++++++ io_uring/kbuf.c | 2 +- io_uring/poll.c | 2 +- io_uring/rw.c | 6 +++--- 6 files changed, 21 insertions(+), 6 deletions(-)
include/linux/io_uring_types.h +3 -0
@@ -464,6 +464,7 @@ enum {REQ_F_ISREG_BIT ,REQ_F_POLL_NO_LAZY_BIT ,REQ_F_CANCEL_SEQ_BIT , +REQ_F_CAN_POLL_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -538,6 +539,8 @@ enum {REQ_F_POLL_NO_LAZY =IO_REQ_FLAG (REQ_F_POLL_NO_LAZY_BIT ),/* cancel sequence is set and valid */ REQ_F_CANCEL_SEQ =IO_REQ_FLAG (REQ_F_CANCEL_SEQ_BIT ), +/* file is pollable */ + REQ_F_CAN_POLL = IO_REQ_FLAG(REQ_F_CAN_POLL_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);
io_uring/io_uring.c +1 -1
@@ -1968,7 +1968,7 @@ void io_wq_submit_work(struct io_wq_work *work)if (req->flags &REQ_F_FORCE_ASYNC ) {bool opcode_poll = def->pollin || def->pollout; -if (opcode_poll &&file_can_poll (req->file)) { +if (opcode_poll &&io_file_can_poll (req)) { needs_poll = true; issue_flags |=IO_URING_F_NONBLOCK ; }
io_uring/io_uring.h +12 -0
@@ -5,6 +5,7 @@#include <linux/lockdep.h> #include <linux/resume_user_mode.h> #include <linux/kasan.h> +#include <linux/poll.h> #include <linux/io_uring_types.h> #include <uapi/linux/eventpoll.h> #include "io-wq.h"
@@ -398,4 +399,15 @@ static inline size_t uring_sqe_size(struct io_ring_ctx *ctx)return 2 *sizeof (struct io_uring_sqe ); return sizeof(struct io_uring_sqe); } + +static inline bool io_file_can_poll (struct io_kiocb *req) +{ +if (req->flags &REQ_F_CAN_POLL ) +return true; +if (file_can_poll (req->file)) { + req->flags |=REQ_F_CAN_POLL ; +return true; + } +return false; +}#endif
io_uring/kbuf.c +1 -1
@@ -180,7 +180,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->buf_list = bl; req->buf_index = buf->bid; -if (issue_flags &IO_URING_F_UNLOCKED || !file_can_poll (req->file)) { +if (issue_flags &IO_URING_F_UNLOCKED || !io_file_can_poll (req)) {/* * If we came in unlocked, we have no choice but to consume the * buffer here, otherwise nothing ensures that the buffer won't
io_uring/poll.c +1 -1
@@ -724,7 +724,7 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)if (!def->pollin && !def->pollout)return IO_APOLL_ABORTED ; -if (!file_can_poll (req->file)) +if (!io_file_can_poll (req))return IO_APOLL_ABORTED ;if (!(req->flags &REQ_F_APOLL_MULTISHOT )) mask |=EPOLLONESHOT ;
io_uring/rw.c +3 -3
@@ -682,7 +682,7 @@ static bool io_rw_should_retry(struct io_kiocb *req)* just use poll if we can, and don't attempt if the fs doesn't * support callback based unlocks */ -if (file_can_poll (req->file) || !(req->file->f_mode &FMODE_BUF_RASYNC )) +if (io_file_can_poll (req) || !(req->file->f_mode &FMODE_BUF_RASYNC ))return false; wait->wait.func = io_async_buf_func;
@@ -831,7 +831,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)* If we can poll, just do that. For a vectored read, we'll * need to copy state first. */ -if (file_can_poll (req->file) && !io_issue_defs[req->opcode].vectored) +if (io_file_can_poll (req) && !io_issue_defs[req->opcode].vectored)return -EAGAIN ;/* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags &IORING_SETUP_IOPOLL ))
@@ -930,7 +930,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)/* * Multishot MUST be used on a pollable file */ -if (!file_can_poll (req->file)) +if (!io_file_can_poll (req))return -EBADFD ; ret =__io_read (req, issue_flags);
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v2 4/6 io_uring: mark the need to lock/unlock the ring as unlikely
In reply to Jens AxboeAny of the fast paths will already have this locked, this helper only exists to deal with io-wq invoking request issue where we do not have the ctx->uring_lock held already. This means that any common or fast path will already have this locked, mark it as such.
--- io_uring/io_uring.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/io_uring.h +2 -2
@@ -208,7 +208,7 @@ static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx, unsigned issue_flags) {lockdep_assert_held (&ctx->uring_lock); -if (issue_flags &IO_URING_F_UNLOCKED ) +if (unlikely (issue_flags &IO_URING_F_UNLOCKED ))mutex_unlock (&ctx->uring_lock); }
@@ -221,7 +221,7 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,* The only exception is when we've detached the request and issue it * from an async worker thread, grab the lock for that case. */ -if (issue_flags &IO_URING_F_UNLOCKED ) +if (unlikely (issue_flags &IO_URING_F_UNLOCKED ))mutex_lock (&ctx->uring_lock);lockdep_assert_held (&ctx->uring_lock); }
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v2 5/6 io_uring: cleanup io_req_complete_post()
In reply to Jens AxboeMove the ctx declaration and assignment up to be generally available in the function, as we use req->ctx at the top anyway.
--- io_uring/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
io_uring/io_uring.c +4 -4
@@ -1024,15 +1024,15 @@ static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)void io_req_complete_post (struct io_kiocb *req,unsigned issue_flags) { -if (req->ctx->task_complete && req->ctx->submitter_task != current) { +struct io_ring_ctx *ctx = req->ctx; + +if (ctx->task_complete && ctx->submitter_task != current) { req->io_task_work.func = io_req_task_complete;io_req_task_work_add (req); }else if (!(issue_flags &IO_URING_F_UNLOCKED ) || - !(req->ctx->flags &IORING_SETUP_IOPOLL )) { + !(ctx->flags &IORING_SETUP_IOPOLL )) {__io_req_complete_post (req, issue_flags); }else { -struct io_ring_ctx *ctx = req->ctx; -mutex_lock (&ctx->uring_lock);__io_req_complete_post (req, issue_flags & ~IO_URING_F_UNLOCKED );mutex_unlock (&ctx->uring_lock);
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v2 6/6 io_uring/rw: remove dead file == NULL check
In reply to Jens AxboeAny read/write opcode has needs_file == true, which means that we would've failed the request long before reaching the issue stage if we didn't successfully assign a file. This check has been dead forever, and is really a leftover from generic code.
--- io_uring/rw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/rw.c +1 -1
@@ -721,7 +721,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode)struct file *file = req->file;int ret; -if (unlikely (!file || !(file->f_mode & mode))) +if (unlikely (!(file->f_mode & mode)))return -EBADF ;if (!(req->flags &REQ_F_FIXED_FILE ))
Signature · 1 line
2.43.0