Replace io_ring_ctx bitfields with flags
Jens Axboe
maintainercover Replace io_ring_ctx bitfields with flags
Hi,
No functional changes here, just gets rid of the bitfields in ctx and replaces then with a single int_flags member and a set of flags. This makes it more obvious that these are manipulated and checked together.
include/linux/io_uring_types.h | 32 +++++++------ io_uring/eventfd.c | 4 +- io_uring/io_uring.c | 82 +++++++++++++++++----------------- io_uring/io_uring.h | 10 +++-- io_uring/msg_ring.c | 2 +- io_uring/register.c | 8 ++-- io_uring/rsrc.c | 8 ++-- io_uring/tctx.c | 2 +- io_uring/timeout.c | 4 +- io_uring/tw.c | 2 +- 10 files changed, 81 insertions(+), 73 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/2 io_uring: switch struct io_ring_ctx internal bitfields to flags
Bitfields cannot be set and checked atomically, and this makes it more clear that these are indeed in shared storage and must be checked and set in a sane fashion. This is in preparation for annotating a few of the known racy, but harmless, flags checking.
No intended functional changes in this patch.
--- include/linux/io_uring_types.h | 32 +++++++------ io_uring/eventfd.c | 4 +- io_uring/io_uring.c | 82 +++++++++++++++++----------------- io_uring/io_uring.h | 9 ++-- io_uring/msg_ring.c | 2 +- io_uring/register.c | 8 ++-- io_uring/rsrc.c | 8 ++-- io_uring/tctx.c | 2 +- io_uring/timeout.c | 4 +- io_uring/tw.c | 2 +- 10 files changed, 80 insertions(+), 73 deletions(-)
include/linux/io_uring_types.h +18 -14
@@ -268,24 +268,28 @@ struct io_alloc_cache {unsigned int init_clear; }; +enum { +IO_RING_F_DRAIN_NEXT =BIT (0 ), +IO_RING_F_OP_RESTRICTED =BIT (1 ), +IO_RING_F_REG_RESTRICTED =BIT (2 ), +IO_RING_F_OFF_TIMEOUT_USED =BIT (3 ), +IO_RING_F_DRAIN_ACTIVE =BIT (4 ), +IO_RING_F_HAS_EVFD =BIT (5 ), +/* all CQEs should be posted only by the submitter task */ +IO_RING_F_TASK_COMPLETE =BIT (6 ), +IO_RING_F_LOCKLESS_CQ =BIT (7 ), +IO_RING_F_SYSCALL_IOPOLL =BIT (8 ), +IO_RING_F_POLL_ACTIVATED =BIT (9 ), +IO_RING_F_DRAIN_DISABLED =BIT (10 ), +IO_RING_F_COMPAT =BIT (11 ), +IO_RING_F_IOWQ_LIMITS_SET =BIT (12 ), +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {unsigned int flags; -unsigned int drain_next:1 ; -unsigned int op_restricted:1 ; -unsigned int reg_restricted:1 ; -unsigned int off_timeout_used:1 ; -unsigned int drain_active:1 ; -unsigned int has_evfd:1 ; -/* all CQEs should be posted only by the submitter task */ -unsigned int task_complete:1 ; -unsigned int lockless_cq:1 ; -unsigned int syscall_iopoll:1 ; -unsigned int poll_activated:1 ; -unsigned int drain_disabled:1 ; -unsigned int compat:1 ; -unsigned int iowq_limits_set :1 ; +unsigned int int_flags;struct task_struct *submitter_task;struct io_rings *rings;
io_uring/eventfd.c +2 -2
@@ -148,7 +148,7 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,spin_unlock (&ctx->completion_lock); ev_fd->eventfd_async = eventfd_async; - ctx->has_evfd = true; + ctx->int_flags |=IO_RING_F_HAS_EVFD ;refcount_set (&ev_fd->refs,1 );atomic_set (&ev_fd->ops,0 );rcu_assign_pointer (ctx->io_ev_fd, ev_fd);
@@ -162,7 +162,7 @@ int io_eventfd_unregister(struct io_ring_ctx *ctx) ev_fd =rcu_dereference_protected (ctx->io_ev_fd,lockdep_is_held (&ctx->uring_lock));if (ev_fd) { - ctx->has_evfd = false; + ctx->int_flags &= ~IO_RING_F_HAS_EVFD ;rcu_assign_pointer (ctx->io_ev_fd,NULL );io_eventfd_put (ev_fd);return 0 ;
io_uring/io_uring.c +42 -40
242 lines not on this page · raw message
io_uring/io_uring.h +5 -4
@@ -211,7 +211,7 @@ static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)if (ctx->flags &IORING_SETUP_IOPOLL ) {lockdep_assert_held (&ctx->uring_lock); - }else if (!ctx->task_complete) { + }else if (!(ctx->int_flags &IO_RING_F_TASK_COMPLETE )) {lockdep_assert_held (&ctx->completion_lock); }else if (ctx->submitter_task) {/*
@@ -228,7 +228,7 @@ static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)static inline bool io_is_compat (struct io_ring_ctx *ctx) { -return IS_ENABLED (CONFIG_COMPAT ) &&unlikely (ctx->compat); +return IS_ENABLED (CONFIG_COMPAT ) &&unlikely (ctx->int_flags &IO_RING_F_COMPAT ); }static inline void io_submit_flush_completions (struct io_ring_ctx *ctx)
@@ -472,8 +472,9 @@ static inline void io_req_complete_defer(struct io_kiocb *req)static inline void io_commit_cqring_flush (struct io_ring_ctx *ctx) { -if (unlikely (ctx->off_timeout_used || - ctx->has_evfd || ctx->poll_activated)) +if (unlikely (ctx->int_flags & (IO_RING_F_OFF_TIMEOUT_USED | +IO_RING_F_HAS_EVFD | +IO_RING_F_POLL_ACTIVATED )))__io_commit_cqring_flush (ctx); }
io_uring/msg_ring.c +1 -1
@@ -67,7 +67,7 @@ void io_msg_ring_cleanup(struct io_kiocb *req)static inline bool io_msg_need_remote (struct io_ring_ctx *target_ctx) { -return target_ctx->task_complete; +return target_ctx->int_flags &IO_RING_F_TASK_COMPLETE ; }static void io_msg_tw_complete (struct io_tw_req tw_req,io_tw_token_t tw)
io_uring/register.c +4 -4
@@ -184,9 +184,9 @@ static __cold int io_register_restrictions(struct io_ring_ctx *ctx,return ret; }if (ctx->restrictions.op_registered) - ctx->op_restricted =1 ; + ctx->int_flags |=IO_RING_F_OP_RESTRICTED ;if (ctx->restrictions.reg_registered) - ctx->reg_restricted =1 ; + ctx->int_flags |=IO_RING_F_REG_RESTRICTED ;return 0 ; }
@@ -384,7 +384,7 @@ static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,for (i =0 ; i <ARRAY_SIZE (new_count); i++)if (new_count[i]) ctx->iowq_limits[i] = new_count[i]; - ctx->iowq_limits_set = true; + ctx->int_flags |=IO_RING_F_IOWQ_LIMITS_SET ;if (tctx && tctx->io_wq) { ret =io_wq_max_workers (tctx->io_wq, new_count);
@@ -725,7 +725,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,if (ctx->submitter_task && ctx->submitter_task != current)return -EEXIST ; -if (ctx->reg_restricted && !(ctx->flags &IORING_SETUP_R_DISABLED )) { +if ((ctx->int_flags &IO_RING_F_REG_RESTRICTED ) && !(ctx->flags &IORING_SETUP_R_DISABLED )) { opcode =array_index_nospec (opcode,IORING_REGISTER_LAST );if (!test_bit (opcode, ctx->restrictions.register_op))return -EACCES ;
io_uring/rsrc.c +4 -4
@@ -295,7 +295,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,u64 tag =0 ; uvec =u64_to_user_ptr (user_data); - iov =iovec_from_user (uvec,1 ,1 , &fast_iov, ctx->compat); + iov =iovec_from_user (uvec,1 ,1 , &fast_iov,io_is_compat (ctx));if (IS_ERR (iov)) { err =PTR_ERR (iov);break ;
@@ -319,7 +319,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, i =array_index_nospec (up->offset + done, ctx->buf_table.nr);io_reset_rsrc_node (ctx, &ctx->buf_table, i); ctx->buf_table.nodes[i] = node; -if (ctx->compat) +if (io_is_compat (ctx)) user_data +=sizeof (struct compat_iovec );else user_data +=sizeof (struct iovec );
@@ -883,12 +883,12 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,if (arg) { uvec = (struct iovec __user *) arg; - iov =iovec_from_user (uvec,1 ,1 , &fast_iov, ctx->compat); + iov =iovec_from_user (uvec,1 ,1 , &fast_iov,io_is_compat (ctx));if (IS_ERR (iov)) { ret =PTR_ERR (iov);break ; } -if (ctx->compat) +if (io_is_compat (ctx)) arg +=sizeof (struct compat_iovec );else arg +=sizeof (struct iovec );
io_uring/tctx.c +1 -1
@@ -121,7 +121,7 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)return ret; tctx = current->io_uring; -if (ctx->iowq_limits_set) { +if (ctx->int_flags &IO_RING_F_IOWQ_LIMITS_SET ) {unsigned int limits[2 ] = { ctx->iowq_limits[0 ], ctx->iowq_limits[1 ], };
io_uring/timeout.c +2 -2
@@ -566,8 +566,8 @@ static int __io_timeout_prep(struct io_kiocb *req,INIT_LIST_HEAD (&timeout->list); timeout->off = off; -if (unlikely (off && !req->ctx->off_timeout_used)) - req->ctx->off_timeout_used = true; +if (unlikely (off && !(req->ctx->int_flags &IO_RING_F_OFF_TIMEOUT_USED ))) + req->ctx->int_flags |=IO_RING_F_OFF_TIMEOUT_USED ;/* * for multishot reqs w/ fixed nr of repeats, repeats tracks the * remaining nr
io_uring/tw.c +1 -1
@@ -222,7 +222,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)if (!head) {io_ctx_mark_taskrun (ctx); -if (ctx->has_evfd) +if (ctx->int_flags &IO_RING_F_HAS_EVFD )io_eventfd_signal (ctx, false); }
Signature · 1 line
2.53.0
Re: [PATCH 1/2] io_uring: switch struct io_ring_ctx internal bitfields to flags
Quoted 55 lines · Jens Axboe
Bitfields cannot be set and checked atomically, and this makes it more clear that these are indeed in shared storage and must be checked and set in a sane fashion. This is in preparation for annotating a few of the known racy, but harmless, flags checking.
No intended functional changes in this patch.
Signed-off-by Jens Axboe <[email protected]>--- include/linux/io_uring_types.h | 32 +++++++------ io_uring/eventfd.c | 4 +- io_uring/io_uring.c | 82 +++++++++++++++++----------------- io_uring/io_uring.h | 9 ++-- io_uring/msg_ring.c | 2 +- io_uring/register.c | 8 ++-- io_uring/rsrc.c | 8 ++-- io_uring/tctx.c | 2 +- io_uring/timeout.c | 4 +- io_uring/tw.c | 2 +- 10 files changed, 80 insertions(+), 73 deletions(-)include/linux/io_uring_types.h +17
-5@@ -268,24 +268,28 @@ struct io_alloc_cache {unsigned int init_clear; }; +enum { +IO_RING_F_DRAIN_NEXT =BIT (0 ), +IO_RING_F_OP_RESTRICTED =BIT (1 ), +IO_RING_F_REG_RESTRICTED =BIT (2 ), +IO_RING_F_OFF_TIMEOUT_USED =BIT (3 ), +IO_RING_F_DRAIN_ACTIVE =BIT (4 ), +IO_RING_F_HAS_EVFD =BIT (5 ), +/* all CQEs should be posted only by the submitter task */ +IO_RING_F_TASK_COMPLETE =BIT (6 ), +IO_RING_F_LOCKLESS_CQ =BIT (7 ), +IO_RING_F_SYSCALL_IOPOLL =BIT (8 ), +IO_RING_F_POLL_ACTIVATED =BIT (9 ), +IO_RING_F_DRAIN_DISABLED =BIT (10 ), +IO_RING_F_COMPAT =BIT (11 ), +IO_RING_F_IOWQ_LIMITS_SET =BIT (12 ), +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {unsigned int flags; -unsigned int drain_next:1 ; -unsigned int op_restricted:1 ; -unsigned int reg_restricted:1 ; -unsigned int off_timeout_used:1 ; -unsigned int drain_active:1 ;
include/linux/io_uring_types.h struct io_alloc_cache { · line 292
Quoted 10 lines
include/linux/io_uring_types.h +1
-9- unsigned int has_evfd: 1; -/* all CQEs should be posted only by the submitter task */ - unsigned int task_complete: 1; - unsigned int lockless_cq: 1; - unsigned int syscall_iopoll: 1; - unsigned int poll_activated: 1; - unsigned int drain_disabled: 1; - unsigned int compat: 1; - unsigned int iowq_limits_set : 1; +unsigned int int_flags;
Jens,
What does the int prefix means in this context?
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerRe: [PATCH 1/2] io_uring: switch struct io_ring_ctx internal bitfields to flags
Quoted 70 lines · Gabriel Krisman Bertazi
Quoted 55 lines · Jens Axboe
Bitfields cannot be set and checked atomically, and this makes it more clear that these are indeed in shared storage and must be checked and set in a sane fashion. This is in preparation for annotating a few of the known racy, but harmless, flags checking.
No intended functional changes in this patch.
Signed-off-by Jens Axboe <[email protected]>--- include/linux/io_uring_types.h | 32 +++++++------ io_uring/eventfd.c | 4 +- io_uring/io_uring.c | 82 +++++++++++++++++----------------- io_uring/io_uring.h | 9 ++-- io_uring/msg_ring.c | 2 +- io_uring/register.c | 8 ++-- io_uring/rsrc.c | 8 ++-- io_uring/tctx.c | 2 +- io_uring/timeout.c | 4 +- io_uring/tw.c | 2 +- 10 files changed, 80 insertions(+), 73 deletions(-)include/linux/io_uring_types.h +17
-5@@ -268,24 +268,28 @@ struct io_alloc_cache {unsigned int init_clear; }; +enum { +IO_RING_F_DRAIN_NEXT =BIT (0 ), +IO_RING_F_OP_RESTRICTED =BIT (1 ), +IO_RING_F_REG_RESTRICTED =BIT (2 ), +IO_RING_F_OFF_TIMEOUT_USED =BIT (3 ), +IO_RING_F_DRAIN_ACTIVE =BIT (4 ), +IO_RING_F_HAS_EVFD =BIT (5 ), +/* all CQEs should be posted only by the submitter task */ +IO_RING_F_TASK_COMPLETE =BIT (6 ), +IO_RING_F_LOCKLESS_CQ =BIT (7 ), +IO_RING_F_SYSCALL_IOPOLL =BIT (8 ), +IO_RING_F_POLL_ACTIVATED =BIT (9 ), +IO_RING_F_DRAIN_DISABLED =BIT (10 ), +IO_RING_F_COMPAT =BIT (11 ), +IO_RING_F_IOWQ_LIMITS_SET =BIT (12 ), +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {unsigned int flags; -unsigned int drain_next:1 ; -unsigned int op_restricted:1 ; -unsigned int reg_restricted:1 ; -unsigned int off_timeout_used:1 ; -unsigned int drain_active:1 ;include/linux/io_uring_types.h struct io_alloc_cache { · line 292
Quoted 10 lines
include/linux/io_uring_types.h +1
-9- unsigned int has_evfd: 1; -/* all CQEs should be posted only by the submitter task */ - unsigned int task_complete: 1; - unsigned int lockless_cq: 1; - unsigned int syscall_iopoll: 1; - unsigned int poll_activated: 1; - unsigned int drain_disabled: 1; - unsigned int compat: 1; - unsigned int iowq_limits_set : 1; +unsigned int int_flags;Jens,
Quoted 1 line
What does the int prefix means in this context?
It's just short for 'internal'.
Signature · 1 line
Jens Axboe
Gabriel Krisman Bertazi
Reviewed-byRe: [PATCH 1/2] io_uring: switch struct io_ring_ctx internal bitfields to flags
Quoted 72 lines · Jens Axboe
Quoted 70 lines · Gabriel Krisman Bertazi
Quoted 55 lines · Jens Axboe
Bitfields cannot be set and checked atomically, and this makes it more clear that these are indeed in shared storage and must be checked and set in a sane fashion. This is in preparation for annotating a few of the known racy, but harmless, flags checking.
No intended functional changes in this patch.
Signed-off-by Jens Axboe <[email protected]>--- include/linux/io_uring_types.h | 32 +++++++------ io_uring/eventfd.c | 4 +- io_uring/io_uring.c | 82 +++++++++++++++++----------------- io_uring/io_uring.h | 9 ++-- io_uring/msg_ring.c | 2 +- io_uring/register.c | 8 ++-- io_uring/rsrc.c | 8 ++-- io_uring/tctx.c | 2 +- io_uring/timeout.c | 4 +- io_uring/tw.c | 2 +- 10 files changed, 80 insertions(+), 73 deletions(-)include/linux/io_uring_types.h +17
-5@@ -268,24 +268,28 @@ struct io_alloc_cache {unsigned int init_clear; }; +enum { +IO_RING_F_DRAIN_NEXT =BIT (0 ), +IO_RING_F_OP_RESTRICTED =BIT (1 ), +IO_RING_F_REG_RESTRICTED =BIT (2 ), +IO_RING_F_OFF_TIMEOUT_USED =BIT (3 ), +IO_RING_F_DRAIN_ACTIVE =BIT (4 ), +IO_RING_F_HAS_EVFD =BIT (5 ), +/* all CQEs should be posted only by the submitter task */ +IO_RING_F_TASK_COMPLETE =BIT (6 ), +IO_RING_F_LOCKLESS_CQ =BIT (7 ), +IO_RING_F_SYSCALL_IOPOLL =BIT (8 ), +IO_RING_F_POLL_ACTIVATED =BIT (9 ), +IO_RING_F_DRAIN_DISABLED =BIT (10 ), +IO_RING_F_COMPAT =BIT (11 ), +IO_RING_F_IOWQ_LIMITS_SET =BIT (12 ), +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {unsigned int flags; -unsigned int drain_next:1 ; -unsigned int op_restricted:1 ; -unsigned int reg_restricted:1 ; -unsigned int off_timeout_used:1 ; -unsigned int drain_active:1 ;include/linux/io_uring_types.h struct io_alloc_cache { · line 292
Quoted 10 lines
include/linux/io_uring_types.h +1
-9- unsigned int has_evfd: 1; -/* all CQEs should be posted only by the submitter task */ - unsigned int task_complete: 1; - unsigned int lockless_cq: 1; - unsigned int syscall_iopoll: 1; - unsigned int poll_activated: 1; - unsigned int drain_disabled: 1; - unsigned int compat: 1; - unsigned int iowq_limits_set : 1; +unsigned int int_flags;Jens,
Quoted 3 lines
Quoted 1 line
What does the int prefix means in this context?
It's just short for 'internal'.
ack. Perhaps a comment indicating this is internal would be useful. Otherwise,
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerRe: [PATCH 1/2] io_uring: switch struct io_ring_ctx internal bitfields to flags
Quoted 74 lines · Gabriel Krisman Bertazi
Quoted 72 lines · Jens Axboe
Quoted 70 lines · Gabriel Krisman Bertazi
Quoted 55 lines · Jens Axboe
Bitfields cannot be set and checked atomically, and this makes it more clear that these are indeed in shared storage and must be checked and set in a sane fashion. This is in preparation for annotating a few of the known racy, but harmless, flags checking.
No intended functional changes in this patch.
Signed-off-by Jens Axboe <[email protected]>--- include/linux/io_uring_types.h | 32 +++++++------ io_uring/eventfd.c | 4 +- io_uring/io_uring.c | 82 +++++++++++++++++----------------- io_uring/io_uring.h | 9 ++-- io_uring/msg_ring.c | 2 +- io_uring/register.c | 8 ++-- io_uring/rsrc.c | 8 ++-- io_uring/tctx.c | 2 +- io_uring/timeout.c | 4 +- io_uring/tw.c | 2 +- 10 files changed, 80 insertions(+), 73 deletions(-)include/linux/io_uring_types.h +17
-5@@ -268,24 +268,28 @@ struct io_alloc_cache {unsigned int init_clear; }; +enum { +IO_RING_F_DRAIN_NEXT =BIT (0 ), +IO_RING_F_OP_RESTRICTED =BIT (1 ), +IO_RING_F_REG_RESTRICTED =BIT (2 ), +IO_RING_F_OFF_TIMEOUT_USED =BIT (3 ), +IO_RING_F_DRAIN_ACTIVE =BIT (4 ), +IO_RING_F_HAS_EVFD =BIT (5 ), +/* all CQEs should be posted only by the submitter task */ +IO_RING_F_TASK_COMPLETE =BIT (6 ), +IO_RING_F_LOCKLESS_CQ =BIT (7 ), +IO_RING_F_SYSCALL_IOPOLL =BIT (8 ), +IO_RING_F_POLL_ACTIVATED =BIT (9 ), +IO_RING_F_DRAIN_DISABLED =BIT (10 ), +IO_RING_F_COMPAT =BIT (11 ), +IO_RING_F_IOWQ_LIMITS_SET =BIT (12 ), +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {unsigned int flags; -unsigned int drain_next:1 ; -unsigned int op_restricted:1 ; -unsigned int reg_restricted:1 ; -unsigned int off_timeout_used:1 ; -unsigned int drain_active:1 ;include/linux/io_uring_types.h struct io_alloc_cache { · line 292
Quoted 10 lines
include/linux/io_uring_types.h +1
-9- unsigned int has_evfd: 1; -/* all CQEs should be posted only by the submitter task */ - unsigned int task_complete: 1; - unsigned int lockless_cq: 1; - unsigned int syscall_iopoll: 1; - unsigned int poll_activated: 1; - unsigned int drain_disabled: 1; - unsigned int compat: 1; - unsigned int iowq_limits_set : 1; +unsigned int int_flags;Jens,
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
What does the int prefix means in this context?
It's just short for 'internal'.
ack. Perhaps a comment indicating this is internal would be useful. Otherwise,
Reviewed-by Gabriel Krisman Bertazi <[email protected]>
Added the comment and your reviewed-by, thanks!
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 2/2 io_uring: mark known and harmless racy ctx->int_flags uses
In reply to Jens AxboeThere are a few of these, where flags are read outside of the uring_lock, yet it's harmless to race on them.
--- io_uring/io_uring.c | 2 +- io_uring/io_uring.h | 7 ++++--- io_uring/tw.c | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-)
io_uring/io_uring.c +1 -1
@@ -2242,7 +2242,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait)struct io_ring_ctx *ctx = file->private_data;__poll_t mask =0 ; -if (unlikely (!(ctx->int_flags &IO_RING_F_POLL_ACTIVATED ))) +if (unlikely (!(data_race (ctx->int_flags) &IO_RING_F_POLL_ACTIVATED )))io_activate_pollwq (ctx);/* * provides mb() which pairs with barrier from wq_has_sleeper
io_uring/io_uring.h +4 -3
@@ -470,11 +470,12 @@ static inline void io_req_complete_defer(struct io_kiocb *req)wq_list_add_tail (&req->comp_list, &state->compl_reqs); } +#define SHOULD_FLUSH_MASK (IO_RING_F_OFF_TIMEOUT_USED | \ + IO_RING_F_HAS_EVFD | IO_RING_F_POLL_ACTIVATED) +static inline void io_commit_cqring_flush (struct io_ring_ctx *ctx) { -if (unlikely (ctx->int_flags & (IO_RING_F_OFF_TIMEOUT_USED | -IO_RING_F_HAS_EVFD | -IO_RING_F_POLL_ACTIVATED ))) +if (unlikely (data_race (ctx->int_flags) &SHOULD_FLUSH_MASK ))__io_commit_cqring_flush (ctx); }
io_uring/tw.c +1 -1
@@ -222,7 +222,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)if (!head) {io_ctx_mark_taskrun (ctx); -if (ctx->int_flags &IO_RING_F_HAS_EVFD ) +if (data_race (ctx->int_flags) &IO_RING_F_HAS_EVFD )io_eventfd_signal (ctx, false); }
Signature · 1 line
2.53.0