Move eventfd cq tracking into io_ev_fd
Jens Axboe
maintainercover Move eventfd cq tracking into io_ev_fd
Hi,
For some reason this ended up being a series of 6 patches, when the goal was really just to move evfd_last_cq_tail out of io_ring_ctx and into struct io_ev_fd, where it belongs. But this slowly builds to that goal, and the final patch does the move unceremoniously.
Patches are on top of current -git with for-6.12/io_uring pulled in.
io_uring/eventfd.c | 137 ++++++++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 45 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/6 io_uring/eventfd: abstract out ev_fd put helper
We call this in two spot, have a helper for it. In preparation for extending this part.
--- io_uring/eventfd.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)
io_uring/eventfd.c +8 -4
@@ -41,6 +41,12 @@ static void io_eventfd_do_signal(struct rcu_head *rcu)io_eventfd_free (rcu); } +static void io_eventfd_put (struct io_ev_fd *ev_fd) +{ +if (refcount_dec_and_test (&ev_fd->refs)) +call_rcu (&ev_fd->rcu, io_eventfd_free); +} +void io_eventfd_signal (struct io_ring_ctx *ctx) {struct io_ev_fd *ev_fd =NULL ;
@@ -77,8 +83,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) } } out: -if (refcount_dec_and_test (&ev_fd->refs)) -call_rcu (&ev_fd->rcu, io_eventfd_free); +io_eventfd_put (ev_fd); }void io_eventfd_flush_signal (struct io_ring_ctx *ctx)
@@ -152,8 +157,7 @@ int io_eventfd_unregister(struct io_ring_ctx *ctx)if (ev_fd) { ctx->has_evfd = false;rcu_assign_pointer (ctx->io_ev_fd,NULL ); -if (refcount_dec_and_test (&ev_fd->refs)) -call_rcu (&ev_fd->rcu, io_eventfd_free); +io_eventfd_put (ev_fd);return 0 ; }
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 2/6 io_uring/eventfd: check for the need to async notifier earlier
In reply to Jens AxboeIt's not necessary to do this post grabbing a reference. With that, we can drop the out goto path as well.
--- io_uring/eventfd.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
io_uring/eventfd.c +2 -3
@@ -69,10 +69,10 @@ void io_eventfd_signal(struct io_ring_ctx *ctx)*/ if (unlikely (!ev_fd))return ; +if (ev_fd->eventfd_async && !io_wq_current_is_worker ()) +return ;if (!refcount_inc_not_zero (&ev_fd->refs))return ; -if (ev_fd->eventfd_async && !io_wq_current_is_worker ()) - goto out;if (likely (eventfd_signal_allowed ())) {eventfd_signal_mask (ev_fd->cq_ev_fd,EPOLL_URING_WAKE );
@@ -82,7 +82,6 @@ void io_eventfd_signal(struct io_ring_ctx *ctx)return ; } } -out:io_eventfd_put (ev_fd); }
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 3/6 io_uring/eventfd: move actual signaling part into separate helper
In reply to Jens AxboeIn preparation for using this from multiple spots, move the signaling into a helper.
--- io_uring/eventfd.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-)
io_uring/eventfd.c +18 -10
@@ -47,6 +47,22 @@ static void io_eventfd_put(struct io_ev_fd *ev_fd)call_rcu (&ev_fd->rcu, io_eventfd_free); } +/* +* Returns true if the caller should put the ev_fd reference, false if not. +*/ +static bool __io_eventfd_signal (struct io_ev_fd *ev_fd) +{ +if (eventfd_signal_allowed ()) { +eventfd_signal_mask (ev_fd->cq_ev_fd,EPOLL_URING_WAKE ); +return true; + } +if (!atomic_fetch_or (BIT (IO_EVENTFD_OP_SIGNAL_BIT ), &ev_fd->ops)) { +call_rcu_hurry (&ev_fd->rcu, io_eventfd_do_signal); +return false; + } +return true; +} +void io_eventfd_signal (struct io_ring_ctx *ctx) {struct io_ev_fd *ev_fd =NULL ;
@@ -73,16 +89,8 @@ void io_eventfd_signal(struct io_ring_ctx *ctx)return ;if (!refcount_inc_not_zero (&ev_fd->refs))return ; - -if (likely (eventfd_signal_allowed ())) { -eventfd_signal_mask (ev_fd->cq_ev_fd,EPOLL_URING_WAKE ); - }else { -if (!atomic_fetch_or (BIT (IO_EVENTFD_OP_SIGNAL_BIT ), &ev_fd->ops)) { -call_rcu_hurry (&ev_fd->rcu, io_eventfd_do_signal); -return ; - } - } -io_eventfd_put (ev_fd); +if (__io_eventfd_signal (ev_fd)) +io_eventfd_put (ev_fd); }void io_eventfd_flush_signal (struct io_ring_ctx *ctx)
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 4/6 io_uring/eventfd: move trigger check into a helper
In reply to Jens AxboeIt's a bit hard to read what guards the triggering, move it into a helper and add a comment explaining it too. This additionally moves the ev_fd == NULL check in there as well.
--- io_uring/eventfd.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
io_uring/eventfd.c +12 -3
@@ -63,6 +63,17 @@ static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)return true; } +/* +* Trigger if eventfd_async isn't set, or if it's set and the caller is +* an async worker. If ev_fd isn't valid, obviously return false. +*/ +static bool io_eventfd_trigger (struct io_ev_fd *ev_fd) +{ +if (ev_fd) +return !ev_fd->eventfd_async ||io_wq_current_is_worker (); +return false; +} +void io_eventfd_signal (struct io_ring_ctx *ctx) {struct io_ev_fd *ev_fd =NULL ;
@@ -83,9 +94,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx)* completed between the NULL check of ctx->io_ev_fd at the start of * the function and rcu_read_lock. */ -if (unlikely (!ev_fd)) -return ; -if (ev_fd->eventfd_async && !io_wq_current_is_worker ()) +if (!io_eventfd_trigger (ev_fd))return ;if (!refcount_inc_not_zero (&ev_fd->refs))return ;
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 5/6 io_uring/eventfd: abstract out ev_fd grab + release helpers
In reply to Jens AxboeIn preparation for needing the ev_fd grabbing (and releasing) from another path, abstract out two helpers for that.
--- io_uring/eventfd.c | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-)
io_uring/eventfd.c +30 -11
@@ -47,6 +47,13 @@ static void io_eventfd_put(struct io_ev_fd *ev_fd)call_rcu (&ev_fd->rcu, io_eventfd_free); } +static void io_eventfd_release (struct io_ev_fd *ev_fd,bool put_ref) +{ +if (put_ref) +io_eventfd_put (ev_fd); +rcu_read_unlock (); +} +/* * Returns true if the caller should put the ev_fd reference, false if not. */
@@ -74,14 +81,18 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)return false; } -void io_eventfd_signal (struct io_ring_ctx *ctx) +/* +* On success, returns with an ev_fd reference grabbed and the RCU read +* lock held. +*/ +static struct io_ev_fd *io_eventfd_grab (struct io_ring_ctx *ctx) { -struct io_ev_fd *ev_fd =NULL ; +struct io_ev_fd *ev_fd;if (READ_ONCE (ctx->rings->cq_flags) &IORING_CQ_EVENTFD_DISABLED ) -return ; +return NULL ; -guard (rcu)(); +rcu_read_lock ();/* * rcu_dereference ctx->io_ev_fd once and use it for both for checking
@@ -90,16 +101,24 @@ void io_eventfd_signal(struct io_ring_ctx *ctx) ev_fd =rcu_dereference (ctx->io_ev_fd);/* -* Check again if ev_fd exists incase an io_eventfd_unregister call +* Check again if ev_fd exists in case an io_eventfd_unregister call * completed between the NULL check of ctx->io_ev_fd at the start of * the function and rcu_read_lock. */ -if (!io_eventfd_trigger (ev_fd)) -return ; -if (!refcount_inc_not_zero (&ev_fd->refs)) -return ; -if (__io_eventfd_signal (ev_fd)) -io_eventfd_put (ev_fd); +if (io_eventfd_trigger (ev_fd) &&refcount_inc_not_zero (&ev_fd->refs)) +return ev_fd; + +rcu_read_unlock (); +return NULL ; +} + +void io_eventfd_signal (struct io_ring_ctx *ctx) +{ +struct io_ev_fd *ev_fd; + + ev_fd =io_eventfd_grab (ctx); +if (ev_fd) +io_eventfd_release (ev_fd,__io_eventfd_signal (ev_fd)); }void io_eventfd_flush_signal (struct io_ring_ctx *ctx)
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 6/6 io_uring/eventfd: move ctx->evfd_last_cq_tail into io_ev_fd
In reply to Jens AxboeEverything else about the io_uring eventfd support is nicely kept private to that code, except the cached_cq_tail tracking. With everything else in place, move io_eventfd_flush_signal() to using the ev_fd grab+release helpers, which then enables the direct use of io_ev_fd for this tracking too.
--- io_uring/eventfd.c | 50 +++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 21 deletions(-)
io_uring/eventfd.c +29 -21
@@ -13,10 +13,12 @@struct io_ev_fd {struct eventfd_ctx *cq_ev_fd; -unsigned int eventfd_async:1 ; -struct rcu_head rcu; +unsigned int eventfd_async; +/* protected by ->completion_lock */ +unsigned last_cq_tail;refcount_t refs;atomic_t ops; +struct rcu_head rcu; };enum {
@@ -123,25 +125,31 @@ void io_eventfd_signal(struct io_ring_ctx *ctx)void io_eventfd_flush_signal (struct io_ring_ctx *ctx) { -bool skip; - -spin_lock (&ctx->completion_lock); - -/* -* Eventfd should only get triggered when at least one event has been -* posted. Some applications rely on the eventfd notification count -* only changing IFF a new CQE has been added to the CQ ring. There's -* no depedency on 1:1 relationship between how many times this -* function is called (and hence the eventfd count) and number of CQEs -* posted to the CQ ring. -*/ - skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; - ctx->evfd_last_cq_tail = ctx->cached_cq_tail; -spin_unlock (&ctx->completion_lock); -if (skip) -return ; +struct io_ev_fd *ev_fd; -io_eventfd_signal (ctx); + ev_fd =io_eventfd_grab (ctx); +if (ev_fd) { +bool skip, put_ref = true; + +/* +* Eventfd should only get triggered when at least one event +* has been posted. Some applications rely on the eventfd +* notification count only changing IFF a new CQE has been +* added to the CQ ring. There's no dependency on 1:1 +* relationship between how many times this function is called +* (and hence the eventfd count) and number of CQEs posted to +* the CQ ring. +*/ +spin_lock (&ctx->completion_lock); + skip = ctx->cached_cq_tail == ev_fd->last_cq_tail; + ev_fd->last_cq_tail = ctx->cached_cq_tail; +spin_unlock (&ctx->completion_lock); + +if (!skip) + put_ref =__io_eventfd_signal (ev_fd); + +io_eventfd_release (ev_fd, put_ref); + } } int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
@@ -172,7 +180,7 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, }spin_lock (&ctx->completion_lock); - ctx->evfd_last_cq_tail = ctx->cached_cq_tail; + ev_fd->last_cq_tail = ctx->cached_cq_tail;spin_unlock (&ctx->completion_lock); ev_fd->eventfd_async = eventfd_async;
Signature · 1 line
2.45.2
Jens Axboe
maintainerRe: [PATCH 6/6] io_uring/eventfd: move ctx->evfd_last_cq_tail into io_ev_fd
On 9/21/24 1:59 AM, Jens Axboe wrote:> Everything else about the io_uring eventfd support is nicely kept
Quoted 4 lines
private to that code, except the
cached_cq_tailtracking. With everything else in place, moveio_eventfd_flush_signal()to using theev_fdgrab+release helpers, which then enables the direct use ofio_ev_fdfor this tracking too.
Of course forgot to refresh io_uring_types.h as well, so this patch is missing the hunk that deletes it from io_ring_ctx. Refreshed version below, that's the only change.
include/linux/io_uring_types.h +0 -4
@@ -413,10 +413,6 @@ struct io_ring_ctx {DECLARE_HASHTABLE (napi_ht,4 );#endif - -/* protected by ->completion_lock */ -unsigned evfd_last_cq_tail; -/* * If IORING_SETUP_NO_MMAP is used, then the below holds * the gup'ed pages for the two rings, and the sqes.
io_uring/eventfd.c +29 -21
71 lines not on this page · raw message
Signature · 1 line
Jens Axboe
Quoted 8 lines · Jens Axboe
For some reason this ended up being a series of 6 patches, when the goal was really just to move
evfd_last_cq_tailout ofio_ring_ctxand into structio_ev_fd, where it belongs. But this slowly builds to that goal, and the final patch does the move unceremoniously.Patches are on top of current -git with for-6.12/io_uring pulled in.
[...]
Applied, thanks!
[1/6] io_uring/eventfd: abstract out ev_fd put helper
commit: a2656907e0e2bd817f18ae5ba0c0bc47373031e0
[2/6] io_uring/eventfd: check for the need to async notifier earlier
commit: e18ccce7024f18ada55e7bb714b28990e1fd3034
[3/6] io_uring/eventfd: move actual signaling part into separate helper
commit: 0fa1e1b85857005aa7777dd3c92fa1f9324909ef
[4/6] io_uring/eventfd: move trigger check into a helper
commit: bc9cd77386b3e5b9e7e4f2b016d0dbe4db2344bd
[5/6] io_uring/eventfd: abstract out ev_fd grab + release helpers
commit: bb4fd7c94e41777b7980e0d06ea521311f7330be
[6/6] io_uring/eventfd: move ctx->evfd_last_cq_tail into io_ev_fd
commit: e4355ab51156d5ca83733229fc6b3dfd7c5a4785Best regards,
Signature · 1 line
Jens Axboe