RCU: Enable callbacks to benefit from expedited grace periods
Subsystems (4)
cover RCU: Enable callbacks to benefit from expedited grace periods
RCU callbacks only track normal grace period sequence numbers. This means callbacks must wait for normal GPs even when expedited GPs have already elapsed.
This series tracks both normal and expedited GP sequences in the callback infrastructure using struct rcu_gp_oldstate, so callbacks advance when either GP type completes.
Commits 1-4 prepare the callback infrastructure: SRCU and Tasks RCU are isolated behind wrapper functions, the segment compaction logic is factored out for reuse, gp_seq is widened to struct rcu_gp_oldstate, and RCU_GET_STATE_NOT_TRACKED is added for subsystems that do not track expedited GPs.
Commit 5 is the core change: rcu_segcblist_advance() checks both normal and expedited GP completion via poll_state_synchronize_rcu_full(), and rcu_accelerate_cbs() captures both GP sequences via get_state_synchronize_rcu_full(). SRCU wrappers become standalone implementations since they cannot use poll_state_synchronize_rcu_full() which reads RCU-specific globals.
Commits 7-9 fix three notification paths so that expedited GP completion triggers callback processing: NOCB rcuog kthreads are woken, rcu_pending() detects the completion, and rcu_core() advances the callbacks.
Commit 10 adds nexp and exp_interval parameters to rcuscale for testing callback drain under concurrent expedited GP load.
Puranjay Mohan (10):
rcu/segcblist: Add SRCU and Tasks RCU wrapper functions rcu/segcblist: Factor out rcu_segcblist_advance_compact() helper rcu/segcblist: Change gp_seq to struct rcu_gp_oldstate gp_seq_full
rcu: Add RCU_GET_STATE_NOT_TRACKED for subsystems without expedited
GPs
rcu: Enable RCU callbacks to benefit from expedited grace periods
rcu: Update comments for gp_seq_full and expedited GP tracking
rcu: Wake NOCB rcuog kthreads on expedited grace period completion
rcu: Detect expedited grace period completion in rcu_pending()
rcu: Advance callbacks for expedited GP completion in rcu_core()
rcuscale: Add concurrent expedited GP threads for callback scaling
tests
include/linux/rcu_segcblist.h | 16 ++-- include/trace/events/rcu.h | 5 +- kernel/rcu/rcu.h | 13 ++- kernel/rcu/rcu_segcblist.c | 145 ++++++++++++++++++++++++---------- kernel/rcu/rcu_segcblist.h | 8 +- kernel/rcu/rcuscale.c | 84 +++++++++++++++++++- kernel/rcu/srcutree.c | 14 ++-- kernel/rcu/tasks.h | 8 +- kernel/rcu/tree.c | 54 +++++++++---- kernel/rcu/tree.h | 1 + kernel/rcu/tree_exp.h | 1 + kernel/rcu/tree_nocb.h | 77 ++++++++++++++---- 12 files changed, 324 insertions(+), 102 deletions(-)
base-commit: 51289896149317e0b6a4abf5d300a569c3353dca
Signature · 1 line
2.52.0
RFC PATCH 1/10 rcu/segcblist: Add SRCU and Tasks RCU wrapper functions
Add srcu_segcblist_advance() and srcu_segcblist_accelerate() wrappers that forward to the core rcu_segcblist_advance() and rcu_segcblist_accelerate() functions, and switch all SRCU (srcutree.c) and Tasks RCU (tasks.h) callers to use these wrappers.
This isolates SRCU and Tasks RCU from upcoming changes to the core advance/accelerate functions, which will switch to struct rcu_gp_oldstate for dual normal/expedited GP tracking. Because SRCU and Tasks RCU use only normal GP sequences, their wrappers will maintain the existing unsigned long interface.
No functional change.
--- kernel/rcu/rcu_segcblist.c | 10 ++++++++++ kernel/rcu/rcu_segcblist.h | 2 ++ kernel/rcu/srcutree.c | 14 +++++++------- kernel/rcu/tasks.h | 8 ++++---- 4 files changed, 23 insertions(+), 11 deletions(-)
kernel/rcu/rcu_segcblist.c +10 -0
@@ -620,3 +620,13 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,rcu_segcblist_init (src_rsclp); } + +void srcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq) +{ +rcu_segcblist_advance (rsclp, seq); +} + +bool srcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq) +{ +return rcu_segcblist_accelerate (rsclp, seq); +}
kernel/rcu/rcu_segcblist.h +2 -0
@@ -143,3 +143,5 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);bool rcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq);void rcu_segcblist_merge (struct rcu_segcblist *dst_rsclp,struct rcu_segcblist *src_rsclp); +void srcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq); +bool srcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq);
kernel/rcu/srcutree.c +7 -7
@@ -1351,7 +1351,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,* 2) The grace period for RCU_WAIT_TAIL is seen as started but not * completed so rcu_seq_current() returns X + SRCU_STATE_SCAN1. * -* 3) This value is passed to rcu_segcblist_advance() which can't move +* 3) This value is passed to srcu_segcblist_advance() which can't move * any segment forward and fails. * * 4) srcu_gp_start_if_needed() still proceeds with callback acceleration.
@@ -1360,15 +1360,15 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,* RCU_NEXT_READY_TAIL segment as started (ie: X + 4 + SRCU_STATE_SCAN1) * so it returns a snapshot of the next grace period, which is X + 12. * -* 5) The value of X + 12 is passed to rcu_segcblist_accelerate() but the +* 5) The value of X + 12 is passed to srcu_segcblist_accelerate() but the * freshly enqueued callback in RCU_NEXT_TAIL can't move to * RCU_NEXT_READY_TAIL which already has callbacks for a previous grace * period (gp_num = X + 8). So acceleration fails. */ s =rcu_seq_snap (&ssp->srcu_sup->srcu_gp_seq);if (rhp) { -rcu_segcblist_advance (&sdp->srcu_cblist, -rcu_seq_current (&ssp->srcu_sup->srcu_gp_seq)); +srcu_segcblist_advance (&sdp->srcu_cblist, +rcu_seq_current (&ssp->srcu_sup->srcu_gp_seq));/* * Acceleration can never fail because the base current gp_seq * used for acceleration is <= the value of gp_seq used for
@@ -1376,7 +1376,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,* always be able to be emptied by the acceleration into the * RCU_NEXT_READY_TAIL or RCU_WAIT_TAIL segments. */ -WARN_ON_ONCE (!rcu_segcblist_accelerate (&sdp->srcu_cblist, s)); +WARN_ON_ONCE (!srcu_segcblist_accelerate (&sdp->srcu_cblist, s)); }if (ULONG_CMP_LT (sdp->srcu_gp_seq_needed, s)) { sdp->srcu_gp_seq_needed = s;
@@ -1891,8 +1891,8 @@ static void srcu_invoke_callbacks(struct work_struct *work)rcu_cblist_init (&ready_cbs);raw_spin_lock_irq_rcu_node (sdp);WARN_ON_ONCE (!rcu_segcblist_segempty (&sdp->srcu_cblist,RCU_NEXT_TAIL )); -rcu_segcblist_advance (&sdp->srcu_cblist, -rcu_seq_current (&ssp->srcu_sup->srcu_gp_seq)); +srcu_segcblist_advance (&sdp->srcu_cblist, +rcu_seq_current (&ssp->srcu_sup->srcu_gp_seq));/* * Although this function is theoretically re-entrant, concurrent * callbacks invocation is disallowed to avoid executing an SRCU barrier
kernel/rcu/tasks.h +4 -4
@@ -480,8 +480,8 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)if (cpu >0 ) ncbsnz += n; } -rcu_segcblist_advance (&rtpcp->cblist,rcu_seq_current (&rtp->tasks_gp_seq)); - (void )rcu_segcblist_accelerate (&rtpcp->cblist,rcu_seq_snap (&rtp->tasks_gp_seq)); +srcu_segcblist_advance (&rtpcp->cblist,rcu_seq_current (&rtp->tasks_gp_seq)); + (void )srcu_segcblist_accelerate (&rtpcp->cblist,rcu_seq_snap (&rtp->tasks_gp_seq));if (rtpcp->urgent_gp >0 &&rcu_segcblist_pend_cbs (&rtpcp->cblist)) {if (rtp->lazy_jiffies) rtpcp->urgent_gp--;
@@ -564,7 +564,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpuif (rcu_segcblist_empty (&rtpcp->cblist))return ;raw_spin_lock_irqsave_rcu_node (rtpcp, flags); -rcu_segcblist_advance (&rtpcp->cblist,rcu_seq_current (&rtp->tasks_gp_seq)); +srcu_segcblist_advance (&rtpcp->cblist,rcu_seq_current (&rtp->tasks_gp_seq));rcu_segcblist_extract_done_cbs (&rtpcp->cblist, &rcl);raw_spin_unlock_irqrestore_rcu_node (rtpcp, flags); len = rcl.len;
@@ -577,7 +577,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu }raw_spin_lock_irqsave_rcu_node (rtpcp, flags);rcu_segcblist_add_len (&rtpcp->cblist, -len); - (void )rcu_segcblist_accelerate (&rtpcp->cblist,rcu_seq_snap (&rtp->tasks_gp_seq)); + (void )srcu_segcblist_accelerate (&rtpcp->cblist,rcu_seq_snap (&rtp->tasks_gp_seq));raw_spin_unlock_irqrestore_rcu_node (rtpcp, flags); }
Signature · 1 line
2.52.0
RFC PATCH 2/10 rcu/segcblist: Factor out rcu_segcblist_advance_compact() helper
In reply to Puranjay MohanThis commit extracts the tail-pointer cleanup and segment compaction logic from rcu_segcblist_advance() into a new static helper function, rcu_segcblist_advance_compact(). This shared logic will be reused by the upcoming srcu_segcblist_advance() standalone implementation, which cannot call the core rcu_segcblist_advance() because that function will use RCU-specific globals.
No functional change.
--- kernel/rcu/rcu_segcblist.c | 50 ++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 18 deletions(-)
kernel/rcu/rcu_segcblist.c +32 -18
@@ -462,13 +462,43 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,WRITE_ONCE (rsclp->tails[RCU_NEXT_TAIL ], rclp->tail); } +/* +* Clean up and compact the segmented callback list after callbacks have been +* advanced to the RCU_DONE_TAIL segment. The @i parameter is the index of the +* first segment that was NOT advanced (i.e., the segment after the last one +* moved to RCU_DONE_TAIL). This function fixes up tail pointers and compacts +* any gaps left by the moved segments. +*/ +static void rcu_segcblist_advance_compact (struct rcu_segcblist *rsclp,int i) +{ +int j; + +/* Clean up tail pointers that might have been misordered above. */ +for (j =RCU_WAIT_TAIL ; j < i; j++) +WRITE_ONCE (rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL ]); + +/* +* Callbacks moved, so there might be an empty RCU_WAIT_TAIL +* and a non-empty RCU_NEXT_READY_TAIL. If so, copy the +* RCU_NEXT_READY_TAIL segment to fill the RCU_WAIT_TAIL gap +* created by the now-ready-to-invoke segments. +*/ +for (j =RCU_WAIT_TAIL ; i <RCU_NEXT_TAIL ; i++, j++) { +if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL ]) +break ;/* No more callbacks. */ +WRITE_ONCE (rsclp->tails[j], rsclp->tails[i]); +rcu_segcblist_move_seglen (rsclp, i, j); + rsclp->gp_seq[j] = rsclp->gp_seq[i]; + } +} +/* * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ void rcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq) { -int i, j; +int i;WARN_ON_ONCE (!rcu_segcblist_is_enabled (rsclp));if (rcu_segcblist_restempty (rsclp,RCU_DONE_TAIL ))
@@ -489,23 +519,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)if (i ==RCU_WAIT_TAIL )return ; -/* Clean up tail pointers that might have been misordered above. */ -for (j =RCU_WAIT_TAIL ; j < i; j++) -WRITE_ONCE (rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL ]); - -/* -* Callbacks moved, so there might be an empty RCU_WAIT_TAIL -* and a non-empty RCU_NEXT_READY_TAIL. If so, copy the -* RCU_NEXT_READY_TAIL segment to fill the RCU_WAIT_TAIL gap -* created by the now-ready-to-invoke segments. -*/ -for (j =RCU_WAIT_TAIL ; i <RCU_NEXT_TAIL ; i++, j++) { -if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL ]) -break ;/* No more callbacks. */ -WRITE_ONCE (rsclp->tails[j], rsclp->tails[i]); -rcu_segcblist_move_seglen (rsclp, i, j); - rsclp->gp_seq[j] = rsclp->gp_seq[i]; - } +rcu_segcblist_advance_compact (rsclp, i); }/*
Signature · 1 line
2.52.0
RFC PATCH 3/10 rcu/segcblist: Change gp_seq to struct rcu_gp_oldstate gp_seq_full
In reply to Puranjay MohanThis commit renames the ->gp_seq[] field in struct rcu_segcblist to ->gp_seq_full[] and changes its type from unsigned long to struct rcu_gp_oldstate. This prepares the callback tracking infrastructure to support both normal and expedited grace periods.
All function signatures are updated to pass struct rcu_gp_oldstate
pointers: rcu_segcblist_nextgp(), rcu_segcblist_advance(), and rcu_segcblist_accelerate() now take struct rcu_gp_oldstate * instead of unsigned long. All callers are updated to use the .rgos_norm field for comparisons and assignments.
The SRCU and Tasks RCU wrappers now construct an rcu_gp_oldstate with just .rgos_norm set and forward to the core functions.
No functional change: only the .rgos_norm field is used in place of gp_seq.
--- include/linux/rcu_segcblist.h | 2 +- include/trace/events/rcu.h | 5 +++-- kernel/rcu/rcu_segcblist.c | 30 +++++++++++++++++------------- kernel/rcu/rcu_segcblist.h | 6 +++--- kernel/rcu/tree.c | 25 ++++++++++++++----------- kernel/rcu/tree_nocb.h | 29 +++++++++++++++-------------- 6 files changed, 53 insertions(+), 44 deletions(-)
include/linux/rcu_segcblist.h +1 -1
@@ -190,7 +190,7 @@ struct rcu_cblist {struct rcu_segcblist {struct rcu_head *head;struct rcu_head **tails[RCU_CBLIST_NSEGS ]; -unsigned long gp_seq[RCU_CBLIST_NSEGS ]; +struct rcu_gp_oldstate gp_seq_full[RCU_CBLIST_NSEGS ];#ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len;#else
include/trace/events/rcu.h +3 -2
@@ -547,10 +547,11 @@ TRACE_EVENT_RCU(rcu_segcb_stats, ),TP_fast_assign ( + int i; __entry->ctx = ctx;memcpy (__entry->seglen, rs->seglen,RCU_CBLIST_NSEGS *sizeof (long )); -memcpy (__entry->gp_seq, rs->gp_seq,RCU_CBLIST_NSEGS *sizeof (unsigned long )); - +for (i =0 ; i <RCU_CBLIST_NSEGS ; i++) + __entry->gp_seq[i] = rs->gp_seq_full[i].rgos_norm; ), TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) "
kernel/rcu/rcu_segcblist.c +17 -13
@@ -238,8 +238,8 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp) {int i; -BUILD_BUG_ON (RCU_NEXT_TAIL +1 !=ARRAY_SIZE (rsclp->gp_seq)); -BUILD_BUG_ON (ARRAY_SIZE (rsclp->tails) !=ARRAY_SIZE (rsclp->gp_seq)); +BUILD_BUG_ON (RCU_NEXT_TAIL +1 !=ARRAY_SIZE (rsclp->gp_seq_full)); +BUILD_BUG_ON (ARRAY_SIZE (rsclp->tails) !=ARRAY_SIZE (rsclp->gp_seq_full)); rsclp->head =NULL ;for (i =0 ; i <RCU_CBLIST_NSEGS ; i++) { rsclp->tails[i] = &rsclp->head;
@@ -307,13 +307,13 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)/* * Return false if there are no CBs awaiting grace periods, otherwise, -* return true and store the nearest waited-upon grace period into *lp. +* return true and store the nearest waited-upon grace period state into *rgosp. */ -bool rcu_segcblist_nextgp (struct rcu_segcblist *rsclp,unsigned long *lp) +bool rcu_segcblist_nextgp (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp) {if (!rcu_segcblist_pend_cbs (rsclp))return false; - *lp = rsclp->gp_seq[RCU_WAIT_TAIL ]; + *rgosp = rsclp->gp_seq_full[RCU_WAIT_TAIL ];return true; }
@@ -488,7 +488,7 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i)break ;/* No more callbacks. */ WRITE_ONCE (rsclp->tails[j], rsclp->tails[i]);rcu_segcblist_move_seglen (rsclp, i, j); - rsclp->gp_seq[j] = rsclp->gp_seq[i]; + rsclp->gp_seq_full[j] = rsclp->gp_seq_full[i]; } }
@@ -496,7 +496,7 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i)* Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ -void rcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq) +void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp) {int i;
@@ -509,7 +509,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)* are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i =RCU_WAIT_TAIL ; i <RCU_NEXT_TAIL ; i++) { -if (ULONG_CMP_LT (seq, rsclp->gp_seq[i])) +if (ULONG_CMP_LT (rgosp->rgos_norm, rsclp->gp_seq_full[i].rgos_norm))break ;WRITE_ONCE (rsclp->tails[RCU_DONE_TAIL ], rsclp->tails[i]);rcu_segcblist_move_seglen (rsclp, i,RCU_DONE_TAIL );
@@ -537,7 +537,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)* ready to invoke. Returns true if there are callbacks that won't be * ready to invoke until seq, false otherwise. */ -bool rcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq) +bool rcu_segcblist_accelerate (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp) {int i, j;
@@ -555,7 +555,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)*/ for (i =RCU_NEXT_READY_TAIL ; i >RCU_DONE_TAIL ; i--)if (!rcu_segcblist_segempty (rsclp, i) && -ULONG_CMP_LT (rsclp->gp_seq[i], seq)) +ULONG_CMP_LT (rsclp->gp_seq_full[i].rgos_norm, rgosp->rgos_norm))break ;/*
@@ -595,7 +595,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)*/ for (; i <RCU_NEXT_TAIL ; i++) {WRITE_ONCE (rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL ]); - rsclp->gp_seq[i] = seq; + rsclp->gp_seq_full[i].rgos_norm = rgosp->rgos_norm; }return true; }
@@ -637,10 +637,14 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,void srcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq) { -rcu_segcblist_advance (rsclp, seq); +struct rcu_gp_oldstate rgos = { .rgos_norm = seq }; + +rcu_segcblist_advance (rsclp, &rgos); }bool srcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq) { -return rcu_segcblist_accelerate (rsclp, seq); +struct rcu_gp_oldstate rgos = { .rgos_norm = seq }; + +return rcu_segcblist_accelerate (rsclp, &rgos); }
kernel/rcu/rcu_segcblist.h +3 -3
@@ -124,7 +124,7 @@ bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);bool rcu_segcblist_pend_cbs (struct rcu_segcblist *rsclp);struct rcu_head *rcu_segcblist_first_cb (struct rcu_segcblist *rsclp);struct rcu_head *rcu_segcblist_first_pend_cb (struct rcu_segcblist *rsclp); -bool rcu_segcblist_nextgp (struct rcu_segcblist *rsclp,unsigned long *lp); +bool rcu_segcblist_nextgp (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp);void rcu_segcblist_enqueue (struct rcu_segcblist *rsclp,struct rcu_head *rhp); bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
@@ -139,8 +139,8 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp);void rcu_segcblist_insert_pend_cbs (struct rcu_segcblist *rsclp,struct rcu_cblist *rclp); -void rcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq); -bool rcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq); +void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp); +bool rcu_segcblist_accelerate (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp);void rcu_segcblist_merge (struct rcu_segcblist *dst_rsclp,struct rcu_segcblist *src_rsclp);void srcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq);
kernel/rcu/tree.c +14 -11
@@ -1142,7 +1142,7 @@ static void rcu_gp_kthread_wake(void)*/ static bool rcu_accelerate_cbs (struct rcu_node *rnp,struct rcu_data *rdp) { -unsigned long gp_seq_req; +struct rcu_gp_oldstate rgos;bool ret = false;rcu_lockdep_assert_cblist_protected (rdp);
@@ -1164,15 +1164,15 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)* accelerating callback invocation to an earlier grace-period * number. */ - gp_seq_req =rcu_seq_snap (&rcu_state.gp_seq); -if (rcu_segcblist_accelerate (&rdp->cblist, gp_seq_req)) - ret =rcu_start_this_gp (rnp, rdp, gp_seq_req); + rgos.rgos_norm =rcu_seq_snap (&rcu_state.gp_seq); +if (rcu_segcblist_accelerate (&rdp->cblist, &rgos)) + ret =rcu_start_this_gp (rnp, rdp, rgos.rgos_norm);/* Trace depending on how much we were able to accelerate. */ if (rcu_segcblist_restempty (&rdp->cblist,RCU_WAIT_TAIL )) -trace_rcu_grace_period (rcu_state.name, gp_seq_req,TPS ("AccWaitCB" )); +trace_rcu_grace_period (rcu_state.name, rgos.rgos_norm,TPS ("AccWaitCB" ));else -trace_rcu_grace_period (rcu_state.name, gp_seq_req,TPS ("AccReadyCB" )); +trace_rcu_grace_period (rcu_state.name, rgos.rgos_norm,TPS ("AccReadyCB" ));trace_rcu_segcb_stats (&rdp->cblist,TPS ("SegCbPostAcc" ));
@@ -1189,14 +1189,14 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)static void rcu_accelerate_cbs_unlocked (struct rcu_node *rnp,struct rcu_data *rdp) { -unsigned long c; +struct rcu_gp_oldstate rgos;bool needwake;rcu_lockdep_assert_cblist_protected (rdp); - c =rcu_seq_snap (&rcu_state.gp_seq); -if (!READ_ONCE (rdp->gpwrap) &&ULONG_CMP_GE (rdp->gp_seq_needed, c)) { + rgos.rgos_norm =rcu_seq_snap (&rcu_state.gp_seq); +if (!READ_ONCE (rdp->gpwrap) &&ULONG_CMP_GE (rdp->gp_seq_needed, rgos.rgos_norm)) {/* Old request still live, so mark recent callbacks. */ - (void )rcu_segcblist_accelerate (&rdp->cblist, c); + (void )rcu_segcblist_accelerate (&rdp->cblist, &rgos);return ; }raw_spin_lock_rcu_node (rnp);/* irqs already disabled. */
@@ -1218,6 +1218,8 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,*/ static bool rcu_advance_cbs (struct rcu_node *rnp,struct rcu_data *rdp) { +struct rcu_gp_oldstate rgos; +rcu_lockdep_assert_cblist_protected (rdp);raw_lockdep_assert_held_rcu_node (rnp);
@@ -1229,7 +1231,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)* Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. */ -rcu_segcblist_advance (&rdp->cblist, rnp->gp_seq); + rgos.rgos_norm = rnp->gp_seq; +rcu_segcblist_advance (&rdp->cblist, &rgos);/* Classify any remaining callbacks. */ return rcu_accelerate_cbs (rnp, rdp);
kernel/rcu/tree_nocb.h +15 -14
@@ -433,7 +433,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool lazy) {unsigned long c; -unsigned long cur_gp_seq; +struct rcu_gp_oldstate cur_gp_seq_full;unsigned long j = jiffies;long ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass);long lazy_len =READ_ONCE (rdp->lazy_len);
@@ -501,8 +501,8 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,return false;// Caller must enqueue the callback. }if (j != rdp->nocb_gp_adv_time && -rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq) && -rcu_seq_done (&rdp->mynode->gp_seq, cur_gp_seq)) { +rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && +rcu_seq_done (&rdp->mynode->gp_seq, cur_gp_seq_full.rgos_norm)) {rcu_advance_cbs_nowake (rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; }
@@ -659,7 +659,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) {bool bypass = false;int __maybe_unused cpu = my_rdp->cpu; -unsigned long cur_gp_seq; +struct rcu_gp_oldstate cur_gp_seq_full;unsigned long flags;bool gotcbs = false;unsigned long j = jiffies;
@@ -730,8 +730,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) needwake_gp = false;if (!rcu_segcblist_restempty (&rdp->cblist,RCU_NEXT_READY_TAIL ) || - (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq) && -rcu_seq_done (&rnp->gp_seq, cur_gp_seq))) { + (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && +rcu_seq_done (&rnp->gp_seq, cur_gp_seq_full.rgos_norm))) {raw_spin_lock_rcu_node (rnp);/* irqs disabled. */ needwake_gp =rcu_advance_cbs (rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist,
@@ -742,10 +742,10 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)WARN_ON_ONCE (wasempty && !rcu_segcblist_restempty (&rdp->cblist,RCU_NEXT_READY_TAIL )); -if (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq)) { +if (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full)) {if (!needwait_gp || -ULONG_CMP_LT (cur_gp_seq, wait_gp_seq)) - wait_gp_seq = cur_gp_seq; +ULONG_CMP_LT (cur_gp_seq_full.rgos_norm, wait_gp_seq)) + wait_gp_seq = cur_gp_seq_full.rgos_norm; needwait_gp = true;trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("NeedWaitGP" ));
@@ -877,7 +877,7 @@ static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)static void nocb_cb_wait (struct rcu_data *rdp) {struct rcu_segcblist *cblist = &rdp->cblist; -unsigned long cur_gp_seq; +struct rcu_gp_oldstate cur_gp_seq_full;unsigned long flags;bool needwake_gp = false;struct rcu_node *rnp = rdp->mynode;
@@ -918,8 +918,8 @@ static void nocb_cb_wait(struct rcu_data *rdp)local_bh_enable ();lockdep_assert_irqs_enabled ();rcu_nocb_lock_irqsave (rdp, flags); -if (rcu_segcblist_nextgp (cblist, &cur_gp_seq) && -rcu_seq_done (&rnp->gp_seq, cur_gp_seq) && +if (rcu_segcblist_nextgp (cblist, &cur_gp_seq_full) && +rcu_seq_done (&rnp->gp_seq, cur_gp_seq_full.rgos_norm) &&raw_spin_trylock_rcu_node (rnp)) {/* irqs already disabled. */ needwake_gp =rcu_advance_cbs (rdp->mynode, rdp);raw_spin_unlock_rcu_node (rnp);/* irqs remain disabled. */
@@ -1569,9 +1569,10 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) nocb_entry_rdp);sprintf (bufd,"%ld" , rsclp->seglen[RCU_DONE_TAIL ]); -sprintf (bufw,"%ld(%ld)" , rsclp->seglen[RCU_WAIT_TAIL ], rsclp->gp_seq[RCU_WAIT_TAIL ]); +sprintf (bufw,"%ld(%ld)" , rsclp->seglen[RCU_WAIT_TAIL ], + rsclp->gp_seq_full[RCU_WAIT_TAIL ].rgos_norm);sprintf (bufr,"%ld(%ld)" , rsclp->seglen[RCU_NEXT_READY_TAIL ], - rsclp->gp_seq[RCU_NEXT_READY_TAIL ]); + rsclp->gp_seq_full[RCU_NEXT_READY_TAIL ].rgos_norm);sprintf (bufn,"%ld" , rsclp->seglen[RCU_NEXT_TAIL ]);sprintf (bufb,"%ld" ,rcu_cblist_n_cbs (&rdp->nocb_bypass)); pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n",
Signature · 1 line
2.52.0
RFC PATCH 4/10 rcu: Add RCU_GET_STATE_NOT_TRACKED for subsystems without expedited GPs
In reply to Puranjay MohanSRCU and Tasks RCU do not track expedited grace periods. When their callback state is checked via poll_state_synchronize_rcu_full(), the uninitialized or zeroed rgos_exp field could cause false-positive completion detection.
This commit adds an RCU_GET_STATE_NOT_TRACKED sentinel value (0x2) that these subsystems can place into rgos_exp to indicate that expedited GP tracking is not applicable. The expedited sequence check in poll_state_synchronize_rcu_full() is guarded to skip entries marked with this sentinel.
This is needed to allow rcu_segcblist_advance() and rcu_accelerate_cbs() to work with both normal and expedited grace periods via get_state_synchronize_rcu_full() and poll_state_synchronize_rcu_full().
--- kernel/rcu/rcu.h | 13 +++++++++++-- kernel/rcu/tree.c | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-)
kernel/rcu/rcu.h +11 -2
@@ -46,16 +46,25 @@* the number of pending readers that will use * this inactive index is bounded). * -* RCU polled GP special control value: +* RCU polled GP special control values: * * RCU_GET_STATE_COMPLETED : State value indicating an already-completed * polled GP has completed. This value covers * both the state and the counter of the * grace-period sequence number. +* +* RCU_GET_STATE_NOT_TRACKED : State value indicating that a GP component +* is not tracked by this subsystem and should +* not be checked. Used by SRCU and RCU Tasks +* which do not track expedited GPs, to prevent +* false-positive completion when their +* gp_seq_full entries are checked via +* poll_state_synchronize_rcu_full(). */ -/* Low-order bit definition for polled grace-period APIs. */ +/* Low-order bit definitions for polled grace-period APIs. */ #define RCU_GET_STATE_COMPLETED 0x1 +#define RCU_GET_STATE_NOT_TRACKED 0x2/* A complete grace period count */ #define RCU_SEQ_GP (RCU_SEQ_STATE_MASK + 1)
kernel/rcu/tree.c +2 -1
@@ -3603,7 +3603,8 @@ bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)if (rgosp->rgos_norm ==RCU_GET_STATE_COMPLETED ||rcu_seq_done_exact (&rnp->gp_seq, rgosp->rgos_norm) || rgosp->rgos_exp ==RCU_GET_STATE_COMPLETED || -rcu_seq_done_exact (&rcu_state.expedited_sequence, rgosp->rgos_exp)) { + (rgosp->rgos_exp !=RCU_GET_STATE_NOT_TRACKED && +rcu_seq_done_exact (&rcu_state.expedited_sequence, rgosp->rgos_exp))) {smp_mb ();/* Ensure GP ends before subsequent accesses. */ return true; }
Signature · 1 line
2.52.0
RFC PATCH 5/10 rcu: Enable RCU callbacks to benefit from expedited grace periods
In reply to Puranjay MohanCurrently, RCU callbacks only track normal grace period sequence numbers. This means callbacks must wait for normal grace periods to complete even when expedited grace periods have already elapsed.
This commit uses the full rcu_gp_oldstate structure (which tracks both normal and expedited GP sequences) throughout the callback infrastructure.
The rcu_segcblist_advance() function now checks both normal and expedited GP completion via poll_state_synchronize_rcu_full(), becoming parameterless since it reads the GP state internally. rcu_segcblist_accelerate() stores the full GP state (both normal and expedited sequences) instead of just the normal sequence.
The rcu_accelerate_cbs() and rcu_accelerate_cbs_unlocked() functions use get_state_synchronize_rcu_full() to capture both GP sequences. The NOCB code uses poll_state_synchronize_rcu_full() for advance checks instead of comparing only the normal GP sequence.
srcu_segcblist_advance() become standalone implementations because compares SRCU sequences directly (it cannot use poll_state_synchronize_rcu_full(), which reads RCU-specific globals). srcu_segcblist_accelerate() sets rgos_exp to RCU_GET_STATE_NOT_TRACKED so poll_state_synchronize_rcu_full() only compares the rgosp->rgos_norm and ignores rgos_exp.
--- kernel/rcu/rcu_segcblist.c | 30 ++++++++++++++++++++++++------ kernel/rcu/rcu_segcblist.h | 2 +- kernel/rcu/tree.c | 9 +++------ kernel/rcu/tree_nocb.h | 33 +++++++++++++++++++++++---------- 4 files changed, 51 insertions(+), 23 deletions(-)
kernel/rcu/rcu_segcblist.c +24 -6
@@ -12,6 +12,7 @@#include <linux/kernel.h> #include <linux/types.h> +#include "rcu.h" #include "rcu_segcblist.h" /* Initialize simple callback list. */
@@ -494,9 +495,9 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i)/* * Advance the callbacks in the specified rcu_segcblist structure based -* on the current value passed in for the grace-period counter. +* on the current value of the grace-period counter. */ -void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp) +void rcu_segcblist_advance (struct rcu_segcblist *rsclp) {int i;
@@ -509,7 +510,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_oldstate ** are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i =RCU_WAIT_TAIL ; i <RCU_NEXT_TAIL ; i++) { -if (ULONG_CMP_LT (rgosp->rgos_norm, rsclp->gp_seq_full[i].rgos_norm)) +if (!poll_state_synchronize_rcu_full (&rsclp->gp_seq_full[i]))break ;WRITE_ONCE (rsclp->tails[RCU_DONE_TAIL ], rsclp->tails[i]);rcu_segcblist_move_seglen (rsclp, i,RCU_DONE_TAIL );
@@ -595,7 +596,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_oldstat*/ for (; i <RCU_NEXT_TAIL ; i++) {WRITE_ONCE (rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL ]); - rsclp->gp_seq_full[i].rgos_norm = rgosp->rgos_norm; + rsclp->gp_seq_full[i] = *rgosp; }return true; }
@@ -637,14 +638,31 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,void srcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq) { -struct rcu_gp_oldstate rgos = { .rgos_norm = seq }; +int i; -rcu_segcblist_advance (rsclp, &rgos); +WARN_ON_ONCE (!rcu_segcblist_is_enabled (rsclp)); +if (rcu_segcblist_restempty (rsclp,RCU_DONE_TAIL )) +return ; + +for (i =RCU_WAIT_TAIL ; i <RCU_NEXT_TAIL ; i++) { +if (ULONG_CMP_LT (seq, rsclp->gp_seq_full[i].rgos_norm)) +break ; +WRITE_ONCE (rsclp->tails[RCU_DONE_TAIL ], rsclp->tails[i]); +rcu_segcblist_move_seglen (rsclp, i,RCU_DONE_TAIL ); + } + +/* If no callbacks moved, nothing more need be done. */ +if (i ==RCU_WAIT_TAIL ) +return ; + +rcu_segcblist_advance_compact (rsclp, i); }bool srcu_segcblist_accelerate (struct rcu_segcblist *rsclp,unsigned long seq) {struct rcu_gp_oldstate rgos = { .rgos_norm = seq }; +if (IS_ENABLED (CONFIG_SMP )) + rgos.rgos_exp =RCU_GET_STATE_NOT_TRACKED ;return rcu_segcblist_accelerate (rsclp, &rgos); }
kernel/rcu/rcu_segcblist.h +1 -1
@@ -139,7 +139,7 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp);void rcu_segcblist_insert_pend_cbs (struct rcu_segcblist *rsclp,struct rcu_cblist *rclp); -void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp); +void rcu_segcblist_advance (struct rcu_segcblist *rsclp);bool rcu_segcblist_accelerate (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp);void rcu_segcblist_merge (struct rcu_segcblist *dst_rsclp,struct rcu_segcblist *src_rsclp);
kernel/rcu/tree.c +3 -6
@@ -1164,7 +1164,7 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)* accelerating callback invocation to an earlier grace-period * number. */ - rgos.rgos_norm =rcu_seq_snap (&rcu_state.gp_seq); +get_state_synchronize_rcu_full (&rgos);if (rcu_segcblist_accelerate (&rdp->cblist, &rgos)) ret =rcu_start_this_gp (rnp, rdp, rgos.rgos_norm);
@@ -1193,7 +1193,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,bool needwake;rcu_lockdep_assert_cblist_protected (rdp); - rgos.rgos_norm =rcu_seq_snap (&rcu_state.gp_seq); +get_state_synchronize_rcu_full (&rgos);if (!READ_ONCE (rdp->gpwrap) &&ULONG_CMP_GE (rdp->gp_seq_needed, rgos.rgos_norm)) {/* Old request still live, so mark recent callbacks. */ (void )rcu_segcblist_accelerate (&rdp->cblist, &rgos);
@@ -1218,8 +1218,6 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,*/ static bool rcu_advance_cbs (struct rcu_node *rnp,struct rcu_data *rdp) { -struct rcu_gp_oldstate rgos; -rcu_lockdep_assert_cblist_protected (rdp);raw_lockdep_assert_held_rcu_node (rnp);
@@ -1231,8 +1229,7 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)* Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. */ - rgos.rgos_norm = rnp->gp_seq; -rcu_segcblist_advance (&rdp->cblist, &rgos); +rcu_segcblist_advance (&rdp->cblist);/* Classify any remaining callbacks. */ return rcu_accelerate_cbs (rnp, rdp);
kernel/rcu/tree_nocb.h +23 -10
@@ -502,7 +502,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, }if (j != rdp->nocb_gp_adv_time &&rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && -rcu_seq_done (&rdp->mynode->gp_seq, cur_gp_seq_full.rgos_norm)) { +poll_state_synchronize_rcu_full (&cur_gp_seq_full)) {rcu_advance_cbs_nowake (rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; }
@@ -731,7 +731,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)if (!rcu_segcblist_restempty (&rdp->cblist,RCU_NEXT_READY_TAIL ) || (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && -rcu_seq_done (&rnp->gp_seq, cur_gp_seq_full.rgos_norm))) { +poll_state_synchronize_rcu_full (&cur_gp_seq_full))) {raw_spin_lock_rcu_node (rnp);/* irqs disabled. */ needwake_gp =rcu_advance_cbs (rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist,
@@ -742,7 +742,18 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)WARN_ON_ONCE (wasempty && !rcu_segcblist_restempty (&rdp->cblist,RCU_NEXT_READY_TAIL )); -if (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full)) { +/* +* Only request a GP wait if the next pending callback's +* GP has not already completed (normal or expedited). +* If poll_state_synchronize_rcu_full() says it completed, +* then rcu_advance_cbs() above already moved those +* callbacks to RCU_DONE_TAIL, so there is no GP to wait +* for. Any remaining callbacks got new (future) GP +* numbers from rcu_accelerate_cbs() inside +* rcu_advance_cbs() and will be handled on the next pass. +*/ +if (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && + !poll_state_synchronize_rcu_full (&cur_gp_seq_full)) {if (!needwait_gp ||ULONG_CMP_LT (cur_gp_seq_full.rgos_norm, wait_gp_seq)) wait_gp_seq = cur_gp_seq_full.rgos_norm;
@@ -919,7 +930,7 @@ static void nocb_cb_wait(struct rcu_data *rdp)lockdep_assert_irqs_enabled ();rcu_nocb_lock_irqsave (rdp, flags);if (rcu_segcblist_nextgp (cblist, &cur_gp_seq_full) && -rcu_seq_done (&rnp->gp_seq, cur_gp_seq_full.rgos_norm) && +poll_state_synchronize_rcu_full (&cur_gp_seq_full) &&raw_spin_trylock_rcu_node (rnp)) {/* irqs already disabled. */ needwake_gp =rcu_advance_cbs (rdp->mynode, rdp);raw_spin_unlock_rcu_node (rnp);/* irqs remain disabled. */
@@ -1548,8 +1559,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp)static void show_rcu_nocb_state (struct rcu_data *rdp) {char bufd[22 ]; -char bufw[45 ]; -char bufr[45 ]; +char bufw[64 ]; +char bufr[64 ];char bufn[22 ];char bufb[22 ];struct rcu_data *nocb_next_rdp;
@@ -1569,10 +1580,12 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) nocb_entry_rdp);sprintf (bufd,"%ld" , rsclp->seglen[RCU_DONE_TAIL ]); -sprintf (bufw,"%ld(%ld)" , rsclp->seglen[RCU_WAIT_TAIL ], - rsclp->gp_seq_full[RCU_WAIT_TAIL ].rgos_norm); -sprintf (bufr,"%ld(%ld)" , rsclp->seglen[RCU_NEXT_READY_TAIL ], - rsclp->gp_seq_full[RCU_NEXT_READY_TAIL ].rgos_norm); +sprintf (bufw,"%ld(%ld/%ld)" , rsclp->seglen[RCU_WAIT_TAIL ], + rsclp->gp_seq_full[RCU_WAIT_TAIL ].rgos_norm, + rsclp->gp_seq_full[RCU_WAIT_TAIL ].rgos_exp); +sprintf (bufr,"%ld(%ld/%ld)" , rsclp->seglen[RCU_NEXT_READY_TAIL ], + rsclp->gp_seq_full[RCU_NEXT_READY_TAIL ].rgos_norm, + rsclp->gp_seq_full[RCU_NEXT_READY_TAIL ].rgos_exp);sprintf (bufn,"%ld" , rsclp->seglen[RCU_NEXT_TAIL ]);sprintf (bufb,"%ld" ,rcu_cblist_n_cbs (&rdp->nocb_bypass)); pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n",
Signature · 1 line
2.52.0
RFC PATCH 6/10 rcu: Update comments for gp_seq_full and expedited GP tracking
In reply to Puranjay MohanThis commit updates documentation comments throughout the RCU callback infrastructure to reflect the transition from single grace-period sequence numbers to the full rcu_gp_oldstate structure that tracks both normal and expedited grace periods.
The ->gp_seq_full[] array documentation in rcu_segcblist.h is updated to describe dual GP tracking. The rcu_segcblist_advance(), rcu_segcblist_accelerate(), and rcu_advance_cbs() comments are updated to reference ->gp_seq_full[] and rgosp instead of the old ->gp_seq[] and seq names.
--- include/linux/rcu_segcblist.h | 14 +++++++----- kernel/rcu/rcu_segcblist.c | 43 +++++++++++++++++++++++------------ kernel/rcu/tree.c | 6 ++--- 3 files changed, 39 insertions(+), 24 deletions(-)
include/linux/rcu_segcblist.h +8 -6
@@ -50,12 +50,14 @@ struct rcu_cblist {* Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also * empty. * -* The ->gp_seq[] array contains the grace-period number at which the -* corresponding segment of callbacks will be ready to invoke. A given -* element of this array is meaningful only when the corresponding segment -* is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks -* are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have -* not yet been assigned a grace-period number). +* The ->gp_seq_full[] array contains the grace-period state at which the +* corresponding segment of callbacks will be ready to invoke. This tracks +* both normal and expedited grace periods, allowing callbacks to complete +* when either type of GP finishes. A given element of this array is +* meaningful only when the corresponding segment is non-empty, and it is +* never valid for RCU_DONE_TAIL (whose callbacks are already ready to +* invoke) or for RCU_NEXT_TAIL (whose callbacks have not yet been assigned +* a grace-period state). */ #define RCU_DONE_TAIL 0/* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1/* Also RCU_NEXT_READY head. */
kernel/rcu/rcu_segcblist.c +28 -15
105 lines not on this page · raw message
kernel/rcu/tree.c +3 -3
@@ -1209,7 +1209,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,/* * Move any callbacks whose grace period has completed to the * RCU_DONE_TAIL sublist, then compact the remaining sublists and -* assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL +* assign ->gp_seq_full[] state to any callbacks in the RCU_NEXT_TAIL * sublist. This function is idempotent, so it does not hurt to * invoke it repeatedly. As long as it is not invoked -too- often... * Returns true if the RCU grace-period kthread needs to be awakened.
@@ -1226,8 +1226,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)return false;/* -* Find all callbacks whose ->gp_seq numbers indicate that they -* are ready to invoke, and put them into the RCU_DONE_TAIL sublist. +* Find all callbacks whose grace periods have completed (either +* normal or expedited) and put them into the RCU_DONE_TAIL sublist. */ rcu_segcblist_advance (&rdp->cblist);
Signature · 1 line
2.52.0
RFC PATCH 7/10 rcu: Wake NOCB rcuog kthreads on expedited grace period completion
In reply to Puranjay MohanWhen an expedited grace period completes, rcu_exp_wait_wake() wakes waiters on rnp->exp_wq[] but does not notify NOCB rcuog kthreads. These kthreads may be sleeping waiting for a grace period to complete. Without this wakeup, callbacks on offloaded CPUs that could benefit from the expedited GP must wait until the rcuog kthread wakes for some other reason (e.g., next normal GP completion or a timer).
Add rcu_exp_wake_nocb() which wakes rcuog kthreads for leaf-node CPUs, deduplicating via rdp->nocb_gp_rdp since multiple CPUs share one rcuog kthread. Uses for_each_leaf_node_possible_cpu() because offline CPUs can have pending callbacks. The function is defined in tree_nocb.h with an empty stub for CONFIG_RCU_NOCB_CPU=n builds.
--- kernel/rcu/tree.h | 1 + kernel/rcu/tree_exp.h | 1 + kernel/rcu/tree_nocb.h | 29 +++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+)
kernel/rcu/tree.h +1 -0
@@ -500,6 +500,7 @@ static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp);static bool wake_nocb_gp (struct rcu_data *rdp); +static void rcu_exp_wake_nocb (struct rcu_node *rnp);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j,bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,
kernel/rcu/tree_exp.h +1 -0
@@ -708,6 +708,7 @@ static void rcu_exp_wait_wake(unsigned long s) }smp_mb ();/* All above changes before wakeup. */ wake_up_all (&rnp->exp_wq[rcu_seq_ctr (s) &0x3 ]); +rcu_exp_wake_nocb (rnp); }trace_rcu_exp_grace_period (rcu_state.name, s,TPS ("endwake" ));mutex_unlock (&rcu_state.exp_wake_mutex);
kernel/rcu/tree_nocb.h +29 -0
@@ -190,6 +190,31 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_swait_queue_head (&rnp->nocb_gp_wq[1 ]); } +/* +* Wake NOCB rcuog kthreads for leaf-node CPUs so that they can advance +* callbacks that were waiting for the just-completed expedited GP. +* Deduplicate via nocb_gp_rdp since multiple CPUs share one rcuog +* kthread. Use for_each_leaf_node_possible_cpu() because offline CPUs +* may have pending callbacks. +*/ +static void rcu_exp_wake_nocb (struct rcu_node *rnp) +{ +struct rcu_data *last_rdp_gp =NULL ; +int cpu; + +if (!rcu_is_leaf_node (rnp)) +return ; + +for_each_leaf_node_possible_cpu (rnp, cpu) { +struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); + +if (rdp->nocb_gp_rdp == last_rdp_gp) +continue ; + last_rdp_gp = rdp->nocb_gp_rdp; +wake_nocb_gp (rdp); + } +} +/* Clear any pending deferred wakeup timer (nocb_gp_lock must be held). */ static void nocb_defer_wakeup_cancel (struct rcu_data *rdp_gp) {
@@ -1668,6 +1693,10 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) { } +static void rcu_exp_wake_nocb (struct rcu_node *rnp) +{ +} +static bool wake_nocb_gp (struct rcu_data *rdp) {return false;
Signature · 1 line
2.52.0
RFC PATCH 8/10 rcu: Detect expedited grace period completion in rcu_pending()
In reply to Puranjay Mohanrcu_pending() is the gatekeeper that decides whether rcu_core() should run on the current CPU's timer tick. Currently it checks if the CPU has callbacks ready to invoke or a grace period has completed or started.
It does not check that an expedited GP has completed. After an expedited GP, callbacks remain in RCU_WAIT_TAIL (not yet advanced to RCU_DONE_TAIL) and So rcu_core() never runs to advance them.
Add a check using rcu_segcblist_nextgp() combined with poll_state_synchronize_rcu_full() to detect when any pending callbacks' grace period has completed.
--- kernel/rcu/tree.c | 7 +++++++ 1 file changed, 7 insertions(+)
kernel/rcu/tree.c +7 -0
@@ -3671,6 +3671,7 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full);static int rcu_pending (int user) {bool gp_in_progress; +struct rcu_gp_oldstate gp_state;struct rcu_data *rdp =this_cpu_ptr (&rcu_data);struct rcu_node *rnp = rdp->mynode;
@@ -3701,6 +3702,12 @@ static int rcu_pending(int user) rcu_segcblist_ready_cbs(&rdp->cblist))return 1 ; +/* Has a GP (normal or expedited) completed for pending callbacks? */ +if (!rcu_rdp_is_offloaded (rdp) && +rcu_segcblist_nextgp (&rdp->cblist, &gp_state) && +poll_state_synchronize_rcu_full (&gp_state)) +return 1 ; +/* Has RCU gone idle with this CPU needing another grace period? */ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp) &&
Signature · 1 line
2.52.0
RFC PATCH 9/10 rcu: Advance callbacks for expedited GP completion in rcu_core()
In reply to Puranjay MohanEven when rcu_pending() triggers rcu_core(), the normal callback advancement path through note_gp_changes() -> __note_gp_changes() bails out when rdp->gp_seq == rnp->gp_seq (no normal GP change). Since expedited GPs do not update rnp->gp_seq, rcu_advance_cbs() is never called and callbacks remain stuck in RCU_WAIT_TAIL.
Add a direct callback advancement block in rcu_core() that checks for GP completion via rcu_segcblist_nextgp() combined with poll_state_synchronize_rcu_full(). When detected, trylock rnp and call rcu_advance_cbs() to move completed callbacks to RCU_DONE_TAIL. Wake the GP kthread if rcu_advance_cbs() requests a new grace period.
Uses trylock to avoid adding contention on rnp->lock. If the lock is contended, callbacks will be advanced on the next tick.
--- kernel/rcu/tree.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
kernel/rcu/tree.c +16 -0
@@ -2853,6 +2853,22 @@ static __latent_entropy void rcu_core(void)/* Update RCU state based on any recent quiescent states. */ rcu_check_quiescent_state (rdp); +/* Advance callbacks if an expedited GP has completed. */ +if (!rcu_rdp_is_offloaded (rdp) && +rcu_segcblist_is_enabled (&rdp->cblist)) { +struct rcu_gp_oldstate gp_state; + +if (rcu_segcblist_nextgp (&rdp->cblist, &gp_state) && +poll_state_synchronize_rcu_full (&gp_state)) { +guard (irqsave)(); +if (raw_spin_trylock_rcu_node (rnp)) { +if (rcu_advance_cbs (rnp, rdp)) +rcu_gp_kthread_wake (); +raw_spin_unlock_rcu_node (rnp); + } + } + } +/* No grace period and unregistered callbacks? */ if (!rcu_gp_in_progress () &&rcu_segcblist_is_enabled (&rdp->cblist) && !rcu_rdp_is_offloaded (rdp)) {
Signature · 1 line
2.52.0
RFC PATCH 10/10 rcuscale: Add concurrent expedited GP threads for callback scaling tests
In reply to Puranjay MohanAdd nexp and exp_interval parameters to rcuscale that spawn kthreads running synchronize_rcu_expedited() in a loop. This generates concurrent expedited GP load while the normal writers measure GP or callback latency.
When combined with gp_async=1 (which uses call_rcu() for writers), this tests how effectively callbacks benefit from expedited grace periods. With RCU callback expedited GP tracking, the async callbacks should complete faster because they piggyback on the expedited GPs rather than waiting for normal GPs.
--- kernel/rcu/rcuscale.c | 84 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-)
kernel/rcu/rcuscale.c +82 -2
@@ -91,6 +91,8 @@ torture_param(int, shutdown_secs, !IS_MODULE(CONFIG_RCU_SCALE_TEST) * 300,torture_param (int, verbose,1 ,"Enable verbose debugging printk()s" );torture_param (int, writer_holdoff,0 ,"Holdoff (us) between GPs, zero to disable" );torture_param (int, writer_holdoff_jiffies,0 ,"Holdoff (jiffies) between GPs, zero to disable" ); +torture_param (int, nexp,0 ,"Number of expedited GP threads to run concurrently" ); +torture_param (int, exp_interval,0 ,"Interval (us) between expedited GPs, zero to disable" );torture_param (int, kfree_rcu_test,0 ,"Do we run a kfree_rcu() scale test?" );torture_param (int, kfree_mult,1 ,"Multiple of kfree_obj size to allocate." );torture_param (int, kfree_by_call_rcu,0 ,"Use call_rcu() to emulate kfree_rcu()?" );
@@ -115,8 +117,10 @@ struct writer_freelist {static int nrealreaders;static int nrealwriters; +static int nrealexp;static struct task_struct **writer_tasks;static struct task_struct **reader_tasks; +static struct task_struct **exp_tasks;static u64 **writer_durations;static bool *writer_done;
@@ -462,6 +466,34 @@ rcu_scale_reader(void *arg)return 0 ; } +/* +* RCU expedited GP kthread. Repeatedly invokes expedited grace periods +* to generate concurrent expedited GP load while the normal-GP writers +* are being measured. This allows measuring the benefit of callbacks +* that can piggyback on expedited grace periods. +*/ +static int +rcu_scale_exp (void *arg) +{ +long me = (long )arg; + +VERBOSE_SCALEOUT_STRING ("rcu_scale_exp task started" ); +set_cpus_allowed_ptr (current,cpumask_of (me % nr_cpu_ids)); +set_user_nice (current,MIN_NICE ); + +if (holdoff) +schedule_timeout_idle (holdoff *HZ ); + +do { +if (exp_interval) +udelay (exp_interval); + cur_ops->exp_sync (); +rcu_scale_wait_shutdown (); + }while (!torture_must_stop ()); +torture_kthread_stopping ("rcu_scale_exp" ); +return 0 ; +} +/* * Allocate a writer_mblock structure for the specified rcu_scale_writer * task.
@@ -664,8 +696,10 @@ static void rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) { pr_alert("%s" SCALE_FLAG -"--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n" , - scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, minruntime, nrealreaders, nrealwriters, writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); +"--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d nexp=%d exp_interval=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n" , + scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, + minruntime, nrealreaders, nrealwriters, nrealexp, exp_interval, + writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); }/*
@@ -809,6 +843,13 @@ kfree_scale_cleanup(void)if (torture_cleanup_begin ())return ; +if (exp_tasks) { +for (i =0 ; i < nrealexp; i++) +torture_stop_kthread (rcu_scale_exp, exp_tasks[i]); +kfree (exp_tasks); + exp_tasks =NULL ; + } +if (kfree_reader_tasks) {for (i =0 ; i < kfree_nrealthreads; i++) torture_stop_kthread(kfree_scale_thread,
@@ -903,6 +944,22 @@ kfree_scale_init(void) goto unwind; } +if (nrealexp >0 && cur_ops->exp_sync) { + exp_tasks =kzalloc_objs (exp_tasks[0 ], nrealexp); +if (!exp_tasks) { +SCALEOUT_ERRSTRING ("out of memory" ); + firsterr = -ENOMEM ; + goto unwind; + } +for (i =0 ; i < nrealexp; i++) { + firsterr =torture_create_kthread (rcu_scale_exp, + (void *)i, + exp_tasks[i]); +if (torture_init_error (firsterr)) + goto unwind; + } + } +while (atomic_read (&n_kfree_scale_thread_started) < kfree_nrealthreads)schedule_timeout_uninterruptible (1 );
@@ -959,6 +1016,13 @@ rcu_scale_cleanup(void)return ; } +if (exp_tasks) { +for (i =0 ; i < nrealexp; i++) +torture_stop_kthread (rcu_scale_exp, exp_tasks[i]); +kfree (exp_tasks); + exp_tasks =NULL ; + } +if (reader_tasks) {for (i =0 ; i < nrealreaders; i++) torture_stop_kthread(rcu_scale_reader,
@@ -1076,6 +1140,7 @@ rcu_scale_init(void)if (kthread_tp) kthread_stime = kthread_tp->stime; } + nrealexp = nexp;if (kfree_rcu_test)return kfree_scale_init ();
@@ -1107,6 +1172,21 @@ rcu_scale_init(void) }while (atomic_read (&n_rcu_scale_reader_started) < nrealreaders)schedule_timeout_uninterruptible (1 ); +if (nrealexp >0 && cur_ops->exp_sync) { + exp_tasks =kzalloc_objs (exp_tasks[0 ], nrealexp); +if (!exp_tasks) { +SCALEOUT_ERRSTRING ("out of memory" ); + firsterr = -ENOMEM ; + goto unwind; + } +for (i =0 ; i < nrealexp; i++) { + firsterr =torture_create_kthread (rcu_scale_exp, + (void *)i, + exp_tasks[i]); +if (torture_init_error (firsterr)) + goto unwind; + } + } writer_tasks =kzalloc_objs (writer_tasks[0 ], nrealwriters); writer_durations =kcalloc (nrealwriters,sizeof (*writer_durations),GFP_KERNEL ); writer_n_durations =kzalloc_objs (*writer_n_durations, nrealwriters);
Signature · 1 line
2.52.0
Frederic Weisbecker
maintainerHi,
Quoted 7 lines · Puranjay Mohan
RCU callbacks only track normal grace period sequence numbers. This means callbacks must wait for normal GPs even when expedited GPs have already elapsed.
This series tracks both normal and expedited GP sequences in the callback infrastructure using struct
rcu_gp_oldstate, so callbacks advance when either GP type completes.
What is the motivation behind this? When most callbacks don't have much latency requirements between queue and invocation?
Thanks.
Signature · 2 lines
Frederic Weisbecker SUSE Labs
Frederic Weisbecker
maintainerRe: [RFC PATCH 03/10] rcu/segcblist: Change gp_seq to struct rcu_gp_oldstate gp_seq_full
In reply to Puranjay MohanHi,
kernel/rcu/rcu_segcblist.c static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) · line 499
Quoted 6 lines · Puranjay Mohan
kernel/rcu/rcu_segcblist.c +1
-1@@ -496,7 +496,7 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i)* Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ -void rcu_segcblist_advance (struct rcu_segcblist *rsclp,unsigned long seq) +void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp)
If the name struct rcu_gp_oldstate makes sense when it refers to grace period sequence numbers waited upon by a given segment, it becomes confusing when passed here as the current grace period number to compare against waited upon snapshots.
So how about struct rcu_gp_seq for everyone instead?
Thanks.
Signature · 2 lines
Frederic Weisbecker SUSE Labs
Frederic Weisbecker
maintainerRe: [RFC PATCH 03/10] rcu/segcblist: Change gp_seq to struct rcu_gp_oldstate gp_seq_full
In reply to Puranjay MohanQuoted 33 lines · Puranjay Mohan
This commit renames the ->gp_seq[] field in struct
rcu_segcblistto ->gp_seq_full[] and changes its type from unsigned long to structrcu_gp_oldstate. This prepares the callback tracking infrastructure to support both normal and expedited grace periods.All function signatures are updated to pass struct
rcu_gp_oldstate
pointers:rcu_segcblist_nextgp(),rcu_segcblist_advance(), andrcu_segcblist_accelerate()now take structrcu_gp_oldstate* instead of unsigned long. All callers are updated to use the .rgos_norm field for comparisons and assignments.The SRCU and Tasks RCU wrappers now construct an
rcu_gp_oldstatewith just .rgos_norm set and forward to the core functions.No functional change: only the .rgos_norm field is used in place of
gp_seq.Reviewed-by Paul E. McKenney <[email protected]>Signed-off-by Puranjay Mohan <[email protected]>--- include/linux/rcu_segcblist.h | 2 +- include/trace/events/rcu.h | 5 +++-- kernel/rcu/rcu_segcblist.c | 30 +++++++++++++++++------------- kernel/rcu/rcu_segcblist.h | 6 +++--- kernel/rcu/tree.c | 25 ++++++++++++++----------- kernel/rcu/tree_nocb.h | 29 +++++++++++++++-------------- 6 files changed, 53 insertions(+), 44 deletions(-)include/linux/rcu_segcblist.h +0
-0
include/linux/rcu_segcblist.h struct rcu_cblist { · line 193
Quoted 5 lines
include/linux/rcu_segcblist.h +1
-1@@ -190,7 +190,7 @@ struct rcu_cblist {struct rcu_segcblist {struct rcu_head *head;struct rcu_head **tails[RCU_CBLIST_NSEGS ]; -unsigned long gp_seq[RCU_CBLIST_NSEGS ]; +struct rcu_gp_oldstate gp_seq_full[RCU_CBLIST_NSEGS ];
That could stay as gp_seq.
Quoted 8 lines
include/linux/rcu_segcblist.h +0
-0#ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len;#else kernel/rcu/rcu_segcblist.c +0
-0
kernel/rcu/rcu_segcblist.c static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) · line 500
Quoted 6 lines
kernel/rcu/rcu_segcblist.c +1
-1@@ -496,7 +496,7 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i)* Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) +void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp)
I don't think we need to rename everything to rgos*, especially as it's not as intuitive as gp_seq.
kernel/rcu/rcu_segcblist.c static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) · line 1234
Quoted 9 lines
kernel/rcu/rcu_segcblist.c +1
-1{int i;@@ -1229,7 +1231,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)* Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. */ -rcu_segcblist_advance (&rdp->cblist, rnp->gp_seq); + rgos.rgos_norm = rnp->gp_seq;
Can we shorten that rgos_norm field to just norm?
So the above would parse better as:
gp_seq->norm = rnp->gp_seq
Quoted 9 lines
kernel/rcu/rcu_segcblist.c +1
-0+rcu_segcblist_advance (&rdp->cblist, &rgos);/* Classify any remaining callbacks. */ return rcu_accelerate_cbs (rnp, rdp);kernel/rcu/tree_nocb.h +0
-0
kernel/rcu/tree_nocb.h static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 436
Quoted 5 lines
kernel/rcu/tree_nocb.h +1
-1@@ -433,7 +433,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool lazy) {unsigned long c; -unsigned long cur_gp_seq; +struct rcu_gp_oldstate cur_gp_seq_full;
This could stay as cur_gp_seq.
kernel/rcu/tree_nocb.h static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, · line 505
Quoted 11 lines
kernel/rcu/tree_nocb.h +2
-2unsigned long j = jiffies;long ncbs =rcu_cblist_n_cbs (&rdp->nocb_bypass);long lazy_len =READ_ONCE (rdp->lazy_len);@@ -501,8 +501,8 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,return false;// Caller must enqueue the callback. }if (j != rdp->nocb_gp_adv_time && -rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq) && -rcu_seq_done (&rdp->mynode->gp_seq, cur_gp_seq)) { +rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && +rcu_seq_done (&rdp->mynode->gp_seq, cur_gp_seq_full.rgos_norm)) {
Because cur_gp_seq.norm would parse much better.
kernel/rcu/tree_nocb.h static void nocb_gp_wait(struct rcu_data *my_rdp) · line 662
Quoted 9 lines
kernel/rcu/tree_nocb.h +1
-1rcu_advance_cbs_nowake (rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; }@@ -659,7 +659,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) {bool bypass = false;int __maybe_unused cpu = my_rdp->cpu; -unsigned long cur_gp_seq; +struct rcu_gp_oldstate cur_gp_seq_full;
Ditto
Quoted 15 lines
kernel/rcu/tree_nocb.h +2
-2unsigned long flags;bool gotcbs = false;unsigned long j = jiffies;@@ -730,8 +730,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) needwake_gp = false;if (!rcu_segcblist_restempty (&rdp->cblist,RCU_NEXT_READY_TAIL ) || - (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq) && -rcu_seq_done (&rnp->gp_seq, cur_gp_seq))) { + (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && +rcu_seq_done (&rnp->gp_seq, cur_gp_seq_full.rgos_norm))) {raw_spin_lock_rcu_node (rnp);/* irqs disabled. */ needwake_gp =rcu_advance_cbs (rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist,
kernel/rcu/tree_nocb.h static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) · line 880
Quoted 5 lines
kernel/rcu/tree_nocb.h +1
-1@@ -877,7 +877,7 @@ static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)static void nocb_cb_wait (struct rcu_data *rdp) {struct rcu_segcblist *cblist = &rdp->cblist; -unsigned long cur_gp_seq; +struct rcu_gp_oldstate cur_gp_seq_full;
Ditto.
kernel/rcu/tree_nocb.h static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) · line 883
Quoted 3 lines
kernel/rcu/tree_nocb.h +0
-0unsigned long flags;bool needwake_gp = false;struct rcu_node *rnp = rdp->mynode;
Thanks.
Signature · 2 lines
Frederic Weisbecker SUSE Labs
Frederic Weisbecker
maintainerRe: [RFC PATCH 05/10] rcu: Enable RCU callbacks to benefit from expedited grace periods
In reply to Puranjay MohanQuoted 60 lines · Puranjay Mohan
Currently, RCU callbacks only track normal grace period sequence numbers. This means callbacks must wait for normal grace periods to complete even when expedited grace periods have already elapsed.
This commit uses the full
rcu_gp_oldstatestructure (which tracks both normal and expedited GP sequences) throughout the callback infrastructure.The
rcu_segcblist_advance()function now checks both normal and expedited GP completion viapoll_state_synchronize_rcu_full(), becoming parameterless since it reads the GP state internally.rcu_segcblist_accelerate()stores the full GP state (both normal and expedited sequences) instead of just the normal sequence.The
rcu_accelerate_cbs()andrcu_accelerate_cbs_unlocked()functions useget_state_synchronize_rcu_full()to capture both GP sequences. The NOCB code usespoll_state_synchronize_rcu_full()for advance checks instead of comparing only the normal GP sequence.
srcu_segcblist_advance()become standalone implementations because compares SRCU sequences directly (it cannot usepoll_state_synchronize_rcu_full(), which reads RCU-specific globals).srcu_segcblist_accelerate()setsrgos_exptoRCU_GET_STATE_NOT_TRACKEDsopoll_state_synchronize_rcu_full()only compares thergosp->rgos_normand ignoresrgos_exp.Reviewed-by Paul E. McKenney <[email protected]>Signed-off-by Puranjay Mohan <[email protected]>--- kernel/rcu/rcu_segcblist.c | 30 ++++++++++++++++++++++++------ kernel/rcu/rcu_segcblist.h | 2 +- kernel/rcu/tree.c | 9 +++------ kernel/rcu/tree_nocb.h | 33 +++++++++++++++++++++++---------- 4 files changed, 51 insertions(+), 23 deletions(-)kernel/rcu/rcu_segcblist.c +3
-2@@ -12,6 +12,7 @@#include <linux/kernel.h> #include <linux/types.h> +#include "rcu.h" #include "rcu_segcblist.h" /* Initialize simple callback list. */ @@ -494,9 +495,9 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i)/* * Advance the callbacks in the specified rcu_segcblist structure based -* on the current value passed in for the grace-period counter. +* on the current value of the grace-period counter. */ -void rcu_segcblist_advance (struct rcu_segcblist *rsclp,struct rcu_gp_oldstate *rgosp) +void rcu_segcblist_advance (struct rcu_segcblist *rsclp) {int i;
kernel/rcu/rcu_segcblist.c void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_oldstate * · line 513
Quoted 5 lines
kernel/rcu/rcu_segcblist.c +1
-1@@ -509,7 +510,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_oldstate ** are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i =RCU_WAIT_TAIL ; i <RCU_NEXT_TAIL ; i++) { -if (ULONG_CMP_LT (rgosp->rgos_norm, rsclp->gp_seq_full[i].rgos_norm)) +if (!poll_state_synchronize_rcu_full (&rsclp->gp_seq_full[i]))
Ok that should work but it's worth noting two subtle changes:
- The sequence number to be compared against the segment snapshot one is now always from the root while it often used to be the one from the leaf node. It shouldn't matter I think especially as now the source of the segment snapshot is always either rcu_seq_snap() on the state or get_state_synchronize_rcu_full() and thus rcu_seq_snap() on the state as well.
- But poll_state_synchronize_rcu_full() does two full memory barriers. Hmm probably the one after reading the root state is necessary because we may not be holding the root node lock. But is the first smp_mb() necessary here?
Thanks.
Signature · 2 lines
Frederic Weisbecker SUSE Labs
Frederic Weisbecker
maintainerRe: [RFC PATCH 08/10] rcu: Detect expedited grace period completion in rcu_pending()
In reply to Puranjay MohanQuoted 31 lines · Puranjay Mohan
rcu_pending()is the gatekeeper that decides whetherrcu_core()should run on the current CPU's timer tick. Currently it checks if the CPU has callbacks ready to invoke or a grace period has completed or started.It does not check that an expedited GP has completed. After an expedited GP, callbacks remain in
RCU_WAIT_TAIL(not yet advanced toRCU_DONE_TAIL) and Sorcu_core()never runs to advance them.Add a check using
rcu_segcblist_nextgp()combined withpoll_state_synchronize_rcu_full()to detect when any pending callbacks' grace period has completed.Reviewed-by Paul E. McKenney <[email protected]>Signed-off-by Puranjay Mohan <[email protected]>--- kernel/rcu/tree.c | 7 +++++++ 1 file changed, 7 insertions(+)kernel/rcu/tree.c +1
-0@@ -3671,6 +3671,7 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full);static int rcu_pending (int user) {bool gp_in_progress; +struct rcu_gp_oldstate gp_state;struct rcu_data *rdp =this_cpu_ptr (&rcu_data);struct rcu_node *rnp = rdp->mynode;
kernel/rcu/tree.c static int rcu_pending(int user) · line 3709
Quoted 8 lines
kernel/rcu/tree.c +5
-0@@ -3701,6 +3702,12 @@ static int rcu_pending(int user) rcu_segcblist_ready_cbs(&rdp->cblist))return 1 ; +/* Has a GP (normal or expedited) completed for pending callbacks? */ +if (!rcu_rdp_is_offloaded (rdp) && +rcu_segcblist_nextgp (&rdp->cblist, &gp_state) && +poll_state_synchronize_rcu_full (&gp_state)) +return 1 ;
Do we need the overhead of at least one and at worst two full memory barriers on every ticks that have pending callbacks? I think that there can be a racy check here, some unordered version of poll_state_synchronize_rcu_full() perhaps, and leave the ordering duty to rcu_core().
Thanks.
Quoted 7 lines
kernel/rcu/tree.c +1
-0+/* Has RCU gone idle with this CPU needing another grace period? */ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp) &&Signature · 1 line
2.52.0
Signature · 2 lines
Frederic Weisbecker SUSE Labs
Frederic Weisbecker
maintainerRe: [RFC PATCH 07/10] rcu: Wake NOCB rcuog kthreads on expedited grace period completion
In reply to Puranjay MohanQuoted 51 lines · Puranjay Mohan
When an expedited grace period completes,
rcu_exp_wait_wake()wakes waiters on rnp->exp_wq[] but does not notify NOCB rcuog kthreads. These kthreads may be sleeping waiting for a grace period to complete. Without this wakeup, callbacks on offloaded CPUs that could benefit from the expedited GP must wait until the rcuog kthread wakes for some other reason (e.g., next normal GP completion or a timer).Add
rcu_exp_wake_nocb()which wakes rcuog kthreads for leaf-node CPUs, deduplicating viardp->nocb_gp_rdpsince multiple CPUs share one rcuog kthread. Usesfor_each_leaf_node_possible_cpu()because offline CPUs can have pending callbacks. The function is defined intree_nocb.hwith an empty stub forCONFIG_RCU_NOCB_CPU=nbuilds.Reviewed-by Paul E. McKenney <[email protected]>Signed-off-by Puranjay Mohan <[email protected]>--- kernel/rcu/tree.h | 1 + kernel/rcu/tree_exp.h | 1 + kernel/rcu/tree_nocb.h | 29 +++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+)kernel/rcu/tree.h +1
-0@@ -500,6 +500,7 @@ static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp);static bool wake_nocb_gp (struct rcu_data *rdp); +static void rcu_exp_wake_nocb (struct rcu_node *rnp);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j,bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,kernel/rcu/tree_exp.h +1
-0@@ -708,6 +708,7 @@ static void rcu_exp_wait_wake(unsigned long s) }smp_mb ();/* All above changes before wakeup. */ wake_up_all (&rnp->exp_wq[rcu_seq_ctr (s) &0x3 ]); +rcu_exp_wake_nocb (rnp); }trace_rcu_exp_grace_period (rcu_state.name, s,TPS ("endwake" ));mutex_unlock (&rcu_state.exp_wake_mutex);kernel/rcu/tree_nocb.h +0
-0@@ -190,6 +190,31 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_swait_queue_head (&rnp->nocb_gp_wq[1 ]);
kernel/rcu/tree_nocb.h static void rcu_init_one_nocb(struct rcu_node *rnp) · line 200
Quoted 10 lines
kernel/rcu/tree_nocb.h +8
-0} +/* +* Wake NOCB rcuog kthreads for leaf-node CPUs so that they can advance +* callbacks that were waiting for the just-completed expedited GP. +* Deduplicate via nocb_gp_rdp since multiple CPUs share one rcuog +* kthread. Use for_each_leaf_node_possible_cpu() because offline CPUs +* may have pending callbacks. +*/ +static void rcu_exp_wake_nocb (struct rcu_node *rnp)
Please consolidate the naming to match rcu_nocb_gp_cleanup().
Quoted 5 lines
kernel/rcu/tree_nocb.h +5
-0+{ +struct rcu_data *last_rdp_gp =NULL ; +int cpu; + +if (!rcu_is_leaf_node (rnp))
kernel/rcu/tree_nocb.h static void rcu_init_one_nocb(struct rcu_node *rnp) · line 215
Quoted 10 lines
kernel/rcu/tree_nocb.h +10
-0+return ; + +for_each_leaf_node_possible_cpu (rnp, cpu) { +struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); + +if (rdp->nocb_gp_rdp == last_rdp_gp) +continue ; + last_rdp_gp = rdp->nocb_gp_rdp; +wake_nocb_gp (rdp); + }
There are two waitqueues for rcuog wake-ups:
1) rdp->rdp_gp->nocb_gp_wq: to wait for callbacks on the queue
2) rnp->nocb_gp_wq: to wait for grace periods
So you're waking up the wrong one.
Something like the below? (untested)
kernel/rcu/tree.c +7 -2
@@ -2193,8 +2193,13 @@ static noinline void rcu_gp_cleanup(void)dump_blkd_tasks (rnp,10 );WARN_ON_ONCE (rnp->qsmask);WRITE_ONCE (rnp->gp_seq, new_gp_seq); -if (!rnp->parent) -smp_mb ();// Order against failing poll_state_synchronize_rcu_full(). +if (!rnp->parent) { +/* +* Order against failing poll_state_synchronize_rcu_full(). +* and also rcu_nocb_cleanup_wake() -> swait_active() +*/ +smp_mb (); + } rdp =this_cpu_ptr (&rcu_data);if (rnp == rdp->mynode) needgp =__note_gp_changes (rnp, rdp) || needgp;
kernel/rcu/tree.h +2 -2
@@ -253,7 +253,7 @@ struct rcu_data {u8 nocb_gp_sleep;/* Is the nocb GP thread asleep? */ u8 nocb_gp_bypass;/* Found a bypass on last scan? */ u8 nocb_gp_gp;/* GP to wait for on last scan? */ -unsigned long nocb_gp_seq;/* If so, ->gp_seq to wait for. */ +struct rcu_gp_oldstate nocb_gp_seq;/* If so, ->gp_seq to wait for. */ unsigned long nocb_gp_loops;/* # passes through wait code. */ struct swait_queue_head nocb_gp_wq;/* For nocb kthreads to sleep on. */ bool nocb_cb_sleep;/* Is the nocb CB thread asleep? */
@@ -498,9 +498,9 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t);static void zero_cpu_stall_ticks (struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq); +static void rcu_nocb_exp_cleanup (struct rcu_node *rnp);static void rcu_init_one_nocb (struct rcu_node *rnp);static bool wake_nocb_gp (struct rcu_data *rdp); -static void rcu_exp_wake_nocb (struct rcu_node *rnp);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j,bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,
kernel/rcu/tree_exp.h +2 -1
@@ -708,7 +708,8 @@ static void rcu_exp_wait_wake(unsigned long s) }smp_mb ();/* All above changes before wakeup. */ wake_up_all (&rnp->exp_wq[rcu_seq_ctr (s) &0x3 ]); -rcu_exp_wake_nocb (rnp); +if (rcu_is_leaf_node (rnp)) +rcu_nocb_exp_cleanup (rnp); }trace_rcu_exp_grace_period (rcu_state.name, s,TPS ("endwake" ));mutex_unlock (&rcu_state.exp_wake_mutex);
kernel/rcu/tree_nocb.h +74 -46
213 lines not on this page · raw message
Hi Frederic,
Thanks a lot for the detailed feedback on all patches, I will send the next version soon with requested changes.
Quoted 8 lines · Frederic Weisbecker
Hi,
Quoted 4 lines · Puranjay Mohan
RCU callbacks only track normal grace period sequence numbers. This means callbacks must wait for normal GPs even when expedited GPs have already elapsed.
Quoted 6 lines
Quoted 3 lines
This series tracks both normal and expedited GP sequences in the callback infrastructure using struct
rcu_gp_oldstate, so callbacks advance when either GP type completes.What is the motivation behind this? When most callbacks don't have much latency requirements between queue and invocation?
Right now there's an asymmetry: synchronize_rcu_expedited() callers get fast object freeing, but call_rcu() callers never benefit from those same grace periods even though they prove the exact same thing. This series closes that gap, the callback infrastructure treats a grace period as a grace period regardless of how it was driven. The practical effect is faster memory reclaim: objects that could already be safely freed stop sitting around waiting for the next normal GP. I'll make this motivation clearer in the next version.
Thanks,
Puranjay
Re: [RFC PATCH 07/10] rcu: Wake NOCB rcuog kthreads on expedited grace period completion
In reply to Frederic WeisbeckerHi Frederic,
Sorry for the late reply.
I have sent another version addressing all your comments:
https://lore.kernel.org/all/[email protected]/
Thanks,
Puranjay
Quoted 379 lines · Frederic Weisbecker
Quoted 51 lines · Puranjay Mohan
When an expedited grace period completes,
rcu_exp_wait_wake()wakes waiters on rnp->exp_wq[] but does not notify NOCB rcuog kthreads. These kthreads may be sleeping waiting for a grace period to complete. Without this wakeup, callbacks on offloaded CPUs that could benefit from the expedited GP must wait until the rcuog kthread wakes for some other reason (e.g., next normal GP completion or a timer).Add
rcu_exp_wake_nocb()which wakes rcuog kthreads for leaf-node CPUs, deduplicating viardp->nocb_gp_rdpsince multiple CPUs share one rcuog kthread. Usesfor_each_leaf_node_possible_cpu()because offline CPUs can have pending callbacks. The function is defined intree_nocb.hwith an empty stub forCONFIG_RCU_NOCB_CPU=nbuilds.Reviewed-by Paul E. McKenney <[email protected]>Signed-off-by Puranjay Mohan <[email protected]>--- kernel/rcu/tree.h | 1 + kernel/rcu/tree_exp.h | 1 + kernel/rcu/tree_nocb.h | 29 +++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+)kernel/rcu/tree.h +1
-0@@ -500,6 +500,7 @@ static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq);static void rcu_init_one_nocb (struct rcu_node *rnp);static bool wake_nocb_gp (struct rcu_data *rdp); +static void rcu_exp_wake_nocb (struct rcu_node *rnp);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j,bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,kernel/rcu/tree_exp.h +1
-0@@ -708,6 +708,7 @@ static void rcu_exp_wait_wake(unsigned long s) }smp_mb ();/* All above changes before wakeup. */ wake_up_all (&rnp->exp_wq[rcu_seq_ctr (s) &0x3 ]); +rcu_exp_wake_nocb (rnp); }trace_rcu_exp_grace_period (rcu_state.name, s,TPS ("endwake" ));mutex_unlock (&rcu_state.exp_wake_mutex);kernel/rcu/tree_nocb.h +0
-0@@ -190,6 +190,31 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_swait_queue_head (&rnp->nocb_gp_wq[1 ]);kernel/rcu/tree_nocb.h static void rcu_init_one_nocb(struct rcu_node *rnp) · line 200
Quoted 10 lines
kernel/rcu/tree_nocb.h +8
-0} +/* +* Wake NOCB rcuog kthreads for leaf-node CPUs so that they can advance +* callbacks that were waiting for the just-completed expedited GP. +* Deduplicate via nocb_gp_rdp since multiple CPUs share one rcuog +* kthread. Use for_each_leaf_node_possible_cpu() because offline CPUs +* may have pending callbacks. +*/ +static void rcu_exp_wake_nocb (struct rcu_node *rnp)Please consolidate the naming to match
rcu_nocb_gp_cleanup().Quoted 5 lines
kernel/rcu/tree_nocb.h +5
-0+{ +struct rcu_data *last_rdp_gp =NULL ; +int cpu; + +if (!rcu_is_leaf_node (rnp))kernel/rcu/tree_nocb.h static void rcu_init_one_nocb(struct rcu_node *rnp) · line 215
Quoted 10 lines
kernel/rcu/tree_nocb.h +10
-0+return ; + +for_each_leaf_node_possible_cpu (rnp, cpu) { +struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); + +if (rdp->nocb_gp_rdp == last_rdp_gp) +continue ; + last_rdp_gp = rdp->nocb_gp_rdp; +wake_nocb_gp (rdp); + }There are two waitqueues for rcuog wake-ups:
1)
rdp->rdp_gp->nocb_gp_wq: to wait for callbacks on the queue
2)rnp->nocb_gp_wq: to wait for grace periodsSo you're waking up the wrong one.
Something like the below? (untested)
kernel/rcu/tree.c +7
-2@@ -2193,8 +2193,13 @@ static noinline void rcu_gp_cleanup(void)dump_blkd_tasks (rnp,10 );WARN_ON_ONCE (rnp->qsmask);WRITE_ONCE (rnp->gp_seq, new_gp_seq); -if (!rnp->parent) -smp_mb ();// Order against failing poll_state_synchronize_rcu_full(). +if (!rnp->parent) { +/* +* Order against failing poll_state_synchronize_rcu_full(). +* and also rcu_nocb_cleanup_wake() -> swait_active() +*/ +smp_mb (); + } rdp =this_cpu_ptr (&rcu_data);if (rnp == rdp->mynode) needgp =__note_gp_changes (rnp, rdp) || needgp;kernel/rcu/tree.h +2
-2@@ -253,7 +253,7 @@ struct rcu_data {u8 nocb_gp_sleep;/* Is the nocb GP thread asleep? */ u8 nocb_gp_bypass;/* Found a bypass on last scan? */ u8 nocb_gp_gp;/* GP to wait for on last scan? */ -unsigned long nocb_gp_seq;/* If so, ->gp_seq to wait for. */ +struct rcu_gp_oldstate nocb_gp_seq;/* If so, ->gp_seq to wait for. */ unsigned long nocb_gp_loops;/* # passes through wait code. */ struct swait_queue_head nocb_gp_wq;/* For nocb kthreads to sleep on. */ bool nocb_cb_sleep;/* Is the nocb CB thread asleep? */ @@ -498,9 +498,9 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t);static void zero_cpu_stall_ticks (struct rcu_data *rdp);static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp);static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq); +static void rcu_nocb_exp_cleanup (struct rcu_node *rnp);static void rcu_init_one_nocb (struct rcu_node *rnp);static bool wake_nocb_gp (struct rcu_data *rdp); -static void rcu_exp_wake_nocb (struct rcu_node *rnp);static bool rcu_nocb_flush_bypass (struct rcu_data *rdp,struct rcu_head *rhp,unsigned long j,bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,kernel/rcu/tree_exp.h +2
-1@@ -708,7 +708,8 @@ static void rcu_exp_wait_wake(unsigned long s) }smp_mb ();/* All above changes before wakeup. */ wake_up_all (&rnp->exp_wq[rcu_seq_ctr (s) &0x3 ]); -rcu_exp_wake_nocb (rnp); +if (rcu_is_leaf_node (rnp)) +rcu_nocb_exp_cleanup (rnp); }trace_rcu_exp_grace_period (rcu_state.name, s,TPS ("endwake" ));mutex_unlock (&rcu_state.exp_wake_mutex);kernel/rcu/tree_nocb.h +74
-46@@ -170,13 +170,59 @@ static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)lockdep_assert_held (&rdp->nocb_lock); } +static void rcu_nocb_cleanup_wake (struct swait_queue_head *sq) +{ +if (swait_active (sq)) +swake_up_all (sq); +} +/* * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended * grace period. */ static void rcu_nocb_gp_cleanup (struct swait_queue_head *sq) { -swake_up_all (sq); +/* +* swait active() can be checked first because: +* +* rcu_gp_cleanup() nocb_gp_wait() +* --------------- -------------- +* WRITE_ONCE(root->gp_seq, new_gp_seq); swait_event_interruptible_exclusive(sq) +* smp_mb() prepare_to_swait() +* if swait_active(sq) list_add_tail(&wait->task_list, &q->task_list); +* swake_up_all(sq) set_current_state() +* smp_mb() +* if (poll_state_synchronize_rcu_full()) +* if (rcu_seq_done_exact(&root->gp_seq, rgosp->rgos_norm)) +* ... +*/ +rcu_nocb_cleanup_wake (sq); +} + +/* +* Wake NOCB rcuog kthreads for leaf-node CPUs so that they can advance +* callbacks that were waiting for the just-completed expedited GP. +* Wake-up waitqueues for both even and odd GP numbers because exp and +* normal sequences don't match. +*/ +static void rcu_nocb_exp_cleanup (struct rcu_node *rnp) +{ +/* +* swait active() can be checked first because: +* +* rcu_exp_wait_wake() nocb_gp_wait() +* --------------- -------------- +* rcu_seq_end(&rcu_state.expedited_sequence); swait_event_interruptible_exclusive(sq) +* smp_mb() prepare_to_swait() +* if swait_active(sq) list_add_tail(&wait->task_list, &q->task_list); +* swake_up_all(sq) set_current_state() +* smp_mb() +* if (poll_state_synchronize_rcu_full()) +* if (rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) +* ... +*/ +rcu_nocb_cleanup_wake (&rnp->nocb_gp_wq[0 ]); +rcu_nocb_cleanup_wake (&rnp->nocb_gp_wq[1 ]); }static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp)@@ -190,31 +236,6 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)init_swait_queue_head (&rnp->nocb_gp_wq[1 ]); } -/* -* Wake NOCB rcuog kthreads for leaf-node CPUs so that they can advance -* callbacks that were waiting for the just-completed expedited GP. -* Deduplicate via nocb_gp_rdp since multiple CPUs share one rcuog -* kthread. Use for_each_leaf_node_possible_cpu() because offline CPUs -* may have pending callbacks. -*/ -static void rcu_exp_wake_nocb (struct rcu_node *rnp) -{ -struct rcu_data *last_rdp_gp =NULL ; -int cpu; - -if (!rcu_is_leaf_node (rnp)) -return ; - -for_each_leaf_node_possible_cpu (rnp, cpu) { -struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); - -if (rdp->nocb_gp_rdp == last_rdp_gp) -continue ; - last_rdp_gp = rdp->nocb_gp_rdp; -wake_nocb_gp (rdp); - } -} -/* Clear any pending deferred wakeup timer (nocb_gp_lock must be held). */ static void nocb_defer_wakeup_cancel (struct rcu_data *rdp_gp) {@@ -684,7 +705,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) {bool bypass = false;int __maybe_unused cpu = my_rdp->cpu; -struct rcu_gp_oldstate cur_gp_seq_full; +struct rcu_gp_oldstate wait_gp_seq = {0 };//remove uninitialized warning unsigned long flags;bool gotcbs = false;unsigned long j = jiffies;@@ -694,7 +715,6 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)bool needwake_gp;struct rcu_data *rdp, *rdp_toggling =NULL ;struct rcu_node *rnp; -unsigned long wait_gp_seq =0 ;// Suppress "use uninitialized" warning. bool wasempty = false;/* @@ -718,6 +738,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)* won't be ignored for long. */ list_for_each_entry (rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { +struct rcu_gp_oldstate cur_gp_seq;long bypass_ncbs;bool flush_bypass = false;long lazy_ncbs;@@ -755,8 +776,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) needwake_gp = false;if (!rcu_segcblist_restempty (&rdp->cblist,RCU_NEXT_READY_TAIL ) || - (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && -poll_state_synchronize_rcu_full (&cur_gp_seq_full))) { + (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq) && +poll_state_synchronize_rcu_full (&cur_gp_seq))) {raw_spin_lock_rcu_node (rnp);/* irqs disabled. */ needwake_gp =rcu_advance_cbs (rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist,@@ -777,11 +798,15 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)* numbers from rcu_accelerate_cbs() inside * rcu_advance_cbs() and will be handled on the next pass. */ -if (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq_full) && - !poll_state_synchronize_rcu_full (&cur_gp_seq_full)) { +if (rcu_segcblist_nextgp (&rdp->cblist, &cur_gp_seq) && + !poll_state_synchronize_rcu_full (&cur_gp_seq)) { +if (!needwait_gp || +ULONG_CMP_LT (cur_gp_seq.rgos_norm, wait_gp_seq.rgos_norm)) + wait_gp_seq.rgos_norm = cur_gp_seq.rgos_norm;if (!needwait_gp || -ULONG_CMP_LT (cur_gp_seq_full.rgos_norm, wait_gp_seq)) - wait_gp_seq = cur_gp_seq_full.rgos_norm; +ULONG_CMP_LT (cur_gp_seq.rgos_exp, wait_gp_seq.rgos_exp)) + wait_gp_seq.rgos_exp = cur_gp_seq.rgos_exp; + needwait_gp = true;trace_rcu_nocb_wake (rcu_state.name, rdp->cpu,TPS ("NeedWaitGP" ));@@ -803,7 +828,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_bypass = bypass; my_rdp->nocb_gp_gp = needwait_gp; - my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq :0 ; +if (needwait_gp) + my_rdp->nocb_gp_seq = wait_gp_seq;// At least one child with non-empty ->nocb_bypass, so set // timer in order to avoid stranding its callbacks. @@ -838,12 +864,12 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)nocb_gp_sleep (my_rdp, cpu); }else { rnp = my_rdp->mynode; -trace_rcu_this_gp (rnp, my_rdp, wait_gp_seq,TPS ("StartWait" )); +trace_rcu_this_gp (rnp, my_rdp, wait_gp_seq.rgos_norm,TPS ("StartWait" ));swait_event_interruptible_exclusive ( - rnp->nocb_gp_wq[rcu_seq_ctr (wait_gp_seq) &0x1 ], -rcu_seq_done (&rnp->gp_seq, wait_gp_seq) || + rnp->nocb_gp_wq[rcu_seq_ctr (wait_gp_seq.rgos_norm) &0x1 ], +poll_state_synchronize_rcu_full (&wait_gp_seq) || !READ_ONCE (my_rdp->nocb_gp_sleep)); -trace_rcu_this_gp (rnp, my_rdp, wait_gp_seq,TPS ("EndWait" )); +trace_rcu_this_gp (rnp, my_rdp, wait_gp_seq.rgos_norm,TPS ("EndWait" )); }if (!rcu_nocb_poll) {@@ -877,7 +903,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp)swake_up_one (&rdp_toggling->nocb_state_wq); } - my_rdp->nocb_gp_seq =-1 ; + my_rdp->nocb_gp_seq.rgos_norm =-1 ; + my_rdp->nocb_gp_seq.rgos_exp =-1 ;WARN_ON (signal_pending (current)); }@@ -1561,7 +1588,7 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) {struct rcu_node *rnp = rdp->mynode; - pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu %c CPU %d%s\n", + pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld/%ld rnp %d:%d %lu %c CPU %d%s\n", rdp->cpu, "kK"[!!rdp->nocb_gp_kthread], "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],@@ -1573,7 +1600,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp)".W" [swait_active (&rnp->nocb_gp_wq[1 ])],".B" [!!rdp->nocb_gp_bypass],".G" [!!rdp->nocb_gp_gp], - (long )rdp->nocb_gp_seq, + (long )rdp->nocb_gp_seq.rgos_norm, + (long )rdp->nocb_gp_seq.rgos_exp, rnp->grplo, rnp->grphi,READ_ONCE (rdp->nocb_gp_loops), rdp->nocb_gp_kthread ?task_state_to_char (rdp->nocb_gp_kthread) :'.' , rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,@@ -1684,16 +1712,16 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) { } -static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp) +static void rcu_nocb_exp_cleanup (struct rcu_node *rnp) { -return NULL ; } -static void rcu_init_one_nocb (struct rcu_node *rnp) +static struct swait_queue_head *rcu_nocb_gp_get (struct rcu_node *rnp) { +return NULL ; } -static void rcu_exp_wake_nocb (struct rcu_node *rnp) +static void rcu_init_one_nocb (struct rcu_node *rnp) { }