SRCU updates for v6.12
Subsystems (2)
Neeraj Upadhyay
maintainercover SRCU updates for v6.12
Hello,
Following are the SRCU updates for v6.12:
1. Faster gp seq wrap-around, courtesy of JP Kobryn.
2. Check for concurrent updates of heuristics, courtesy of
Paul E. McKenney.
3. Mark callbacks not currently participating in barrier operation,
courtesy of Paul E. McKenney.
Git tree: https://git.kernel.org/pub/scm/linux/kernel/git/neeraj.upadhyay/linux-rcu.git/log/?h=srcu.12.08.24a
- Neeraj
------------------------------------------------------------------------
include/linux/rcupdate.h | 3 +++ include/linux/srcutree.h | 15 ++++++++++++++- kernel/rcu/rcu.h | 3 --- kernel/rcu/srcutree.c | 11 ++++++++--- 4 files changed, 25 insertions(+), 7 deletions(-)
[email protected]
maintainerPATCH 1/3 srcu: faster gp seq wrap-around
From: JP Kobryn <[email protected]>
Using a higher value for the initial gp sequence counters allows for wrapping to occur faster. It can help with surfacing any issues that may be happening as a result of the wrap around.
--- include/linux/rcupdate.h | 3 +++ include/linux/srcutree.h | 15 ++++++++++++++- kernel/rcu/rcu.h | 3 --- kernel/rcu/srcutree.c | 7 ++++--- 4 files changed, 21 insertions(+), 7 deletions(-)
include/linux/rcupdate.h +3 -0
@@ -34,6 +34,9 @@#define ULONG_CMP_GE (a, b) (ULONG_MAX / 2 >= (a) - (b))#define ULONG_CMP_LT (a, b) (ULONG_MAX / 2 < (a) - (b)) +#define RCU_SEQ_CTR_SHIFT 2 +#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) +/* Exported common interfaces */ void call_rcu (struct rcu_head *head,rcu_callback_t func);void rcu_barrier_tasks (void );
include/linux/srcutree.h +14 -1
@@ -129,10 +129,23 @@ struct srcu_struct {#define SRCU_STATE_SCAN1 1#define SRCU_STATE_SCAN2 2 +/* +* Values for initializing gp sequence fields. Higher values allow wrap arounds to +* occur earlier. +* The second value with state is useful in the case of static initialization of +* srcu_usage where srcu_gp_seq_needed is expected to have some state value in its +* lower bits (or else it will appear to be already initialized within +* the call check_init_srcu_struct()). +*/ +#define SRCU_GP_SEQ_INITIAL_VAL ((0UL - 100UL) << RCU_SEQ_CTR_SHIFT) +#define SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE (SRCU_GP_SEQ_INITIAL_VAL - 1) +#define __SRCU_USAGE_INIT (name) \ { \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ - .srcu_gp_seq_needed = -1UL, \ + .srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL, \ + .srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE, \ + .srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL, \ .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \ }
kernel/rcu/rcu.h +0 -3
@@ -54,9 +54,6 @@* grace-period sequence number. */ -#define RCU_SEQ_CTR_SHIFT 2 -#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) -/* Low-order bit definition for polled grace-period APIs. */ #define RCU_GET_STATE_COMPLETED 0x1
kernel/rcu/srcutree.c +4 -3
@@ -247,7 +247,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)mutex_init (&ssp->srcu_sup->srcu_cb_mutex);mutex_init (&ssp->srcu_sup->srcu_gp_mutex); ssp->srcu_idx =0 ; - ssp->srcu_sup->srcu_gp_seq =0 ; + ssp->srcu_sup->srcu_gp_seq =SRCU_GP_SEQ_INITIAL_VAL ; ssp->srcu_sup->srcu_barrier_seq =0 ;mutex_init (&ssp->srcu_sup->srcu_barrier_mutex);atomic_set (&ssp->srcu_sup->srcu_barrier_cpu_cnt,0 );
@@ -258,7 +258,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)if (!ssp->sda) goto err_free_sup;init_srcu_struct_data (ssp); - ssp->srcu_sup->srcu_gp_seq_needed_exp =0 ; + ssp->srcu_sup->srcu_gp_seq_needed_exp =SRCU_GP_SEQ_INITIAL_VAL ; ssp->srcu_sup->srcu_last_gp_end =ktime_get_mono_fast_ns ();if (READ_ONCE (ssp->srcu_sup->srcu_size_state) ==SRCU_SIZE_SMALL &&SRCU_SIZING_IS_INIT ()) {if (!init_srcu_struct_nodes (ssp,GFP_ATOMIC ))
@@ -266,7 +266,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)WRITE_ONCE (ssp->srcu_sup->srcu_size_state,SRCU_SIZE_BIG ); } ssp->srcu_sup->srcu_ssp = ssp; -smp_store_release (&ssp->srcu_sup->srcu_gp_seq_needed,0 );/* Init done. */ +smp_store_release (&ssp->srcu_sup->srcu_gp_seq_needed, +SRCU_GP_SEQ_INITIAL_VAL );/* Init done. */ return 0 ; err_free_sda:
Signature · 1 line
2.40.1
[email protected]
maintainerPATCH 2/3 srcu: Check for concurrent updates of heuristics
In reply to Neeraj UpadhyayFrom: "Paul E. McKenney" <[email protected]>
SRCU maintains the ->srcu_n_exp_nodelay and ->reschedule_count values to guide heuristics governing auto-expediting of normal SRCU grace periods and grace-period-state-machine delays. This commit adds KCSAN ASSERT_EXCLUSIVE_WRITER() calls to check for concurrent updates to these fields.
--- kernel/rcu/srcutree.c | 2 ++ 1 file changed, 2 insertions(+)
kernel/rcu/srcutree.c +2 -0
@@ -629,6 +629,7 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)if (time_after (j, gpstart)) jbase += j - gpstart;if (!jbase) { +ASSERT_EXCLUSIVE_WRITER (sup->srcu_n_exp_nodelay);WRITE_ONCE (sup->srcu_n_exp_nodelay,READ_ONCE (sup->srcu_n_exp_nodelay) +1 );if (READ_ONCE (sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase) jbase =1 ;
@@ -1819,6 +1820,7 @@ static void process_srcu(struct work_struct *work) }else { j = jiffies;if (READ_ONCE (sup->reschedule_jiffies) == j) { +ASSERT_EXCLUSIVE_WRITER (sup->reschedule_count);WRITE_ONCE (sup->reschedule_count,READ_ONCE (sup->reschedule_count) +1 );if (READ_ONCE (sup->reschedule_count) > srcu_max_nodelay) curdelay =1 ;
Signature · 1 line
2.40.1
[email protected]
maintainerPATCH 3/3 srcu: Mark callbacks not currently participating in barrier operation
In reply to Neeraj UpadhyayFrom: "Paul E. McKenney" <[email protected]>
SRCU keeps a count of the number of callbacks that the current srcu_barrier() is waiting on, but there is currently no easy way to work out which callback is stuck. One way to do this is to mark idle SRCU-barrier callbacks by making the ->next pointer point to the callback itself, and this commit does just that.
Later commits will use this for debug output.
--- kernel/rcu/srcutree.c | 2 ++ 1 file changed, 2 insertions(+)
kernel/rcu/srcutree.c +2 -0
@@ -137,6 +137,7 @@ static void init_srcu_struct_data(struct srcu_struct *ssp) sdp->srcu_cblist_invoking = false; sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq; sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq; + sdp->srcu_barrier_head.next = &sdp->srcu_barrier_head; sdp->mynode =NULL ; sdp->cpu = cpu;INIT_WORK (&sdp->work, srcu_invoke_callbacks);
@@ -1562,6 +1563,7 @@ static void srcu_barrier_cb(struct rcu_head *rhp)struct srcu_data *sdp;struct srcu_struct *ssp; + rhp->next = rhp;// Mark the callback as having been invoked. sdp = container_of(rhp, struct srcu_data, srcu_barrier_head); ssp = sdp->ssp;if (atomic_dec_and_test (&ssp->srcu_sup->srcu_barrier_cpu_cnt))
Signature · 1 line
2.40.1