srcu: faster srcu gp seq wrap-around
Subsystems (2)
PATCH 1/1 srcu: faster srcu gp seq wrap-around
Using a higher value for the initial gp sequence counter values allows for wrapping to occur faster. It can help with surfacing any issues that may be happening as a result of the wrap around.
--- include/linux/rcupdate.h | 3 +++ include/linux/srcutree.h | 15 ++++++++++++++- kernel/rcu/rcu.h | 3 --- kernel/rcu/srcutree.c | 6 ++++-- 4 files changed, 21 insertions(+), 6 deletions(-)
include/linux/rcupdate.h +3 -0
@@ -35,6 +35,9 @@#define ULONG_CMP_GE (a, b) (ULONG_MAX / 2 >= (a) - (b))#define ULONG_CMP_LT (a, b) (ULONG_MAX / 2 < (a) - (b)) +#define RCU_SEQ_CTR_SHIFT 2 +#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) +/* Exported common interfaces */ void call_rcu (struct rcu_head *head,rcu_callback_t func);void rcu_barrier_tasks (void );
include/linux/srcutree.h +14 -1
@@ -129,10 +129,23 @@ struct srcu_struct {#define SRCU_STATE_SCAN1 1#define SRCU_STATE_SCAN2 2 +/* +* Values for initializing gp sequence fields. Higher values allow wrap arounds to +* occur earlier. +* The second value with state is useful in the case of static initialization of +* srcu_usage where srcu_gp_seq_needed is expected to have some state value in its +* lower bits (or else it will appear to be already initialized within +* the call check_init_srcu_struct()). +*/ +#define SRCU_GP_SEQ_INITIAL_VAL ((0UL - 100UL) << RCU_SEQ_CTR_SHIFT) +#define SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE (SRCU_GP_SEQ_INITIAL_VAL - 1) +#define __SRCU_USAGE_INIT (name) \ { \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ - .srcu_gp_seq_needed = -1UL, \ + .srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL, \ + .srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE, \ + .srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL, \ .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \ }
kernel/rcu/rcu.h +0 -3
@@ -54,9 +54,6 @@* grace-period sequence number. */ -#define RCU_SEQ_CTR_SHIFT 2 -#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) -/* Low-order bit definition for polled grace-period APIs. */ #define RCU_GET_STATE_COMPLETED 0x1
kernel/rcu/srcutree.c +4 -2
@@ -247,7 +247,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)mutex_init (&ssp->srcu_sup->srcu_cb_mutex);mutex_init (&ssp->srcu_sup->srcu_gp_mutex); ssp->srcu_idx =0 ; - ssp->srcu_sup->srcu_gp_seq =0 ; + ssp->srcu_sup->srcu_gp_seq =SRCU_GP_SEQ_INITIAL_VAL ; ssp->srcu_sup->srcu_barrier_seq =0 ;mutex_init (&ssp->srcu_sup->srcu_barrier_mutex);atomic_set (&ssp->srcu_sup->srcu_barrier_cpu_cnt,0 );
@@ -258,7 +258,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)if (!ssp->sda) goto err_free_sup;init_srcu_struct_data (ssp); - ssp->srcu_sup->srcu_gp_seq_needed_exp =0 ; + ssp->srcu_sup->srcu_gp_seq_needed_exp =SRCU_GP_SEQ_INITIAL_VAL ; ssp->srcu_sup->srcu_last_gp_end =ktime_get_mono_fast_ns ();if (READ_ONCE (ssp->srcu_sup->srcu_size_state) ==SRCU_SIZE_SMALL &&SRCU_SIZING_IS_INIT ()) {if (!init_srcu_struct_nodes (ssp,GFP_ATOMIC ))
@@ -267,6 +267,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) } ssp->srcu_sup->srcu_ssp = ssp;smp_store_release (&ssp->srcu_sup->srcu_gp_seq_needed,0 );/* Init done. */ +smp_store_release (&ssp->srcu_sup->srcu_gp_seq_needed, +SRCU_GP_SEQ_INITIAL_VAL );/* Init done. */ return 0 ; err_free_sda:
Signature · 1 line
2.45.2
Neeraj Upadhyay
maintainerHi,
Quoted 39 lines · JP Kobryn
Using a higher value for the initial gp sequence counter values allows for wrapping to occur faster. It can help with surfacing any issues that may be happening as a result of the wrap around.
Signed-off-by JP Kobryn <[email protected]>--- include/linux/rcupdate.h | 3 +++ include/linux/srcutree.h | 15 ++++++++++++++- kernel/rcu/rcu.h | 3 --- kernel/rcu/srcutree.c | 6 ++++-- 4 files changed, 21 insertions(+), 6 deletions(-)include/linux/rcupdate.h +3
-0@@ -35,6 +35,9 @@#define ULONG_CMP_GE (a, b) (ULONG_MAX / 2 >= (a) - (b))#define ULONG_CMP_LT (a, b) (ULONG_MAX / 2 < (a) - (b)) +#define RCU_SEQ_CTR_SHIFT 2 +#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) +/* Exported common interfaces */ void call_rcu (struct rcu_head *head,rcu_callback_t func);void rcu_barrier_tasks (void );include/linux/srcutree.h +5
-0@@ -129,10 +129,23 @@ struct srcu_struct {#define SRCU_STATE_SCAN1 1#define SRCU_STATE_SCAN2 2 +/* +* Values for initializing gp sequence fields. Higher values allow wrap arounds to +* occur earlier. +* The second value with state is useful in the case of static initialization of +* srcu_usage where srcu_gp_seq_needed is expected to have some state value in its
include/linux/srcutree.h struct srcu_struct { · line 135
Quoted 10 lines
include/linux/srcutree.h +6
-1+* lower bits (or else it will appear to be already initialized within +* the call check_init_srcu_struct()). +*/ +#define SRCU_GP_SEQ_INITIAL_VAL ((0UL - 100UL) << RCU_SEQ_CTR_SHIFT) +#define SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE (SRCU_GP_SEQ_INITIAL_VAL - 1) +#define __SRCU_USAGE_INIT (name) \ { \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ - .srcu_gp_seq_needed = -1UL, \
As srcu_gp_seq* counters are initialized before first use, if we do not modify this file, we can move the SRCU_GP_SEQ_INITIAL_VAL and SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE macros to srcutree.c. We then also don't need to move RCU_SEQ_CTR_SHIFT and RCU_SEQ_STATE_MASK from rcu.h (keeping them internal to RCU). Is there a scenario where that won't work?
Quoted 43 lines
include/linux/srcutree.h +3
-0+ .srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL, \ + .srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE, \ + .srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL, \ .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \ }kernel/rcu/rcu.h +0
-3@@ -54,9 +54,6 @@* grace-period sequence number. */ -#define RCU_SEQ_CTR_SHIFT 2 -#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) -/* Low-order bit definition for polled grace-period APIs. */ #define RCU_GET_STATE_COMPLETED 0x1kernel/rcu/srcutree.c +2
-2@@ -247,7 +247,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)mutex_init (&ssp->srcu_sup->srcu_cb_mutex);mutex_init (&ssp->srcu_sup->srcu_gp_mutex); ssp->srcu_idx =0 ; - ssp->srcu_sup->srcu_gp_seq =0 ; + ssp->srcu_sup->srcu_gp_seq =SRCU_GP_SEQ_INITIAL_VAL ; ssp->srcu_sup->srcu_barrier_seq =0 ;mutex_init (&ssp->srcu_sup->srcu_barrier_mutex);atomic_set (&ssp->srcu_sup->srcu_barrier_cpu_cnt,0 );@@ -258,7 +258,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)if (!ssp->sda) goto err_free_sup;init_srcu_struct_data (ssp); - ssp->srcu_sup->srcu_gp_seq_needed_exp =0 ; + ssp->srcu_sup->srcu_gp_seq_needed_exp =SRCU_GP_SEQ_INITIAL_VAL ; ssp->srcu_sup->srcu_last_gp_end =ktime_get_mono_fast_ns ();if (READ_ONCE (ssp->srcu_sup->srcu_size_state) ==SRCU_SIZE_SMALL &&SRCU_SIZING_IS_INIT ()) {if (!init_srcu_struct_nodes (ssp,GFP_ATOMIC ))
kernel/rcu/srcutree.c static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) · line 269
Quoted 3 lines
kernel/rcu/srcutree.c +0
-0@@ -267,6 +267,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) } ssp->srcu_sup->srcu_ssp = ssp;smp_store_release (&ssp->srcu_sup->srcu_gp_seq_needed,0 );/* Init done. */
Remove this line?
- Neeraj
Quoted 7 lines
kernel/rcu/srcutree.c +2
-0+smp_store_release (&ssp->srcu_sup->srcu_gp_seq_needed, +SRCU_GP_SEQ_INITIAL_VAL );/* Init done. */ return 0 ; err_free_sda:Signature · 1 line
2.45.2