SRCU updates, including SRCU-fast
Subsystems (6)
Paul E. McKenney
maintainercover SRCU updates, including SRCU-fast
Hello!
This series contains SRCU updates, perhaps most notably the addition of srcu_read_lock_fast() and srcu_read_unlock_fast(), which on my laptop are about 20% faster than rcu_read_lock_trace() and rcu_read_unlock_trace(). There are of course drawbacks:
o Lack of CPU stall warnings.
o SRCU-fast readers permitted only where rcu_is_watching().
o A pointer-sized return value from srcu_read_lock_fast() must
be passed to the corresponding srcu_read_unlock_fast().
o In the absence of readers, a synchronize_srcu() having _fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.
Whether or not this can replace RCU Tasks Trace, it should replace the SRCU-lite API, where the only drawback is a pointer-sized return value from srcu_read_lock_fast() compared to the int-sized return value from srcu_read_lock_lite(). And yes, I should have thought harder before doing that SRCU-lite...
The series is as follows:
1. Make Tiny SRCU able to operate in preemptible kernels.
2. Define SRCU_READ_FLAVOR_ALL in terms of symbols.
3. Use ->srcu_gp_seq for rcutorture reader batch.
4. Pull ->srcu_{un,}lock_count into a new srcu_ctr structure.
5. Make SRCU readers use ->srcu_ctrs for counter selection.
6. Make Tree SRCU updates independent of ->srcu_idx.
7. Force synchronization for srcu_get_delay().
8. Rename srcu_check_read_flavor_lite() to
srcu_check_read_flavor_force().
9. Add SRCU_READ_FLAVOR_SLOWGP to flag need for synchronize_rcu().
10. Pull pointer-to-integer conversion into __srcu_ptr_to_ctr().
11. Pull integer-to-pointer conversion into __srcu_ctr_to_ptr().
12. Move SRCU Tree/Tiny definitions from srcu.h.
13. Add SRCU-fast readers.
14. Add ability to test srcu_read_{,un}lock_fast().
15. Add srcu_read_lock_fast() support using "srcu-fast".
16. Make scenario SRCU-P use srcu_read_lock_fast().
17. Fix srcu_read_unlock_{lite,nmisafe}() kernel-doc.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/srcu.h | 3 b/include/linux/srcutiny.h | 2 b/include/linux/srcutree.h | 13 b/kernel/rcu/rcu.h | 9 b/kernel/rcu/rcutorture.c | 2 b/kernel/rcu/refscale.c | 32 + b/kernel/rcu/srcutiny.c | 6 b/kernel/rcu/srcutree.c | 2 b/tools/testing/selftests/rcutorture/configs/rcu/SRCU-P.boot | 1 include/linux/srcu.h | 66 ++- include/linux/srcutiny.h | 25 + include/linux/srcutree.h | 77 +++ kernel/rcu/rcutorture.c | 9 kernel/rcu/srcutree.c | 233 +++++------ 14 files changed, 332 insertions(+), 148 deletions(-)
Paul E. McKenney
maintainerPATCH 1/17 srcu: Make Tiny SRCU able to operate in preemptible kernels
Given that SRCU allows its read-side critical sections are not just preemptible, but also allow general blocking, there is not much reason to restrict Tiny SRCU to non-preemptible kernels. This commit therefore removes Tiny SRCU dependencies on non-preemptibility, primarily surrounding its interaction with rcutorture and early boot.
--- kernel/rcu/rcu.h | 9 ++++++--- kernel/rcu/srcutiny.c | 6 ++++++ 2 files changed, 12 insertions(+), 3 deletions(-)
kernel/rcu/rcu.h +6 -3
@@ -611,8 +611,6 @@ void srcutorture_get_gp_data(struct srcu_struct *sp, int *flags,static inline bool rcu_watching_zero_in_eqs (int cpu,int *vp) {return false; }static inline unsigned long rcu_get_gp_seq (void ) {return 0 ; }static inline unsigned long rcu_exp_batches_completed (void ) {return 0 ; } -static inline unsigned long -srcu_batches_completed (struct srcu_struct *sp) {return 0 ; }static inline void rcu_force_quiescent_state (void ) { }static inline bool rcu_check_boost_fail (unsigned long gp_state,int *cpup) {return true; }static inline void show_rcu_gp_kthreads (void ) { }
@@ -624,7 +622,6 @@ static inline void rcu_gp_slow_unregister(atomic_t *rgssp) { }bool rcu_watching_zero_in_eqs (int cpu,int *vp);unsigned long rcu_get_gp_seq (void );unsigned long rcu_exp_batches_completed (void ); -unsigned long srcu_batches_completed (struct srcu_struct *sp);bool rcu_check_boost_fail (unsigned long gp_state,int *cpup);void show_rcu_gp_kthreads (void );int rcu_get_gp_kthreads_prio (void );
@@ -636,6 +633,12 @@ void rcu_gp_slow_register(atomic_t *rgssp);void rcu_gp_slow_unregister (atomic_t *rgssp);#endif /* #else #ifdef CONFIG_TINY_RCU */ +#ifdef CONFIG_TINY_SRCU +static inline unsigned long srcu_batches_completed (struct srcu_struct *sp) {return 0 ; } +#else // #ifdef CONFIG_TINY_SRCU +unsigned long srcu_batches_completed (struct srcu_struct *sp); +#endif // #else // #ifdef CONFIG_TINY_SRCU +#ifdef CONFIG_RCU_NOCB_CPU void rcu_bind_current_to_nocb (void );#else
kernel/rcu/srcutiny.c +6 -0
@@ -20,7 +20,11 @@#include "rcu_segcblist.h" #include "rcu.h" +#ifndef CONFIG_TREE_RCU int rcu_scheduler_active__read_mostly ; +#else // #ifndef CONFIG_TREE_RCU +extern int rcu_scheduler_active; +#endif // #else // #ifndef CONFIG_TREE_RCU static LIST_HEAD (srcu_boot_list);static bool srcu_init_done;
@@ -282,11 +286,13 @@ bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) }EXPORT_SYMBOL_GPL (poll_state_synchronize_srcu); +#ifndef CONFIG_TREE_RCU /* Lockdep diagnostics. */ void __init rcu_scheduler_starting (void ) { rcu_scheduler_active =RCU_SCHEDULER_RUNNING ; } +#endif // #ifndef CONFIG_TREE_RCU /* * Queue work for srcu_struct structures with early boot callbacks.
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 2/17 srcu: Define SRCU_READ_FLAVOR_ALL in terms of symbols
In reply to Paul E. McKenneyThis commit defines SRCU_READ_FLAVOR_ALL in terms of the SRCU_READ_FLAVOR_* definitions instead of a hexadecimal constant.
--- include/linux/srcu.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/linux/srcu.h +2 -1
@@ -47,7 +47,8 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). -#define SRCU_READ_FLAVOR_ALL 0x7 // All of the above. +#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ + SRCU_READ_FLAVOR_LITE) // All of the above.#ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h>
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 3/17 srcu: Use ->srcu_gp_seq for rcutorture reader batch
In reply to Paul E. McKenneyThis commit stops using ->srcu_idx for rcutorture's reader-batch consistency checking, using ->srcu_gp_seq instead. This is a first step towards a faster srcu_read_{,un}lock_lite() that avoids the array accesses that use ->srcu_idx.
--- kernel/rcu/rcutorture.c | 2 ++ kernel/rcu/srcutree.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
kernel/rcu/rcutorture.c +2 -0
@@ -791,6 +791,7 @@ static struct rcu_torture_ops srcu_ops = { .readunlock = srcu_torture_read_unlock, .readlock_held = torture_srcu_read_lock_held, .get_gp_seq = srcu_torture_completed, + .gp_diff = rcu_seq_diff, .deferred_free = srcu_torture_deferred_free, .sync = srcu_torture_synchronize, .exp_sync = srcu_torture_synchronize_expedited,
@@ -834,6 +835,7 @@ static struct rcu_torture_ops srcud_ops = { .readunlock = srcu_torture_read_unlock, .readlock_held = torture_srcu_read_lock_held, .get_gp_seq = srcu_torture_completed, + .gp_diff = rcu_seq_diff, .deferred_free = srcu_torture_deferred_free, .sync = srcu_torture_synchronize, .exp_sync = srcu_torture_synchronize_expedited,
kernel/rcu/srcutree.c +1 -1
@@ -1679,7 +1679,7 @@ EXPORT_SYMBOL_GPL(srcu_barrier);*/ unsigned long srcu_batches_completed (struct srcu_struct *ssp) { -return READ_ONCE (ssp->srcu_idx); +return READ_ONCE (ssp->srcu_sup->srcu_gp_seq); }EXPORT_SYMBOL_GPL (srcu_batches_completed);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 4/17 srcu: Pull ->srcu_{un,}lock_count into a new srcu_ctr structure
In reply to Paul E. McKenneyThis commit prepares for array-index-free srcu_read_lock*() by moving the ->srcu_{un,}lock_count fields into a new srcu_ctr structure. This will permit ->srcu_index to be replaced by a per-CPU pointer to this structure.
--- include/linux/srcutree.h | 13 +++-- kernel/rcu/srcutree.c | 115 +++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 62 deletions(-)
include/linux/srcutree.h +9 -4
@@ -17,14 +17,19 @@struct srcu_node ;struct srcu_struct ; +/* One element of the srcu_data srcu_ctrs array. */ +struct srcu_ctr { +atomic_long_t srcu_locks;/* Locks per CPU. */ +atomic_long_t srcu_unlocks;/* Unlocks per CPU. */ +}; +/* * Per-CPU structure feeding into leaf srcu_node, similar in function * to rcu_node. */ struct srcu_data {/* Read-side state. */ -atomic_long_t srcu_lock_count[2 ];/* Locks per CPU. */ -atomic_long_t srcu_unlock_count[2 ];/* Unlocks per CPU. */ +struct srcu_ctr srcu_ctrs[2 ];/* Locks and unlocks per CPU. */ int srcu_reader_flavor;/* Reader flavor for srcu_struct structure? */ /* Values: SRCU_READ_FLAVOR_.* */
@@ -221,7 +226,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_lite()." ); idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -this_cpu_inc (ssp->sda->srcu_lock_count[idx].counter);/* Y */ +this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_locks.counter);/* Y */ barrier ();/* Avoid leaking the critical section. */ return idx; }
@@ -240,7 +245,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)static inline void __srcu_read_unlock_lite (struct srcu_struct *ssp,int idx) {barrier ();/* Avoid leaking the critical section. */ -this_cpu_inc (ssp->sda->srcu_unlock_count[idx].counter);/* Z */ +this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter);/* Z */ RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_lite()." ); }
kernel/rcu/srcutree.c +57 -58
221 lines not on this page · raw message
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 5/17 srcu: Make SRCU readers use ->srcu_ctrs for counter selection
In reply to Paul E. McKenneyThis commit causes SRCU readers to use ->srcu_ctrs for counter selection instead of ->srcu_idx. This takes another step towards array-indexing-free SRCU readers.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/srcutree.h | 9 +++++---- kernel/rcu/srcutree.c | 23 +++++++++++++---------- 2 files changed, 18 insertions(+), 14 deletions(-)
include/linux/srcutree.h +5 -4
@@ -101,6 +101,7 @@ struct srcu_usage {*/ struct srcu_struct {unsigned int srcu_idx;/* Current rdr array element. */ +struct srcu_ctr __percpu *srcu_ctrp;struct srcu_data __percpu *sda;/* Per-CPU srcu_data array. */ struct lockdep_map dep_map;struct srcu_usage *srcu_sup;/* Update-side data. */
@@ -167,6 +168,7 @@ struct srcu_struct {#define __SRCU_STRUCT_INIT (name, usage_name, pcpu_name) \ { \ .sda = &pcpu_name, \ + .srcu_ctrp = &pcpu_name.srcu_ctrs[0], \ __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ }
@@ -222,13 +224,12 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf);*/ static inline int __srcu_read_lock_lite (struct srcu_struct *ssp) { -int idx; +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp);RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_lite()." ); - idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_locks.counter);/* Y */ +this_cpu_inc (scp->srcu_locks.counter);/* Y */ barrier ();/* Avoid leaking the critical section. */ -return idx; +return scp - &ssp->sda->srcu_ctrs[0 ]; }/*
kernel/rcu/srcutree.c +13 -10
@@ -253,8 +253,10 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)atomic_set (&ssp->srcu_sup->srcu_barrier_cpu_cnt,0 );INIT_DELAYED_WORK (&ssp->srcu_sup->work, process_srcu); ssp->srcu_sup->sda_is_static = is_static; -if (!is_static) +if (!is_static) { ssp->sda = alloc_percpu(struct srcu_data); + ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0 ]; + }if (!ssp->sda) goto err_free_sup;init_srcu_struct_data (ssp);
@@ -742,12 +744,11 @@ EXPORT_SYMBOL_GPL(__srcu_check_read_flavor);*/ int __srcu_read_lock (struct srcu_struct *ssp) { -int idx; +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); - idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_locks.counter); +this_cpu_inc (scp->srcu_locks.counter);smp_mb ();/* B */ /* Avoid leaking the critical section. */ -return idx; +return scp - &ssp->sda->srcu_ctrs[0 ]; }EXPORT_SYMBOL_GPL (__srcu_read_lock);
@@ -772,13 +773,12 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock);*/ int __srcu_read_lock_nmisafe (struct srcu_struct *ssp) { -int idx; -struct srcu_data *sdp =raw_cpu_ptr (ssp->sda); +struct srcu_ctr __percpu *scpp =READ_ONCE (ssp->srcu_ctrp); +struct srcu_ctr *scp =raw_cpu_ptr (scpp); - idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -atomic_long_inc (&sdp->srcu_ctrs[idx].srcu_locks); +atomic_long_inc (&scp->srcu_locks);smp_mb__after_atomic ();/* B */ /* Avoid leaking the critical section. */ -return idx; +return scpp - &ssp->sda->srcu_ctrs[0 ]; }EXPORT_SYMBOL_GPL (__srcu_read_lock_nmisafe);
@@ -1152,6 +1152,8 @@ static void srcu_flip(struct srcu_struct *ssp)smp_mb ();/* E */ /* Pairs with B and C. */ WRITE_ONCE (ssp->srcu_idx, ssp->srcu_idx +1 );// Flip the counter. +WRITE_ONCE (ssp->srcu_ctrp, + &ssp->sda->srcu_ctrs[!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0 ])]);/* * Ensure that if the updater misses an __srcu_read_unlock()
@@ -2004,6 +2006,7 @@ static int srcu_module_coming(struct module *mod) ssp->sda = alloc_percpu(struct srcu_data);if (WARN_ON_ONCE (!ssp->sda))return -ENOMEM ; + ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0 ]; }return 0 ; }
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 6/17 srcu: Make Tree SRCU updates independent of ->srcu_idx
In reply to Paul E. McKenneyThis commit makes Tree SRCU updates independent of ->srcu_idx, then drop ->srcu_idx.
--- include/linux/srcutree.h | 1 - kernel/rcu/srcutree.c | 68 ++++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 35 deletions(-)
include/linux/srcutree.h +0 -1
@@ -100,7 +100,6 @@ struct srcu_usage {* Per-SRCU-domain structure, similar in function to rcu_state. */ struct srcu_struct { -unsigned int srcu_idx;/* Current rdr array element. */ struct srcu_ctr __percpu *srcu_ctrp;struct srcu_data __percpu *sda;/* Per-CPU srcu_data array. */ struct lockdep_map dep_map;
kernel/rcu/srcutree.c +34 -34
158 lines not on this page · raw message
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 7/17 srcu: Force synchronization for srcu_get_delay()
In reply to Paul E. McKenneyCurrently, srcu_get_delay() can be called concurrently, for example, by a CPU that is the first to request a new grace period and the CPU processing the current grace period. Although concurrent access is harmless, it unnecessarily expands the state space. Additionally, all calls to srcu_get_delay() are from slow paths.
This commit therefore protects all calls to srcu_get_delay() with ssp->srcu_sup->lock, which is already held on the invocation from the srcu_funnel_gp_start() function. While in the area, this commit also adds a lockdep_assert_held() to srcu_get_delay() itself.
--- kernel/rcu/srcutree.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
kernel/rcu/srcutree.c +10 -1
@@ -648,6 +648,7 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)unsigned long jbase =SRCU_INTERVAL ;struct srcu_usage *sup = ssp->srcu_sup; +lockdep_assert_held (&ACCESS_PRIVATE (ssp->srcu_sup, lock));if (srcu_gp_is_expedited (ssp)) jbase =0 ;if (rcu_seq_state (READ_ONCE (sup->srcu_gp_seq))) {
@@ -675,9 +676,13 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)void cleanup_srcu_struct (struct srcu_struct *ssp) {int cpu; +unsigned long delay;struct srcu_usage *sup = ssp->srcu_sup; -if (WARN_ON (!srcu_get_delay (ssp))) +spin_lock_irq_rcu_node (ssp->srcu_sup); + delay =srcu_get_delay (ssp); +spin_unlock_irq_rcu_node (ssp->srcu_sup); +if (WARN_ON (!delay))return ;/* Just leak it! */ if (WARN_ON (srcu_readers_active (ssp)))return ;/* Just leak it! */
@@ -1101,7 +1106,9 @@ static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) {unsigned long curdelay; +spin_lock_irq_rcu_node (ssp->srcu_sup); curdelay = !srcu_get_delay (ssp); +spin_unlock_irq_rcu_node (ssp->srcu_sup);for (;;) {if (srcu_readers_active_idx_check (ssp, idx))
@@ -1854,7 +1861,9 @@ static void process_srcu(struct work_struct *work) ssp = sup->srcu_ssp;srcu_advance_state (ssp); +spin_lock_irq_rcu_node (ssp->srcu_sup); curdelay =srcu_get_delay (ssp); +spin_unlock_irq_rcu_node (ssp->srcu_sup);if (curdelay) {WRITE_ONCE (sup->reschedule_count,0 ); }else {
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 8/17 srcu: Rename srcu_check_read_flavor_lite() to srcu_check_read_flavor_force()
In reply to Paul E. McKenneyThis commit renames the srcu_check_read_flavor_lite() function to srcu_check_read_flavor_force() and adds a read_flavor argument in order to support an srcu_read_lock_fast() variant that is to avoid array indexing in both the lock and unlock primitives.
--- include/linux/srcu.h | 2 +- include/linux/srcutiny.h | 2 +- include/linux/srcutree.h | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-)
include/linux/srcu.h +1 -1
@@ -279,7 +279,7 @@ static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp) {int retval; -srcu_check_read_flavor_lite (ssp); +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_LITE ); retval =__srcu_read_lock_lite (ssp);rcu_try_lock_acquire (&ssp->dep_map);return retval;
include/linux/srcutiny.h +1 -1
@@ -82,7 +82,7 @@ static inline void srcu_barrier(struct srcu_struct *ssp) }#define srcu_check_read_flavor (ssp, read_flavor) do { } while (0) -#define srcu_check_read_flavor_lite (ssp) do { } while (0) +#define srcu_check_read_flavor_force (ssp, read_flavor) do { } while (0)/* Defined here to avoid size increase for non-torture kernels. */ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
include/linux/srcutree.h +6 -4
@@ -251,16 +251,18 @@ static inline void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)void __srcu_check_read_flavor (struct srcu_struct *ssp,int read_flavor); -// Record _lite() usage even for CONFIG_PROVE_RCU=n kernels. -static inline void srcu_check_read_flavor_lite (struct srcu_struct *ssp) +// Record reader usage even for CONFIG_PROVE_RCU=n kernels. This is +// needed only for flavors that require grace-period smp_mb() calls to be +// promoted to synchronize_rcu(). +static inline void srcu_check_read_flavor_force (struct srcu_struct *ssp,int read_flavor) {struct srcu_data *sdp =raw_cpu_ptr (ssp->sda); -if (likely (READ_ONCE (sdp->srcu_reader_flavor) &SRCU_READ_FLAVOR_LITE )) +if (likely (READ_ONCE (sdp->srcu_reader_flavor) & read_flavor))return ;// Note that the cmpxchg() in __srcu_check_read_flavor() is fully ordered. -__srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_LITE ); +__srcu_check_read_flavor (ssp, read_flavor); }// Record non-_lite() usage only for CONFIG_PROVE_RCU=y kernels.
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 9/17 srcu: Add SRCU_READ_FLAVOR_SLOWGP to flag need for synchronize_rcu()
In reply to Paul E. McKenneyThis commit switches from a direct test of SRCU_READ_FLAVOR_LITE to a new SRCU_READ_FLAVOR_SLOWGP macro to check for substituting synchronize_rcu() for smp_mb() in SRCU grace periods. Right now, SRCU_READ_FLAVOR_SLOWGP is exactly SRCU_READ_FLAVOR_LITE, but the addition of the _fast() flavor of SRCU will change that.
--- include/linux/srcu.h | 3 +++ kernel/rcu/srcutree.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-)
include/linux/srcu.h +3 -0
@@ -49,6 +49,9 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ SRCU_READ_FLAVOR_LITE) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE +// Flavors requiring synchronize_rcu() +// instead of smp_mb(). #ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h>
kernel/rcu/srcutree.c +3 -3
@@ -449,7 +449,7 @@ static bool srcu_readers_lock_idx(struct srcu_struct *ssp, int idx, bool gp, uns }WARN_ONCE (IS_ENABLED (CONFIG_PROVE_RCU ) && (mask & (mask -1 )),"Mixed reader flavors for srcu_struct at %ps.\n" , ssp); -if (mask &SRCU_READ_FLAVOR_LITE && !gp) +if (mask &SRCU_READ_FLAVOR_SLOWGP && !gp)return false;return sum == unlocks; }
@@ -487,7 +487,7 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)unsigned long unlocks; unlocks =srcu_readers_unlock_idx (ssp, idx, &rdm); - did_gp = !!(rdm &SRCU_READ_FLAVOR_LITE ); + did_gp = !!(rdm &SRCU_READ_FLAVOR_SLOWGP );/* * Make sure that a lock is always counted if the corresponding
@@ -1205,7 +1205,7 @@ static bool srcu_should_expedite(struct srcu_struct *ssp)check_init_srcu_struct (ssp);/* If _lite() readers, don't do unsolicited expediting. */ -if (this_cpu_read (ssp->sda->srcu_reader_flavor) &SRCU_READ_FLAVOR_LITE ) +if (this_cpu_read (ssp->sda->srcu_reader_flavor) &SRCU_READ_FLAVOR_SLOWGP )return false;/* If the local srcu_data structure has callbacks, not idle. */ sdp =raw_cpu_ptr (ssp->sda);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 10/17 srcu: Pull pointer-to-integer conversion into __srcu_ptr_to_ctr()
In reply to Paul E. McKenneyThis commit abstracts the srcu_read_lock*() pointer-to-integer conversion into a new __srcu_ptr_to_ctr(). This will be used in rcutorture for testing an srcu_read_lock_fast() that returns a pointer rather than an integer.
--- include/linux/srcutree.h | 9 ++++++++- kernel/rcu/srcutree.c | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-)
include/linux/srcutree.h +8 -1
@@ -211,6 +211,13 @@ void synchronize_srcu_expedited(struct srcu_struct *ssp);void srcu_barrier (struct srcu_struct *ssp);void srcu_torture_stats_print (struct srcu_struct *ssp,char *tt,char *tf); +// Converts a per-CPU pointer to an ->srcu_ctrs[] array element to that +// element's index. +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return scpp - &ssp->sda->srcu_ctrs[0 ]; +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching
@@ -228,7 +235,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_lite()." );this_cpu_inc (scp->srcu_locks.counter);/* Y */ barrier ();/* Avoid leaking the critical section. */ -return scp - &ssp->sda->srcu_ctrs[0 ]; +return __srcu_ptr_to_ctr (ssp, scp); }/*
kernel/rcu/srcutree.c +2 -2
@@ -753,7 +753,7 @@ int __srcu_read_lock(struct srcu_struct *ssp)this_cpu_inc (scp->srcu_locks.counter);smp_mb ();/* B */ /* Avoid leaking the critical section. */ -return scp - &ssp->sda->srcu_ctrs[0 ]; +return __srcu_ptr_to_ctr (ssp, scp); }EXPORT_SYMBOL_GPL (__srcu_read_lock);
@@ -783,7 +783,7 @@ int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)atomic_long_inc (&scp->srcu_locks);smp_mb__after_atomic ();/* B */ /* Avoid leaking the critical section. */ -return scpp - &ssp->sda->srcu_ctrs[0 ]; +return __srcu_ptr_to_ctr (ssp, scpp); }EXPORT_SYMBOL_GPL (__srcu_read_lock_nmisafe);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 11/17 srcu: Pull integer-to-pointer conversion into __srcu_ctr_to_ptr()
In reply to Paul E. McKenneyThis commit abstracts the srcu_read_unlock*() integer-to-pointer conversion into a new __srcu_ctr_to_ptr(). This will be used in rcutorture for testing an srcu_read_unlock_fast() that avoids array-indexing overhead by taking a pointer rather than an integer.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/srcutree.h | 9 ++++++++- kernel/rcu/srcutree.c | 6 ++---- 2 files changed, 10 insertions(+), 5 deletions(-)
include/linux/srcutree.h +8 -1
@@ -218,6 +218,13 @@ static inline bool __srcu_ptr_to_ctr(struct srcu_struct *ssp, struct srcu_ctr __return scpp - &ssp->sda->srcu_ctrs[0 ]; } +// Converts an integer to a per-CPU pointer to the corresponding +// ->srcu_ctrs[] array element. +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return &ssp->sda->srcu_ctrs[idx]; +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching
@@ -252,7 +259,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)static inline void __srcu_read_unlock_lite (struct srcu_struct *ssp,int idx) {barrier ();/* Avoid leaking the critical section. */ -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter);/* Z */ +this_cpu_inc (__srcu_ctr_to_ptr (ssp, idx)->srcu_unlocks.counter);/* Z */ RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_lite()." ); }
kernel/rcu/srcutree.c +2 -4
@@ -765,7 +765,7 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock);void __srcu_read_unlock (struct srcu_struct *ssp,int idx) {smp_mb ();/* C */ /* Avoid leaking the critical section. */ -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter); +this_cpu_inc (__srcu_ctr_to_ptr (ssp, idx)->srcu_unlocks.counter); }EXPORT_SYMBOL_GPL (__srcu_read_unlock);
@@ -794,10 +794,8 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe);*/ void __srcu_read_unlock_nmisafe (struct srcu_struct *ssp,int idx) { -struct srcu_data *sdp =raw_cpu_ptr (ssp->sda); -smp_mb__before_atomic ();/* C */ /* Avoid leaking the critical section. */ -atomic_long_inc (&sdp->srcu_ctrs[idx].srcu_unlocks); +atomic_long_inc (&raw_cpu_ptr (__srcu_ctr_to_ptr (ssp, idx))->srcu_unlocks); }EXPORT_SYMBOL_GPL (__srcu_read_unlock_nmisafe);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 12/17 srcu: Move SRCU Tree/Tiny definitions from srcu.h
In reply to Paul E. McKenneyThere are a couple of definitions under "#ifdef CONFIG_TINY_SRCU" in include/linux/srcu.h. There is no point in them being there, so this commit moves them to include/linux/srcutiny.h and include/linux/srcutree.c, thus eliminating that #ifdef.
--- include/linux/srcu.h | 10 +--------- include/linux/srcutiny.h | 3 +++ include/linux/srcutree.h | 1 + 3 files changed, 5 insertions(+), 9 deletions(-)
include/linux/srcu.h +1 -9
@@ -52,6 +52,7 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE// Flavors requiring synchronize_rcu() // instead of smp_mb(). +void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);#ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h>
@@ -64,15 +65,6 @@ int init_srcu_struct(struct srcu_struct *ssp);void call_srcu (struct srcu_struct *ssp,struct rcu_head *head,void (*func)(struct rcu_head *head));void cleanup_srcu_struct (struct srcu_struct *ssp); -int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); -void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); -#ifdef CONFIG_TINY_SRCU -#define __srcu_read_lock_lite __srcu_read_lock -#define __srcu_read_unlock_lite __srcu_read_unlock -#else // #ifdef CONFIG_TINY_SRCU -int __srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp); -void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx) __releases(ssp); -#endif // #else // #ifdef CONFIG_TINY_SRCU void synchronize_srcu (struct srcu_struct *ssp);#define SRCU_GET_STATE_COMPLETED 0x1
include/linux/srcutiny.h +3 -0
@@ -71,6 +71,9 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +#define __srcu_read_lock_lite __srcu_read_lock +#define __srcu_read_unlock_lite __srcu_read_unlock +static inline void synchronize_srcu_expedited (struct srcu_struct *ssp) {synchronize_srcu (ssp);
include/linux/srcutree.h +1 -0
@@ -207,6 +207,7 @@ struct srcu_struct { #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)#define DEFINE_STATIC_SRCU (name) __DEFINE_SRCU(name, static) +int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);void synchronize_srcu_expedited (struct srcu_struct *ssp);void srcu_barrier (struct srcu_struct *ssp);void srcu_torture_stats_print (struct srcu_struct *ssp,char *tt,char *tf);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 13/17 srcu: Add SRCU-fast readers
In reply to Paul E. McKenneyThis commit adds srcu_read_{,un}lock_fast(), which is similar to srcu_read_{,un}lock_lite(), but avoids the array-indexing and pointer-following overhead. On a microbenchmark featuring tight loops around empty readers, this results in about a 20% speedup compared to RCU Tasks Trace on my x86 laptop.
Please note that SRCU-fast has drawbacks compared to RCU Tasks Trace, including:
o Lack of CPU stall warnings.
o SRCU-fast readers permitted only where rcu_is_watching().
o A pointer-sized return value from srcu_read_lock_fast() must
be passed to the corresponding srcu_read_unlock_fast().
o In the absence of readers, a synchronize_srcu() having _fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.
SRCU-fast also has a drawback compared to SRCU-lite, namely that the return value from srcu_read_lock_fast()-fast is a 64-bit pointer and that from srcu_read_lock_lite() is only a 32-bit int.
[ paulmck: Apply feedback from Akira Yokosawa. ]
--- include/linux/srcu.h | 47 ++++++++++++++++++++++++++++++++++++++-- include/linux/srcutiny.h | 22 +++++++++++++++++++ include/linux/srcutree.h | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-)
include/linux/srcu.h +45 -2
@@ -47,9 +47,10 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). +#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_LITE) // All of the above. -#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE + SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)// Flavors requiring synchronize_rcu() // instead of smp_mb(). void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
@@ -253,6 +254,33 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)return retval; } +/** +* srcu_read_lock_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter an SRCU read-side critical section, but for a light-weight +* smp_mb()-free reader. See srcu_read_lock() for more information. +* +* If srcu_read_lock_fast() is ever used on an srcu_struct structure, +* then none of the other flavors may be used, whether before, during, +* or after. Note that grace-period auto-expediting is disabled for _fast +* srcu_struct structures because auto-expedited grace periods invoke +* synchronize_rcu_expedited(), IPIs and all. +* +* Note that srcu_read_lock_fast() can be invoked only from those contexts +* where RCU is watching, that is, from contexts where it would be legal +* to invoke rcu_read_lock(). Otherwise, lockdep will complain. +*/ +static inline struct srcu_ctr __percpu *srcu_read_lock_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +struct srcu_ctr __percpu *retval; + +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); + retval =__srcu_read_lock_fast (ssp); +rcu_try_lock_acquire (&ssp->dep_map); +return retval; +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader.
@@ -356,6 +384,21 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)__srcu_read_unlock (ssp, idx); } +/** +* srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit a light-weight SRCU read-side critical section. +*/ +static inline void srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +srcu_lock_release (&ssp->dep_map); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader.
include/linux/srcutiny.h +22 -0
@@ -71,6 +71,28 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +struct srcu_ctr ; + +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return (int )(intptr_t )(struct srcu_ctr __force __kernel *)scpp; +} + +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return (struct srcu_ctr __percpu *)(intptr_t )idx; +} + +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +return __srcu_ctr_to_ptr (ssp,__srcu_read_lock (ssp)); +} + +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +__srcu_read_unlock (ssp,__srcu_ptr_to_ctr (ssp, scp)); +} +#define __srcu_read_lock_lite __srcu_read_lock#define __srcu_read_unlock_lite __srcu_read_unlock
include/linux/srcutree.h +38 -0
@@ -226,6 +226,44 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssreturn &ssp->sda->srcu_ctrs[idx]; } +/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +} + +/* +* Removes the count for the old reader from the appropriate +* per-CPU element of the srcu_struct. Note that this may well be a +* different CPU than that which was incremented by the corresponding +* srcu_read_lock_fast(), but it must be within the same task. +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +barrier ();/* Avoid leaking the critical section. */ +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching
Signature · 1 line
2.40.1
Re: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
Quoted 10 lines · Paul E. McKenney
include/linux/srcutree.h +9
-0+/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture.
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 247
Quoted 10 lines
include/linux/srcutree.h +10
-0+*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +}
This doesn't look fast.
If I'm reading this correctly,
even with debugs off RCU_LOCKDEP_WARN() will still call rcu_is_watching() and this doesn't look cheap or fast.
Re: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
Quoted 18 lines · Alexei Starovoitov
Quoted 17 lines · Paul E. McKenney
include/linux/srcutree.h +16
-0+/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */
Quoted 8 lines
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 247
Quoted 3 lines
include/linux/srcutree.h +3
-0+barrier ();/* Avoid leaking the critical section. */ +return scp; +}This doesn't look fast.
If I'm reading this correctly,
even with debugs offRCU_LOCKDEP_WARN()will still callrcu_is_watching()and this doesn't look cheap or fast.
this is the while (0 && (c)), thing, right? can't the compiler throw that out in DCE ?
Paul E. McKenney
maintainerRe: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
In reply to Alexei StarovoitovQuoted 18 lines · Alexei Starovoitov
Quoted 17 lines · Paul E. McKenney
include/linux/srcutree.h +16
-0+/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */
Quoted 8 lines
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 247
Quoted 3 lines
include/linux/srcutree.h +3
-0+barrier ();/* Avoid leaking the critical section. */ +return scp; +}This doesn't look fast.
If I'm reading this correctly,
even with debugs offRCU_LOCKDEP_WARN()will still callrcu_is_watching()and this doesn't look cheap or fast.
Here is the CONFIG_PROVE_RCU=n definition:
#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))The "0" in the "0 && (c)" should prevent that call to rcu_is_watching().
But why not see what the compiler thinks? I added the following function to kernel/rcu/srcutree.c:
struct srcu_ctr __percpu *test_srcu_read_lock_fast(struct srcu_struct *ssp)
{
struct srcu_ctr __percpu *p;
p = srcu_read_lock_fast(ssp);
return p;
}This function compiles to the following code:
Dump of assembler code for function test_srcu_read_lock_fast:
0xffffffff811220c0 <+0>: endbr64 0xffffffff811220c4 <+4>: sub $0x8,%rsp 0xffffffff811220c8 <+8>: mov 0x8(%rdi),%rax 0xffffffff811220cc <+12>: add %gs:0x7eef3944(%rip),%rax # 0x15a18 <this_cpu_off> 0xffffffff811220d4 <+20>: mov 0x20(%rax),%eax 0xffffffff811220d7 <+23>: test $0x8,%al 0xffffffff811220d9 <+25>: je 0xffffffff811220eb <test_srcu_read_lock_fast+43> 0xffffffff811220db <+27>: mov (%rdi),%rax 0xffffffff811220de <+30>: incq %gs:(%rax) 0xffffffff811220e2 <+34>: add $0x8,%rsp 0xffffffff811220e6 <+38>: jmp 0xffffffff81f5fe60 <__x86_return_thunk> 0xffffffff811220eb <+43>: mov $0x8,%esi 0xffffffff811220f0 <+48>: mov %rdi,(%rsp) 0xffffffff811220f4 <+52>: call 0xffffffff8111fb90 <__srcu_check_read_flavor> 0xffffffff811220f9 <+57>: mov (%rsp),%rdi 0xffffffff811220fd <+61>: jmp 0xffffffff811220db <test_srcu_read_lock_fast+27>
The first call to srcu_read_lock_fast() invokes __srcu_check_read_flavor(), but after that the "je" instruction will fall through. So the common-case code path executes only the part of this function up to and including the "jmp 0xffffffff81f5fe60 <__x86_return_thunk>".
Does that serve?
Thanx, Paul
Re: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
Quoted 66 lines · Paul E. McKenney
Quoted 18 lines · Alexei Starovoitov
Quoted 17 lines · Paul E. McKenney
include/linux/srcutree.h +16
-0+/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ Quoted 8 lines
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 247
Quoted 3 lines
include/linux/srcutree.h +3
-0+barrier ();/* Avoid leaking the critical section. */ +return scp; +}This doesn't look fast.
If I'm reading this correctly,
even with debugs offRCU_LOCKDEP_WARN()will still callrcu_is_watching()and this doesn't look cheap or fast.Here is the
CONFIG_PROVE_RCU=ndefinition:#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))The "0" in the "0 && (c)" should prevent that call to
rcu_is_watching().But why not see what the compiler thinks? I added the following function to
kernel/rcu/srcutree.c:struct
srcu_ctr__percpu*test_srcu_read_lock_fast(structsrcu_struct*ssp){ struct srcu_ctr __percpu *p; p = srcu_read_lock_fast(ssp); return p; }This function compiles to the following code:
Dump of assembler code for function
test_srcu_read_lock_fast:0xffffffff811220c0 <+0>: endbr64 0xffffffff811220c4 <+4>: sub $0x8,%rsp 0xffffffff811220c8 <+8>: mov 0x8(%rdi),%rax 0xffffffff811220cc <+12>: add %gs:0x7eef3944(%rip),%rax # 0x15a18 <this_cpu_off> 0xffffffff811220d4 <+20>: mov 0x20(%rax),%eax 0xffffffff811220d7 <+23>: test $0x8,%al 0xffffffff811220d9 <+25>: je 0xffffffff811220eb <test_srcu_read_lock_fast+43> 0xffffffff811220db <+27>: mov (%rdi),%rax 0xffffffff811220de <+30>: incq %gs:(%rax) 0xffffffff811220e2 <+34>: add $0x8,%rsp 0xffffffff811220e6 <+38>: jmp 0xffffffff81f5fe60 <__x86_return_thunk> 0xffffffff811220eb <+43>: mov $0x8,%esi 0xffffffff811220f0 <+48>: mov %rdi,(%rsp) 0xffffffff811220f4 <+52>: call 0xffffffff8111fb90 <__srcu_check_read_flavor> 0xffffffff811220f9 <+57>: mov (%rsp),%rdi 0xffffffff811220fd <+61>: jmp 0xffffffff811220db <test_srcu_read_lock_fast+27>
Quoted 6 lines
The first call to
srcu_read_lock_fast()invokes__srcu_check_read_flavor(), but after that the "je" instruction will fall through. So the common-case code path executes only the part of this function up to and including the "jmp0xffffffff81f5fe60<__x86_return_thunk>".Does that serve?
Thanks for checking.
I was worried that in case of:
while (0 && (c))
the compiler might need to still emit (c) since it cannot prove that there are no side effects there.
I vaguely recall now that C standard allows to ignore 2nd expression when the first expression satisfies the boolean condition.
Re: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
In reply to Paul E. McKenneyQuoted 167 lines · Paul E. McKenney
This commit adds srcu_read_{,un}lock_fast(), which is similar to srcu_read_{,un}lock_lite(), but avoids the array-indexing and pointer-following overhead. On a microbenchmark featuring tight loops around empty readers, this results in about a 20% speedup compared to RCU Tasks Trace on my x86 laptop.
Please note that SRCU-fast has drawbacks compared to RCU Tasks Trace, including:
o Lack of CPU stall warnings. o SRCU-fast readers permitted only where rcu_is_watching(). o A pointer-sized return value from srcu_read_lock_fast() mustbe passed to the corresponding
srcu_read_unlock_fast().
o In the absence of readers, asynchronize_srcu()having_fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.SRCU-fast also has a drawback compared to SRCU-lite, namely that the return value from srcu_read_lock_fast()-fast is a 64-bit pointer and that from
srcu_read_lock_lite()is only a 32-bit int.[ paulmck: Apply feedback from Akira Yokosawa. ]
Signed-off-by Paul E. McKenney <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Andrii Nakryiko <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Kent Overstreet <[email protected]>Cc <[email protected]>--- include/linux/srcu.h | 47 ++++++++++++++++++++++++++++++++++++++-- include/linux/srcutiny.h | 22 +++++++++++++++++++ include/linux/srcutree.h | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-)include/linux/srcu.h +45
-2@@ -47,9 +47,10 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). +#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_LITE) // All of the above. -#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE + SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)// Flavors requiring synchronize_rcu() // instead of smp_mb(). void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);@@ -253,6 +254,33 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)return retval; } +/** +* srcu_read_lock_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter an SRCU read-side critical section, but for a light-weight +* smp_mb()-free reader. See srcu_read_lock() for more information. +* +* If srcu_read_lock_fast() is ever used on an srcu_struct structure, +* then none of the other flavors may be used, whether before, during, +* or after. Note that grace-period auto-expediting is disabled for _fast +* srcu_struct structures because auto-expedited grace periods invoke +* synchronize_rcu_expedited(), IPIs and all. +* +* Note that srcu_read_lock_fast() can be invoked only from those contexts +* where RCU is watching, that is, from contexts where it would be legal +* to invoke rcu_read_lock(). Otherwise, lockdep will complain. +*/ +static inline struct srcu_ctr __percpu *srcu_read_lock_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +struct srcu_ctr __percpu *retval; + +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); + retval =__srcu_read_lock_fast (ssp); +rcu_try_lock_acquire (&ssp->dep_map); +return retval; +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader. @@ -356,6 +384,21 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)__srcu_read_unlock (ssp, idx); } +/** +* srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit a light-weight SRCU read-side critical section. +*/ +static inline void srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +srcu_lock_release (&ssp->dep_map); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. include/linux/srcutiny.h +22
-0@@ -71,6 +71,28 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +struct srcu_ctr ; + +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return (int )(intptr_t )(struct srcu_ctr __force __kernel *)scpp; +} + +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return (struct srcu_ctr __percpu *)(intptr_t )idx; +} + +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +return __srcu_ctr_to_ptr (ssp,__srcu_read_lock (ssp)); +} + +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +__srcu_read_unlock (ssp,__srcu_ptr_to_ctr (ssp, scp)); +} +#define __srcu_read_lock_lite __srcu_read_lock#define __srcu_read_unlock_lite __srcu_read_unlockinclude/linux/srcutree.h +15
-0@@ -226,6 +226,44 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssreturn &ssp->sda->srcu_ctrs[idx]; } +/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." );
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 253
Quoted 10 lines
include/linux/srcutree.h +10
-0+this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +} + +/* +* Removes the count for the old reader from the appropriate +* per-CPU element of the srcu_struct. Note that this may well be a +* different CPU than that which was incremented by the corresponding +* srcu_read_lock_fast(), but it must be within the same task.
hm... why the "same task" restriction? With uretprobes we take srcu_read_lock under a traced task, but we can "release" this lock from timer interrupt, which could be in the context of any task.
Quoted 19 lines
include/linux/srcutree.h +13
-1+* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +barrier ();/* Avoid leaking the critical section. */ +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching -- 2.40.1
Paul E. McKenney
maintainerRe: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
Quoted 174 lines · Andrii Nakryiko
Quoted 173 lines · Paul E. McKenney
This commit adds srcu_read_{,un}lock_fast(), which is similar to srcu_read_{,un}lock_lite(), but avoids the array-indexing and pointer-following overhead. On a microbenchmark featuring tight loops around empty readers, this results in about a 20% speedup compared to RCU Tasks Trace on my x86 laptop.
Please note that SRCU-fast has drawbacks compared to RCU Tasks Trace, including:
o Lack of CPU stall warnings. o SRCU-fast readers permitted only where rcu_is_watching(). o A pointer-sized return value from srcu_read_lock_fast() mustbe passed to the corresponding
srcu_read_unlock_fast().
o In the absence of readers, asynchronize_srcu()having_fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.SRCU-fast also has a drawback compared to SRCU-lite, namely that the return value from srcu_read_lock_fast()-fast is a 64-bit pointer and that from
srcu_read_lock_lite()is only a 32-bit int.[ paulmck: Apply feedback from Akira Yokosawa. ]
Signed-off-by Paul E. McKenney <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Andrii Nakryiko <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Kent Overstreet <[email protected]>Cc <[email protected]>--- include/linux/srcu.h | 47 ++++++++++++++++++++++++++++++++++++++-- include/linux/srcutiny.h | 22 +++++++++++++++++++ include/linux/srcutree.h | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-)include/linux/srcu.h +45
-2@@ -47,9 +47,10 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). +#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_LITE) // All of the above. -#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE + SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)// Flavors requiring synchronize_rcu() // instead of smp_mb(). void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);@@ -253,6 +254,33 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)return retval; } +/** +* srcu_read_lock_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter an SRCU read-side critical section, but for a light-weight +* smp_mb()-free reader. See srcu_read_lock() for more information. +* +* If srcu_read_lock_fast() is ever used on an srcu_struct structure, +* then none of the other flavors may be used, whether before, during, +* or after. Note that grace-period auto-expediting is disabled for _fast +* srcu_struct structures because auto-expedited grace periods invoke +* synchronize_rcu_expedited(), IPIs and all. +* +* Note that srcu_read_lock_fast() can be invoked only from those contexts +* where RCU is watching, that is, from contexts where it would be legal +* to invoke rcu_read_lock(). Otherwise, lockdep will complain. +*/ +static inline struct srcu_ctr __percpu *srcu_read_lock_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +struct srcu_ctr __percpu *retval; + +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); + retval =__srcu_read_lock_fast (ssp); +rcu_try_lock_acquire (&ssp->dep_map); +return retval; +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader. @@ -356,6 +384,21 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)__srcu_read_unlock (ssp, idx); } +/** +* srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit a light-weight SRCU read-side critical section. +*/ +static inline void srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +srcu_lock_release (&ssp->dep_map); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. include/linux/srcutiny.h +22
-0@@ -71,6 +71,28 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +struct srcu_ctr ; + +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return (int )(intptr_t )(struct srcu_ctr __force __kernel *)scpp; +} + +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return (struct srcu_ctr __percpu *)(intptr_t )idx; +} + +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +return __srcu_ctr_to_ptr (ssp,__srcu_read_lock (ssp)); +} + +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +__srcu_read_unlock (ssp,__srcu_ptr_to_ctr (ssp, scp)); +} +#define __srcu_read_lock_lite __srcu_read_lock#define __srcu_read_unlock_lite __srcu_read_unlockinclude/linux/srcutree.h +21
-0@@ -226,6 +226,44 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssreturn &ssp->sda->srcu_ctrs[idx]; } +/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +} + +/*
Quoted 8 lines
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 253
Quoted 4 lines
include/linux/srcutree.h +4
-0+* Removes the count for the old reader from the appropriate +* per-CPU element of the srcu_struct. Note that this may well be a +* different CPU than that which was incremented by the corresponding +* srcu_read_lock_fast(), but it must be within the same task. hm... why the "same task" restriction? With uretprobes we take
srcu_read_lockunder a traced task, but we can "release" this lock from timer interrupt, which could be in the context of any task.
A kneejerk reaction on my part? ;-)
But the good news is that this restriction is easy for me to relax. I adjust the comments and remove the rcu_try_lock_acquire() from srcu_read_lock_fast() and the srcu_lock_release() from srcu_read_unlock_fast().
But in that case, I should rename this to srcu_down_read_fast() and srcu_up_read_fast() or similar, as these names would clearly indicate that they can be used cross-task. (And from interrupt handlers, but not from NMI handlers.)
Also, srcu_read_lock() and srcu_read_unlock() have srcu_lock_acquire() and srcu_lock_release() calls, so I am surprised that lockdep didn't complain when you invoked srcu_read_lock() from task context and srcu_read_unlock() from a timer interrupt.
Thanx, Paul
Quoted 19 lines
Quoted 19 lines
include/linux/srcutree.h +13
-1+* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +barrier ();/* Avoid leaking the critical section. */ +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching -- 2.40.1
Re: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
Quoted 197 lines · Paul E. McKenney
Quoted 174 lines · Andrii Nakryiko
Quoted 173 lines · Paul E. McKenney
This commit adds srcu_read_{,un}lock_fast(), which is similar to srcu_read_{,un}lock_lite(), but avoids the array-indexing and pointer-following overhead. On a microbenchmark featuring tight loops around empty readers, this results in about a 20% speedup compared to RCU Tasks Trace on my x86 laptop.
Please note that SRCU-fast has drawbacks compared to RCU Tasks Trace, including:
o Lack of CPU stall warnings. o SRCU-fast readers permitted only where rcu_is_watching(). o A pointer-sized return value from srcu_read_lock_fast() mustbe passed to the corresponding
srcu_read_unlock_fast().
o In the absence of readers, asynchronize_srcu()having_fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.SRCU-fast also has a drawback compared to SRCU-lite, namely that the return value from srcu_read_lock_fast()-fast is a 64-bit pointer and that from
srcu_read_lock_lite()is only a 32-bit int.[ paulmck: Apply feedback from Akira Yokosawa. ]
Signed-off-by Paul E. McKenney <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Andrii Nakryiko <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Kent Overstreet <[email protected]>Cc <[email protected]>--- include/linux/srcu.h | 47 ++++++++++++++++++++++++++++++++++++++-- include/linux/srcutiny.h | 22 +++++++++++++++++++ include/linux/srcutree.h | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-)include/linux/srcu.h +45
-2@@ -47,9 +47,10 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). +#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_LITE) // All of the above. -#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE + SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)// Flavors requiring synchronize_rcu() // instead of smp_mb(). void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);@@ -253,6 +254,33 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)return retval; } +/** +* srcu_read_lock_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter an SRCU read-side critical section, but for a light-weight +* smp_mb()-free reader. See srcu_read_lock() for more information. +* +* If srcu_read_lock_fast() is ever used on an srcu_struct structure, +* then none of the other flavors may be used, whether before, during, +* or after. Note that grace-period auto-expediting is disabled for _fast +* srcu_struct structures because auto-expedited grace periods invoke +* synchronize_rcu_expedited(), IPIs and all. +* +* Note that srcu_read_lock_fast() can be invoked only from those contexts +* where RCU is watching, that is, from contexts where it would be legal +* to invoke rcu_read_lock(). Otherwise, lockdep will complain. +*/ +static inline struct srcu_ctr __percpu *srcu_read_lock_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +struct srcu_ctr __percpu *retval; + +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); + retval =__srcu_read_lock_fast (ssp); +rcu_try_lock_acquire (&ssp->dep_map); +return retval; +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader. @@ -356,6 +384,21 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)__srcu_read_unlock (ssp, idx); } +/** +* srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit a light-weight SRCU read-side critical section. +*/ +static inline void srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +srcu_lock_release (&ssp->dep_map); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. include/linux/srcutiny.h +22
-0@@ -71,6 +71,28 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +struct srcu_ctr ; + +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return (int )(intptr_t )(struct srcu_ctr __force __kernel *)scpp; +} + +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return (struct srcu_ctr __percpu *)(intptr_t )idx; +} + +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +return __srcu_ctr_to_ptr (ssp,__srcu_read_lock (ssp)); +} + +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +__srcu_read_unlock (ssp,__srcu_ptr_to_ctr (ssp, scp)); +} +#define __srcu_read_lock_lite __srcu_read_lock#define __srcu_read_unlock_lite __srcu_read_unlockinclude/linux/srcutree.h +21
-0@@ -226,6 +226,44 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssreturn &ssp->sda->srcu_ctrs[idx]; } +/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +} + +/* Quoted 8 lines
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 253
Quoted 4 lines
include/linux/srcutree.h +4
-0+* Removes the count for the old reader from the appropriate +* per-CPU element of the srcu_struct. Note that this may well be a +* different CPU than that which was incremented by the corresponding +* srcu_read_lock_fast(), but it must be within the same task. hm... why the "same task" restriction? With uretprobes we take
srcu_read_lockunder a traced task, but we can "release" this lock from timer interrupt, which could be in the context of any task.A kneejerk reaction on my part? ;-)
But the good news is that this restriction is easy for me to relax. I adjust the comments and remove the
rcu_try_lock_acquire()fromsrcu_read_lock_fast()and thesrcu_lock_release()fromsrcu_read_unlock_fast().But in that case, I should rename this to
srcu_down_read_fast()andsrcu_up_read_fast()or similar, as these names would clearly indicate that they can be used cross-task. (And from interrupt handlers, but not from NMI handlers.)
Quoted 4 lines
Also,
srcu_read_lock()andsrcu_read_unlock()havesrcu_lock_acquire()andsrcu_lock_release()calls, so I am surprised that lockdep didn't complain when you invokedsrcu_read_lock()from task context andsrcu_read_unlock()from a timer interrupt.
That's because I'm using __srcu_read_lock and __srcu_read_unlock ;) See this part in kernel/events/uprobes.c:
/* __srcu_read_lock() because SRCU lock survives switch to user space */ srcu_idx = __srcu_read_lock(&uretprobes_srcu);
Quoted 22 lines
Thanx, Paul
Quoted 19 lines
Quoted 19 lines
include/linux/srcutree.h +13
-1+* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +barrier ();/* Avoid leaking the critical section. */ +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching -- 2.40.1
Paul E. McKenney
maintainerRe: [PATCH rcu 13/17] srcu: Add SRCU-fast readers
Quoted 203 lines · Andrii Nakryiko
Quoted 197 lines · Paul E. McKenney
Quoted 174 lines · Andrii Nakryiko
Quoted 173 lines · Paul E. McKenney
This commit adds srcu_read_{,un}lock_fast(), which is similar to srcu_read_{,un}lock_lite(), but avoids the array-indexing and pointer-following overhead. On a microbenchmark featuring tight loops around empty readers, this results in about a 20% speedup compared to RCU Tasks Trace on my x86 laptop.
Please note that SRCU-fast has drawbacks compared to RCU Tasks Trace, including:
o Lack of CPU stall warnings. o SRCU-fast readers permitted only where rcu_is_watching(). o A pointer-sized return value from srcu_read_lock_fast() mustbe passed to the corresponding
srcu_read_unlock_fast().
o In the absence of readers, asynchronize_srcu()having_fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.SRCU-fast also has a drawback compared to SRCU-lite, namely that the return value from srcu_read_lock_fast()-fast is a 64-bit pointer and that from
srcu_read_lock_lite()is only a 32-bit int.[ paulmck: Apply feedback from Akira Yokosawa. ]
Signed-off-by Paul E. McKenney <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Andrii Nakryiko <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Kent Overstreet <[email protected]>Cc <[email protected]>--- include/linux/srcu.h | 47 ++++++++++++++++++++++++++++++++++++++-- include/linux/srcutiny.h | 22 +++++++++++++++++++ include/linux/srcutree.h | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-)include/linux/srcu.h +45
-2@@ -47,9 +47,10 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). +#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_LITE) // All of the above. -#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE + SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)// Flavors requiring synchronize_rcu() // instead of smp_mb(). void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);@@ -253,6 +254,33 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)return retval; } +/** +* srcu_read_lock_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter an SRCU read-side critical section, but for a light-weight +* smp_mb()-free reader. See srcu_read_lock() for more information. +* +* If srcu_read_lock_fast() is ever used on an srcu_struct structure, +* then none of the other flavors may be used, whether before, during, +* or after. Note that grace-period auto-expediting is disabled for _fast +* srcu_struct structures because auto-expedited grace periods invoke +* synchronize_rcu_expedited(), IPIs and all. +* +* Note that srcu_read_lock_fast() can be invoked only from those contexts +* where RCU is watching, that is, from contexts where it would be legal +* to invoke rcu_read_lock(). Otherwise, lockdep will complain. +*/ +static inline struct srcu_ctr __percpu *srcu_read_lock_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +struct srcu_ctr __percpu *retval; + +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); + retval =__srcu_read_lock_fast (ssp); +rcu_try_lock_acquire (&ssp->dep_map); +return retval; +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader. @@ -356,6 +384,21 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)__srcu_read_unlock (ssp, idx); } +/** +* srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit a light-weight SRCU read-side critical section. +*/ +static inline void srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +srcu_lock_release (&ssp->dep_map); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. include/linux/srcutiny.h +22
-0@@ -71,6 +71,28 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +struct srcu_ctr ; + +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return (int )(intptr_t )(struct srcu_ctr __force __kernel *)scpp; +} + +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return (struct srcu_ctr __percpu *)(intptr_t )idx; +} + +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +return __srcu_ctr_to_ptr (ssp,__srcu_read_lock (ssp)); +} + +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +__srcu_read_unlock (ssp,__srcu_ptr_to_ctr (ssp, scp)); +} +#define __srcu_read_lock_lite __srcu_read_lock#define __srcu_read_unlock_lite __srcu_read_unlockinclude/linux/srcutree.h +21
-0@@ -226,6 +226,44 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssreturn &ssp->sda->srcu_ctrs[idx]; } +/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +} + +/* Quoted 8 lines
include/linux/srcutree.h static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss · line 253
Quoted 4 lines
include/linux/srcutree.h +4
-0+* Removes the count for the old reader from the appropriate +* per-CPU element of the srcu_struct. Note that this may well be a +* different CPU than that which was incremented by the corresponding +* srcu_read_lock_fast(), but it must be within the same task. hm... why the "same task" restriction? With uretprobes we take
srcu_read_lockunder a traced task, but we can "release" this lock from timer interrupt, which could be in the context of any task.A kneejerk reaction on my part? ;-)
But the good news is that this restriction is easy for me to relax. I adjust the comments and remove the
rcu_try_lock_acquire()fromsrcu_read_lock_fast()and thesrcu_lock_release()fromsrcu_read_unlock_fast().But in that case, I should rename this to
srcu_down_read_fast()andsrcu_up_read_fast()or similar, as these names would clearly indicate that they can be used cross-task. (And from interrupt handlers, but not from NMI handlers.)Quoted 4 lines
Also,
srcu_read_lock()andsrcu_read_unlock()havesrcu_lock_acquire()andsrcu_lock_release()calls, so I am surprised that lockdep didn't complain when you invokedsrcu_read_lock()from task context andsrcu_read_unlock()from a timer interrupt.
Quoted 5 lines
That's because I'm using
__srcu_read_lockand__srcu_read_unlock;) See this part inkernel/events/uprobes.c:/*
__srcu_read_lock()because SRCU lock survives switch to user space */srcu_idx=__srcu_read_lock(&uretprobes_srcu);
I clearly should have read that code more carefully! ;-)
This happens to work for srcu_read_lock() and srcu_read_lock_nmisafe(), but invoking __srcu_read_lock_lite() or __srcu_read_lock_fast() would fail (rarely!) with too-short grace periods.
So thank you for catching this! I will adjust the name and semantics.
Thanx, Paul
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
include/linux/srcutree.h +13
-1+* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +barrier ();/* Avoid leaking the critical section. */ +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching -- 2.40.1
Paul E. McKenney
maintainerPATCH 14/17 rcutorture: Add ability to test srcu_read_{,un}lock_fast()
In reply to Paul E. McKenneyThis commit permits rcutorture to test srcu_read_{,un}lock_fast(), which is specified by the rcutorture.reader_flavor=0x8 kernel boot parameter.
--- kernel/rcu/rcutorture.c | 9 +++++++++ 1 file changed, 9 insertions(+)
kernel/rcu/rcutorture.c +9 -0
@@ -677,6 +677,7 @@ static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)static int srcu_torture_read_lock (void ) {int idx; +struct srcu_ctr __percpu *scp;int ret =0 ;if ((reader_flavor &SRCU_READ_FLAVOR_NORMAL ) || !(reader_flavor &SRCU_READ_FLAVOR_ALL )) {
@@ -694,6 +695,12 @@ static int srcu_torture_read_lock(void)WARN_ON_ONCE (idx & ~0x1 ); ret += idx <<2 ; } +if (reader_flavor &SRCU_READ_FLAVOR_FAST ) { + scp =srcu_read_lock_fast (srcu_ctlp); + idx =__srcu_ptr_to_ctr (srcu_ctlp, scp); +WARN_ON_ONCE (idx & ~0x1 ); + ret += idx <<3 ; + }return ret; }
@@ -719,6 +726,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)static void srcu_torture_read_unlock (int idx) {WARN_ON_ONCE ((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1 ))); +if (reader_flavor &SRCU_READ_FLAVOR_FAST ) +srcu_read_unlock_fast (srcu_ctlp,__srcu_ctr_to_ptr (srcu_ctlp, (idx &0x8 ) >>3 ));if (reader_flavor &SRCU_READ_FLAVOR_LITE )srcu_read_unlock_lite (srcu_ctlp, (idx &0x4 ) >>2 );if (reader_flavor &SRCU_READ_FLAVOR_NMI )
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 15/17 refscale: Add srcu_read_lock_fast() support using "srcu-fast"
In reply to Paul E. McKenneyThis commit creates a new srcu-fast option for the refscale.scale_type module parameter that selects srcu_read_lock_fast() and srcu_read_unlock_fast().
--- kernel/rcu/refscale.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-)
kernel/rcu/refscale.c +31 -1
@@ -216,6 +216,36 @@ static const struct ref_scale_ops srcu_ops = { .name = "srcu" }; +static void srcu_fast_ref_scale_read_section (const int nloops) +{ +int i; +struct srcu_ctr __percpu *scp; + +for (i = nloops; i >=0 ; i--) { + scp =srcu_read_lock_fast (srcu_ctlp); +srcu_read_unlock_fast (srcu_ctlp, scp); + } +} + +static void srcu_fast_ref_scale_delay_section (const int nloops,const int udl,const int ndl) +{ +int i; +struct srcu_ctr __percpu *scp; + +for (i = nloops; i >=0 ; i--) { + scp =srcu_read_lock_fast (srcu_ctlp); +un_delay (udl, ndl); +srcu_read_unlock_fast (srcu_ctlp, scp); + } +} + +static const struct ref_scale_ops srcu_fast_ops = { + .init = rcu_sync_scale_init, + .readsection = srcu_fast_ref_scale_read_section, + .delaysection = srcu_fast_ref_scale_delay_section, + .name ="srcu-fast" +}; +static void srcu_lite_ref_scale_read_section (const int nloops) {int i;
@@ -1163,7 +1193,7 @@ ref_scale_init(void)long i;int firsterr =0 ;static const struct ref_scale_ops *scale_ops[] = { - &rcu_ops, &srcu_ops, &srcu_lite_ops, RCU_TRACE_OPS RCU_TASKS_OPS + &rcu_ops, &srcu_ops, &srcu_fast_ops, &srcu_lite_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 16/17 rcutorture: Make scenario SRCU-P use srcu_read_lock_fast()
In reply to Paul E. McKenneyThis commit causes the rcutorture SRCU-P scenario use the srcu_read_lock_fast() and srcu_read_unlock_fast() functions. This will cause these two functions to be regularly tested by several developers (myself included), for example, those who use torture.sh as an RCU acceptance test.
--- tools/testing/selftests/rcutorture/configs/rcu/SRCU-P.boot | 1 + 1 file changed, 1 insertion(+)
tools/testing/selftests/rcutorture/configs/rcu/SRCU-P.boot +1 -0
@@ -2,3 +2,4 @@ rcutorture.torture_type=srcud rcupdate.rcu_self_test=1 rcutorture.fwd_progress=3 srcutree.big_cpu_lim=5 +rcutorture.reader_flavor=0x8
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH 17/17 srcu: Fix srcu_read_unlock_{lite,nmisafe}() kernel-doc
In reply to Paul E. McKenneyThe srcu_read_unlock_lite() and srcu_read_unlock_nmisafe() both say that their idx parameters must come from srcu_read_lock(). This would be bad, because a given srcu_struct structure may be used only with one flavor of SRCU reader. This commit therefore updates the srcu_read_unlock_lite() kernel-doc header to say that its idx parameter must be obtained from srcu_read_lock_lite() and the srcu_read_unlock_nmisafe() kernel-doc header to say that its idx parameter must be obtained from srcu_read_lock_nmisafe().
--- include/linux/srcu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/srcu.h +2 -2
@@ -402,7 +402,7 @@ static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ct/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. -* @idx: return value from corresponding srcu_read_lock(). +* @idx: return value from corresponding srcu_read_lock_lite(). * * Exit a light-weight SRCU read-side critical section. */
@@ -418,7 +418,7 @@ static inline void srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)/** * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. -* @idx: return value from corresponding srcu_read_lock(). +* @idx: return value from corresponding srcu_read_lock_nmisafe(). * * Exit an SRCU read-side critical section, but in an NMI-safe manner. */
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerHello!
This series contains SRCU updates, perhaps most notably the addition of srcu_read_lock_fast() and srcu_read_unlock_fast(), which on my laptop are about 20% faster than rcu_read_lock_trace() and rcu_read_unlock_trace(). There are of course drawbacks:
o Lack of CPU stall warnings.
o SRCU-fast readers permitted only where rcu_is_watching().
o A pointer-sized return value from srcu_read_lock_fast() must
be passed to the corresponding srcu_read_unlock_fast().
o In the absence of readers, a synchronize_srcu() having _fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.
Whether or not this can replace RCU Tasks Trace, it should replace the SRCU-lite API, where the only drawback is a pointer-sized return value from srcu_read_lock_fast() compared to the int-sized return value from srcu_read_lock_lite(). And yes, I should have thought harder before doing that SRCU-lite...
This series is as follows:
1. Make Tiny SRCU able to operate in preemptible kernels.
2. Define SRCU_READ_FLAVOR_ALL in terms of symbols.
3. Use ->srcu_gp_seq for rcutorture reader batch.
4. Pull ->srcu_{un,}lock_count into a new srcu_ctr structure.
5. Make SRCU readers use ->srcu_ctrs for counter selection.
6. Make Tree SRCU updates independent of ->srcu_idx.
7. Force synchronization for srcu_get_delay().
8. Rename srcu_check_read_flavor_lite() to
srcu_check_read_flavor_force().
9. Add SRCU_READ_FLAVOR_SLOWGP to flag need for synchronize_rcu().
10. Pull pointer-to-integer conversion into __srcu_ptr_to_ctr().
11. Pull integer-to-pointer conversion into __srcu_ctr_to_ptr().
12. Move SRCU Tree/Tiny definitions from srcu.h.
13. Add SRCU-fast readers.
14. Add ability to test srcu_read_{,un}lock_fast().
15. Add srcu_read_lock_fast() support using "srcu-fast".
16. Make scenario SRCU-P use srcu_read_lock_fast().
17. Fix srcu_read_unlock_{lite,nmisafe}() kernel-doc.
18. Document that srcu_{read_lock,down_read}() can share srcu_struct.
19. Add srcu_down_read_fast() and srcu_up_read_fast().
20. Make SRCU-fast also be NMI-safe.
Changes since v1:
o Explicitly document that srcu_{read_lock,down_read}() can share
an srcu_struct.
o Add srcu_down_read_fast() and srcu_up_read_fast() based on
feedback from Andrii Nakryiko.
o Make SRCU-fast readers NMI-safe based on feedback from Alexei
Starovoitov.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/srcu.h | 3 b/include/linux/srcutiny.h | 2 b/include/linux/srcutree.h | 13 b/kernel/rcu/rcu.h | 9 b/kernel/rcu/rcutorture.c | 2 b/kernel/rcu/refscale.c | 32 + b/kernel/rcu/srcutiny.c | 6 b/kernel/rcu/srcutree.c | 2 b/tools/testing/selftests/rcutorture/configs/rcu/SRCU-P.boot | 1 include/linux/srcu.h | 103 ++++ include/linux/srcutiny.h | 25 + include/linux/srcutree.h | 111 ++++- kernel/rcu/rcutorture.c | 9 kernel/rcu/srcutree.c | 233 +++++------ 14 files changed, 392 insertions(+), 159 deletions(-)
Paul E. McKenney
maintainerPATCH v2 1/1 01/20] srcu: Make Tiny SRCU able to operate in preemptible kernels
Given that SRCU allows its read-side critical sections are not just preemptible, but also allow general blocking, there is not much reason to restrict Tiny SRCU to non-preemptible kernels. This commit therefore removes Tiny SRCU dependencies on non-preemptibility, primarily surrounding its interaction with rcutorture and early boot.
--- kernel/rcu/rcu.h | 9 ++++++--- kernel/rcu/srcutiny.c | 6 ++++++ 2 files changed, 12 insertions(+), 3 deletions(-)
kernel/rcu/rcu.h +6 -3
@@ -611,8 +611,6 @@ void srcutorture_get_gp_data(struct srcu_struct *sp, int *flags,static inline bool rcu_watching_zero_in_eqs (int cpu,int *vp) {return false; }static inline unsigned long rcu_get_gp_seq (void ) {return 0 ; }static inline unsigned long rcu_exp_batches_completed (void ) {return 0 ; } -static inline unsigned long -srcu_batches_completed (struct srcu_struct *sp) {return 0 ; }static inline void rcu_force_quiescent_state (void ) { }static inline bool rcu_check_boost_fail (unsigned long gp_state,int *cpup) {return true; }static inline void show_rcu_gp_kthreads (void ) { }
@@ -624,7 +622,6 @@ static inline void rcu_gp_slow_unregister(atomic_t *rgssp) { }bool rcu_watching_zero_in_eqs (int cpu,int *vp);unsigned long rcu_get_gp_seq (void );unsigned long rcu_exp_batches_completed (void ); -unsigned long srcu_batches_completed (struct srcu_struct *sp);bool rcu_check_boost_fail (unsigned long gp_state,int *cpup);void show_rcu_gp_kthreads (void );int rcu_get_gp_kthreads_prio (void );
@@ -636,6 +633,12 @@ void rcu_gp_slow_register(atomic_t *rgssp);void rcu_gp_slow_unregister (atomic_t *rgssp);#endif /* #else #ifdef CONFIG_TINY_RCU */ +#ifdef CONFIG_TINY_SRCU +static inline unsigned long srcu_batches_completed (struct srcu_struct *sp) {return 0 ; } +#else // #ifdef CONFIG_TINY_SRCU +unsigned long srcu_batches_completed (struct srcu_struct *sp); +#endif // #else // #ifdef CONFIG_TINY_SRCU +#ifdef CONFIG_RCU_NOCB_CPU void rcu_bind_current_to_nocb (void );#else
kernel/rcu/srcutiny.c +6 -0
@@ -20,7 +20,11 @@#include "rcu_segcblist.h" #include "rcu.h" +#ifndef CONFIG_TREE_RCU int rcu_scheduler_active__read_mostly ; +#else // #ifndef CONFIG_TREE_RCU +extern int rcu_scheduler_active; +#endif // #else // #ifndef CONFIG_TREE_RCU static LIST_HEAD (srcu_boot_list);static bool srcu_init_done;
@@ -282,11 +286,13 @@ bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) }EXPORT_SYMBOL_GPL (poll_state_synchronize_srcu); +#ifndef CONFIG_TREE_RCU /* Lockdep diagnostics. */ void __init rcu_scheduler_starting (void ) { rcu_scheduler_active =RCU_SCHEDULER_RUNNING ; } +#endif // #ifndef CONFIG_TREE_RCU /* * Queue work for srcu_struct structures with early boot callbacks.
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 10/20] srcu: Pull pointer-to-integer conversion into __srcu_ptr_to_ctr()
In reply to Paul E. McKenneyThis commit abstracts the srcu_read_lock*() pointer-to-integer conversion into a new __srcu_ptr_to_ctr(). This will be used in rcutorture for testing an srcu_read_lock_fast() that returns a pointer rather than an integer.
--- include/linux/srcutree.h | 9 ++++++++- kernel/rcu/srcutree.c | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-)
include/linux/srcutree.h +8 -1
@@ -211,6 +211,13 @@ void synchronize_srcu_expedited(struct srcu_struct *ssp);void srcu_barrier (struct srcu_struct *ssp);void srcu_torture_stats_print (struct srcu_struct *ssp,char *tt,char *tf); +// Converts a per-CPU pointer to an ->srcu_ctrs[] array element to that +// element's index. +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return scpp - &ssp->sda->srcu_ctrs[0 ]; +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching
@@ -228,7 +235,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_lite()." );this_cpu_inc (scp->srcu_locks.counter);/* Y */ barrier ();/* Avoid leaking the critical section. */ -return scp - &ssp->sda->srcu_ctrs[0 ]; +return __srcu_ptr_to_ctr (ssp, scp); }/*
kernel/rcu/srcutree.c +2 -2
@@ -753,7 +753,7 @@ int __srcu_read_lock(struct srcu_struct *ssp)this_cpu_inc (scp->srcu_locks.counter);smp_mb ();/* B */ /* Avoid leaking the critical section. */ -return scp - &ssp->sda->srcu_ctrs[0 ]; +return __srcu_ptr_to_ctr (ssp, scp); }EXPORT_SYMBOL_GPL (__srcu_read_lock);
@@ -783,7 +783,7 @@ int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)atomic_long_inc (&scp->srcu_locks);smp_mb__after_atomic ();/* B */ /* Avoid leaking the critical section. */ -return scpp - &ssp->sda->srcu_ctrs[0 ]; +return __srcu_ptr_to_ctr (ssp, scpp); }EXPORT_SYMBOL_GPL (__srcu_read_lock_nmisafe);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 11/20] srcu: Pull integer-to-pointer conversion into __srcu_ctr_to_ptr()
In reply to Paul E. McKenneyThis commit abstracts the srcu_read_unlock*() integer-to-pointer conversion into a new __srcu_ctr_to_ptr(). This will be used in rcutorture for testing an srcu_read_unlock_fast() that avoids array-indexing overhead by taking a pointer rather than an integer.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/srcutree.h | 9 ++++++++- kernel/rcu/srcutree.c | 6 ++---- 2 files changed, 10 insertions(+), 5 deletions(-)
include/linux/srcutree.h +8 -1
@@ -218,6 +218,13 @@ static inline bool __srcu_ptr_to_ctr(struct srcu_struct *ssp, struct srcu_ctr __return scpp - &ssp->sda->srcu_ctrs[0 ]; } +// Converts an integer to a per-CPU pointer to the corresponding +// ->srcu_ctrs[] array element. +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return &ssp->sda->srcu_ctrs[idx]; +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching
@@ -252,7 +259,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)static inline void __srcu_read_unlock_lite (struct srcu_struct *ssp,int idx) {barrier ();/* Avoid leaking the critical section. */ -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter);/* Z */ +this_cpu_inc (__srcu_ctr_to_ptr (ssp, idx)->srcu_unlocks.counter);/* Z */ RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_lite()." ); }
kernel/rcu/srcutree.c +2 -4
@@ -765,7 +765,7 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock);void __srcu_read_unlock (struct srcu_struct *ssp,int idx) {smp_mb ();/* C */ /* Avoid leaking the critical section. */ -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter); +this_cpu_inc (__srcu_ctr_to_ptr (ssp, idx)->srcu_unlocks.counter); }EXPORT_SYMBOL_GPL (__srcu_read_unlock);
@@ -794,10 +794,8 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe);*/ void __srcu_read_unlock_nmisafe (struct srcu_struct *ssp,int idx) { -struct srcu_data *sdp =raw_cpu_ptr (ssp->sda); -smp_mb__before_atomic ();/* C */ /* Avoid leaking the critical section. */ -atomic_long_inc (&sdp->srcu_ctrs[idx].srcu_unlocks); +atomic_long_inc (&raw_cpu_ptr (__srcu_ctr_to_ptr (ssp, idx))->srcu_unlocks); }EXPORT_SYMBOL_GPL (__srcu_read_unlock_nmisafe);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 12/20] srcu: Move SRCU Tree/Tiny definitions from srcu.h
In reply to Paul E. McKenneyThere are a couple of definitions under "#ifdef CONFIG_TINY_SRCU" in include/linux/srcu.h. There is no point in them being there, so this commit moves them to include/linux/srcutiny.h and include/linux/srcutree.c, thus eliminating that #ifdef.
--- include/linux/srcu.h | 10 +--------- include/linux/srcutiny.h | 3 +++ include/linux/srcutree.h | 1 + 3 files changed, 5 insertions(+), 9 deletions(-)
include/linux/srcu.h +1 -9
@@ -52,6 +52,7 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE// Flavors requiring synchronize_rcu() // instead of smp_mb(). +void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);#ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h>
@@ -64,15 +65,6 @@ int init_srcu_struct(struct srcu_struct *ssp);void call_srcu (struct srcu_struct *ssp,struct rcu_head *head,void (*func)(struct rcu_head *head));void cleanup_srcu_struct (struct srcu_struct *ssp); -int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); -void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); -#ifdef CONFIG_TINY_SRCU -#define __srcu_read_lock_lite __srcu_read_lock -#define __srcu_read_unlock_lite __srcu_read_unlock -#else // #ifdef CONFIG_TINY_SRCU -int __srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp); -void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx) __releases(ssp); -#endif // #else // #ifdef CONFIG_TINY_SRCU void synchronize_srcu (struct srcu_struct *ssp);#define SRCU_GET_STATE_COMPLETED 0x1
include/linux/srcutiny.h +3 -0
@@ -71,6 +71,9 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +#define __srcu_read_lock_lite __srcu_read_lock +#define __srcu_read_unlock_lite __srcu_read_unlock +static inline void synchronize_srcu_expedited (struct srcu_struct *ssp) {synchronize_srcu (ssp);
include/linux/srcutree.h +1 -0
@@ -207,6 +207,7 @@ struct srcu_struct { #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)#define DEFINE_STATIC_SRCU (name) __DEFINE_SRCU(name, static) +int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);void synchronize_srcu_expedited (struct srcu_struct *ssp);void srcu_barrier (struct srcu_struct *ssp);void srcu_torture_stats_print (struct srcu_struct *ssp,char *tt,char *tf);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 02/20] srcu: Define SRCU_READ_FLAVOR_ALL in terms of symbols
In reply to Paul E. McKenneyThis commit defines SRCU_READ_FLAVOR_ALL in terms of the SRCU_READ_FLAVOR_* definitions instead of a hexadecimal constant.
--- include/linux/srcu.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/linux/srcu.h +2 -1
@@ -47,7 +47,8 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). -#define SRCU_READ_FLAVOR_ALL 0x7 // All of the above. +#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ + SRCU_READ_FLAVOR_LITE) // All of the above.#ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h>
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 03/20] srcu: Use ->srcu_gp_seq for rcutorture reader batch
In reply to Paul E. McKenneyThis commit stops using ->srcu_idx for rcutorture's reader-batch consistency checking, using ->srcu_gp_seq instead. This is a first step towards a faster srcu_read_{,un}lock_lite() that avoids the array accesses that use ->srcu_idx.
--- kernel/rcu/rcutorture.c | 2 ++ kernel/rcu/srcutree.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
kernel/rcu/rcutorture.c +2 -0
@@ -791,6 +791,7 @@ static struct rcu_torture_ops srcu_ops = { .readunlock = srcu_torture_read_unlock, .readlock_held = torture_srcu_read_lock_held, .get_gp_seq = srcu_torture_completed, + .gp_diff = rcu_seq_diff, .deferred_free = srcu_torture_deferred_free, .sync = srcu_torture_synchronize, .exp_sync = srcu_torture_synchronize_expedited,
@@ -834,6 +835,7 @@ static struct rcu_torture_ops srcud_ops = { .readunlock = srcu_torture_read_unlock, .readlock_held = torture_srcu_read_lock_held, .get_gp_seq = srcu_torture_completed, + .gp_diff = rcu_seq_diff, .deferred_free = srcu_torture_deferred_free, .sync = srcu_torture_synchronize, .exp_sync = srcu_torture_synchronize_expedited,
kernel/rcu/srcutree.c +1 -1
@@ -1679,7 +1679,7 @@ EXPORT_SYMBOL_GPL(srcu_barrier);*/ unsigned long srcu_batches_completed (struct srcu_struct *ssp) { -return READ_ONCE (ssp->srcu_idx); +return READ_ONCE (ssp->srcu_sup->srcu_gp_seq); }EXPORT_SYMBOL_GPL (srcu_batches_completed);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 04/20] srcu: Pull ->srcu_{un,}lock_count into a new srcu_ctr structure
In reply to Paul E. McKenneyThis commit prepares for array-index-free srcu_read_lock*() by moving the ->srcu_{un,}lock_count fields into a new srcu_ctr structure. This will permit ->srcu_index to be replaced by a per-CPU pointer to this structure.
--- include/linux/srcutree.h | 13 +++-- kernel/rcu/srcutree.c | 115 +++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 62 deletions(-)
include/linux/srcutree.h +9 -4
@@ -17,14 +17,19 @@struct srcu_node ;struct srcu_struct ; +/* One element of the srcu_data srcu_ctrs array. */ +struct srcu_ctr { +atomic_long_t srcu_locks;/* Locks per CPU. */ +atomic_long_t srcu_unlocks;/* Unlocks per CPU. */ +}; +/* * Per-CPU structure feeding into leaf srcu_node, similar in function * to rcu_node. */ struct srcu_data {/* Read-side state. */ -atomic_long_t srcu_lock_count[2 ];/* Locks per CPU. */ -atomic_long_t srcu_unlock_count[2 ];/* Unlocks per CPU. */ +struct srcu_ctr srcu_ctrs[2 ];/* Locks and unlocks per CPU. */ int srcu_reader_flavor;/* Reader flavor for srcu_struct structure? */ /* Values: SRCU_READ_FLAVOR_.* */
@@ -221,7 +226,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_lite()." ); idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -this_cpu_inc (ssp->sda->srcu_lock_count[idx].counter);/* Y */ +this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_locks.counter);/* Y */ barrier ();/* Avoid leaking the critical section. */ return idx; }
@@ -240,7 +245,7 @@ static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)static inline void __srcu_read_unlock_lite (struct srcu_struct *ssp,int idx) {barrier ();/* Avoid leaking the critical section. */ -this_cpu_inc (ssp->sda->srcu_unlock_count[idx].counter);/* Z */ +this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_unlocks.counter);/* Z */ RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_lite()." ); }
kernel/rcu/srcutree.c +57 -58
221 lines not on this page · raw message
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 05/20] srcu: Make SRCU readers use ->srcu_ctrs for counter selection
In reply to Paul E. McKenneyThis commit causes SRCU readers to use ->srcu_ctrs for counter selection instead of ->srcu_idx. This takes another step towards array-indexing-free SRCU readers.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/srcutree.h | 9 +++++---- kernel/rcu/srcutree.c | 23 +++++++++++++---------- 2 files changed, 18 insertions(+), 14 deletions(-)
include/linux/srcutree.h +5 -4
@@ -101,6 +101,7 @@ struct srcu_usage {*/ struct srcu_struct {unsigned int srcu_idx;/* Current rdr array element. */ +struct srcu_ctr __percpu *srcu_ctrp;struct srcu_data __percpu *sda;/* Per-CPU srcu_data array. */ struct lockdep_map dep_map;struct srcu_usage *srcu_sup;/* Update-side data. */
@@ -167,6 +168,7 @@ struct srcu_struct {#define __SRCU_STRUCT_INIT (name, usage_name, pcpu_name) \ { \ .sda = &pcpu_name, \ + .srcu_ctrp = &pcpu_name.srcu_ctrs[0], \ __SRCU_STRUCT_INIT_COMMON(name, usage_name) \ }
@@ -222,13 +224,12 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf);*/ static inline int __srcu_read_lock_lite (struct srcu_struct *ssp) { -int idx; +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp);RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_lite()." ); - idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_locks.counter);/* Y */ +this_cpu_inc (scp->srcu_locks.counter);/* Y */ barrier ();/* Avoid leaking the critical section. */ -return idx; +return scp - &ssp->sda->srcu_ctrs[0 ]; }/*
kernel/rcu/srcutree.c +13 -10
@@ -253,8 +253,10 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)atomic_set (&ssp->srcu_sup->srcu_barrier_cpu_cnt,0 );INIT_DELAYED_WORK (&ssp->srcu_sup->work, process_srcu); ssp->srcu_sup->sda_is_static = is_static; -if (!is_static) +if (!is_static) { ssp->sda = alloc_percpu(struct srcu_data); + ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0 ]; + }if (!ssp->sda) goto err_free_sup;init_srcu_struct_data (ssp);
@@ -742,12 +744,11 @@ EXPORT_SYMBOL_GPL(__srcu_check_read_flavor);*/ int __srcu_read_lock (struct srcu_struct *ssp) { -int idx; +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); - idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -this_cpu_inc (ssp->sda->srcu_ctrs[idx].srcu_locks.counter); +this_cpu_inc (scp->srcu_locks.counter);smp_mb ();/* B */ /* Avoid leaking the critical section. */ -return idx; +return scp - &ssp->sda->srcu_ctrs[0 ]; }EXPORT_SYMBOL_GPL (__srcu_read_lock);
@@ -772,13 +773,12 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock);*/ int __srcu_read_lock_nmisafe (struct srcu_struct *ssp) { -int idx; -struct srcu_data *sdp =raw_cpu_ptr (ssp->sda); +struct srcu_ctr __percpu *scpp =READ_ONCE (ssp->srcu_ctrp); +struct srcu_ctr *scp =raw_cpu_ptr (scpp); - idx =READ_ONCE (ssp->srcu_idx) &0x1 ; -atomic_long_inc (&sdp->srcu_ctrs[idx].srcu_locks); +atomic_long_inc (&scp->srcu_locks);smp_mb__after_atomic ();/* B */ /* Avoid leaking the critical section. */ -return idx; +return scpp - &ssp->sda->srcu_ctrs[0 ]; }EXPORT_SYMBOL_GPL (__srcu_read_lock_nmisafe);
@@ -1152,6 +1152,8 @@ static void srcu_flip(struct srcu_struct *ssp)smp_mb ();/* E */ /* Pairs with B and C. */ WRITE_ONCE (ssp->srcu_idx, ssp->srcu_idx +1 );// Flip the counter. +WRITE_ONCE (ssp->srcu_ctrp, + &ssp->sda->srcu_ctrs[!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0 ])]);/* * Ensure that if the updater misses an __srcu_read_unlock()
@@ -2004,6 +2006,7 @@ static int srcu_module_coming(struct module *mod) ssp->sda = alloc_percpu(struct srcu_data);if (WARN_ON_ONCE (!ssp->sda))return -ENOMEM ; + ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0 ]; }return 0 ; }
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 06/20] srcu: Make Tree SRCU updates independent of ->srcu_idx
In reply to Paul E. McKenneyThis commit makes Tree SRCU updates independent of ->srcu_idx, then drop ->srcu_idx.
--- include/linux/srcutree.h | 1 - kernel/rcu/srcutree.c | 68 ++++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 35 deletions(-)
include/linux/srcutree.h +0 -1
@@ -100,7 +100,6 @@ struct srcu_usage {* Per-SRCU-domain structure, similar in function to rcu_state. */ struct srcu_struct { -unsigned int srcu_idx;/* Current rdr array element. */ struct srcu_ctr __percpu *srcu_ctrp;struct srcu_data __percpu *sda;/* Per-CPU srcu_data array. */ struct lockdep_map dep_map;
kernel/rcu/srcutree.c +34 -34
158 lines not on this page · raw message
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 07/20] srcu: Force synchronization for srcu_get_delay()
In reply to Paul E. McKenneyCurrently, srcu_get_delay() can be called concurrently, for example, by a CPU that is the first to request a new grace period and the CPU processing the current grace period. Although concurrent access is harmless, it unnecessarily expands the state space. Additionally, all calls to srcu_get_delay() are from slow paths.
This commit therefore protects all calls to srcu_get_delay() with ssp->srcu_sup->lock, which is already held on the invocation from the srcu_funnel_gp_start() function. While in the area, this commit also adds a lockdep_assert_held() to srcu_get_delay() itself.
--- kernel/rcu/srcutree.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
kernel/rcu/srcutree.c +10 -1
@@ -648,6 +648,7 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)unsigned long jbase =SRCU_INTERVAL ;struct srcu_usage *sup = ssp->srcu_sup; +lockdep_assert_held (&ACCESS_PRIVATE (ssp->srcu_sup, lock));if (srcu_gp_is_expedited (ssp)) jbase =0 ;if (rcu_seq_state (READ_ONCE (sup->srcu_gp_seq))) {
@@ -675,9 +676,13 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)void cleanup_srcu_struct (struct srcu_struct *ssp) {int cpu; +unsigned long delay;struct srcu_usage *sup = ssp->srcu_sup; -if (WARN_ON (!srcu_get_delay (ssp))) +spin_lock_irq_rcu_node (ssp->srcu_sup); + delay =srcu_get_delay (ssp); +spin_unlock_irq_rcu_node (ssp->srcu_sup); +if (WARN_ON (!delay))return ;/* Just leak it! */ if (WARN_ON (srcu_readers_active (ssp)))return ;/* Just leak it! */
@@ -1101,7 +1106,9 @@ static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) {unsigned long curdelay; +spin_lock_irq_rcu_node (ssp->srcu_sup); curdelay = !srcu_get_delay (ssp); +spin_unlock_irq_rcu_node (ssp->srcu_sup);for (;;) {if (srcu_readers_active_idx_check (ssp, idx))
@@ -1854,7 +1861,9 @@ static void process_srcu(struct work_struct *work) ssp = sup->srcu_ssp;srcu_advance_state (ssp); +spin_lock_irq_rcu_node (ssp->srcu_sup); curdelay =srcu_get_delay (ssp); +spin_unlock_irq_rcu_node (ssp->srcu_sup);if (curdelay) {WRITE_ONCE (sup->reschedule_count,0 ); }else {
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 08/20] srcu: Rename srcu_check_read_flavor_lite() to srcu_check_read_flavor_force()
In reply to Paul E. McKenneyThis commit renames the srcu_check_read_flavor_lite() function to srcu_check_read_flavor_force() and adds a read_flavor argument in order to support an srcu_read_lock_fast() variant that is to avoid array indexing in both the lock and unlock primitives.
--- include/linux/srcu.h | 2 +- include/linux/srcutiny.h | 2 +- include/linux/srcutree.h | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-)
include/linux/srcu.h +1 -1
@@ -279,7 +279,7 @@ static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp) {int retval; -srcu_check_read_flavor_lite (ssp); +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_LITE ); retval =__srcu_read_lock_lite (ssp);rcu_try_lock_acquire (&ssp->dep_map);return retval;
include/linux/srcutiny.h +1 -1
@@ -82,7 +82,7 @@ static inline void srcu_barrier(struct srcu_struct *ssp) }#define srcu_check_read_flavor (ssp, read_flavor) do { } while (0) -#define srcu_check_read_flavor_lite (ssp) do { } while (0) +#define srcu_check_read_flavor_force (ssp, read_flavor) do { } while (0)/* Defined here to avoid size increase for non-torture kernels. */ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
include/linux/srcutree.h +6 -4
@@ -251,16 +251,18 @@ static inline void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)void __srcu_check_read_flavor (struct srcu_struct *ssp,int read_flavor); -// Record _lite() usage even for CONFIG_PROVE_RCU=n kernels. -static inline void srcu_check_read_flavor_lite (struct srcu_struct *ssp) +// Record reader usage even for CONFIG_PROVE_RCU=n kernels. This is +// needed only for flavors that require grace-period smp_mb() calls to be +// promoted to synchronize_rcu(). +static inline void srcu_check_read_flavor_force (struct srcu_struct *ssp,int read_flavor) {struct srcu_data *sdp =raw_cpu_ptr (ssp->sda); -if (likely (READ_ONCE (sdp->srcu_reader_flavor) &SRCU_READ_FLAVOR_LITE )) +if (likely (READ_ONCE (sdp->srcu_reader_flavor) & read_flavor))return ;// Note that the cmpxchg() in __srcu_check_read_flavor() is fully ordered. -__srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_LITE ); +__srcu_check_read_flavor (ssp, read_flavor); }// Record non-_lite() usage only for CONFIG_PROVE_RCU=y kernels.
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 09/20] srcu: Add SRCU_READ_FLAVOR_SLOWGP to flag need for synchronize_rcu()
In reply to Paul E. McKenneyThis commit switches from a direct test of SRCU_READ_FLAVOR_LITE to a new SRCU_READ_FLAVOR_SLOWGP macro to check for substituting synchronize_rcu() for smp_mb() in SRCU grace periods. Right now, SRCU_READ_FLAVOR_SLOWGP is exactly SRCU_READ_FLAVOR_LITE, but the addition of the _fast() flavor of SRCU will change that.
--- include/linux/srcu.h | 3 +++ kernel/rcu/srcutree.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-)
include/linux/srcu.h +3 -0
@@ -49,6 +49,9 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ SRCU_READ_FLAVOR_LITE) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE +// Flavors requiring synchronize_rcu() +// instead of smp_mb(). #ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h>
kernel/rcu/srcutree.c +3 -3
@@ -449,7 +449,7 @@ static bool srcu_readers_lock_idx(struct srcu_struct *ssp, int idx, bool gp, uns }WARN_ONCE (IS_ENABLED (CONFIG_PROVE_RCU ) && (mask & (mask -1 )),"Mixed reader flavors for srcu_struct at %ps.\n" , ssp); -if (mask &SRCU_READ_FLAVOR_LITE && !gp) +if (mask &SRCU_READ_FLAVOR_SLOWGP && !gp)return false;return sum == unlocks; }
@@ -487,7 +487,7 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)unsigned long unlocks; unlocks =srcu_readers_unlock_idx (ssp, idx, &rdm); - did_gp = !!(rdm &SRCU_READ_FLAVOR_LITE ); + did_gp = !!(rdm &SRCU_READ_FLAVOR_SLOWGP );/* * Make sure that a lock is always counted if the corresponding
@@ -1205,7 +1205,7 @@ static bool srcu_should_expedite(struct srcu_struct *ssp)check_init_srcu_struct (ssp);/* If _lite() readers, don't do unsolicited expediting. */ -if (this_cpu_read (ssp->sda->srcu_reader_flavor) &SRCU_READ_FLAVOR_LITE ) +if (this_cpu_read (ssp->sda->srcu_reader_flavor) &SRCU_READ_FLAVOR_SLOWGP )return false;/* If the local srcu_data structure has callbacks, not idle. */ sdp =raw_cpu_ptr (ssp->sda);
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 13/20] srcu: Add SRCU-fast readers
In reply to Paul E. McKenneyThis commit adds srcu_read_{,un}lock_fast(), which is similar to srcu_read_{,un}lock_lite(), but avoids the array-indexing and pointer-following overhead. On a microbenchmark featuring tight loops around empty readers, this results in about a 20% speedup compared to RCU Tasks Trace on my x86 laptop.
Please note that SRCU-fast has drawbacks compared to RCU Tasks Trace, including:
o Lack of CPU stall warnings.
o SRCU-fast readers permitted only where rcu_is_watching().
o A pointer-sized return value from srcu_read_lock_fast() must
be passed to the corresponding srcu_read_unlock_fast().
o In the absence of readers, a synchronize_srcu() having _fast()
readers will incur the latency of at least two normal RCU grace periods.
o RCU Tasks Trace priority boosting could be easily added.
Boosting SRCU readers is more difficult.
SRCU-fast also has a drawback compared to SRCU-lite, namely that the return value from srcu_read_lock_fast()-fast is a 64-bit pointer and that from srcu_read_lock_lite() is only a 32-bit int.
[ paulmck: Apply feedback from Akira Yokosawa. ]
--- include/linux/srcu.h | 47 ++++++++++++++++++++++++++++++++++++++-- include/linux/srcutiny.h | 22 +++++++++++++++++++ include/linux/srcutree.h | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-)
include/linux/srcu.h +45 -2
@@ -47,9 +47,10 @@ int init_srcu_struct(struct srcu_struct *ssp);#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite(). +#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_LITE) // All of the above. -#define SRCU_READ_FLAVOR_SLOWGP SRCU_READ_FLAVOR_LITE + SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above. +#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)// Flavors requiring synchronize_rcu() // instead of smp_mb(). void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
@@ -253,6 +254,33 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)return retval; } +/** +* srcu_read_lock_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter an SRCU read-side critical section, but for a light-weight +* smp_mb()-free reader. See srcu_read_lock() for more information. +* +* If srcu_read_lock_fast() is ever used on an srcu_struct structure, +* then none of the other flavors may be used, whether before, during, +* or after. Note that grace-period auto-expediting is disabled for _fast +* srcu_struct structures because auto-expedited grace periods invoke +* synchronize_rcu_expedited(), IPIs and all. +* +* Note that srcu_read_lock_fast() can be invoked only from those contexts +* where RCU is watching, that is, from contexts where it would be legal +* to invoke rcu_read_lock(). Otherwise, lockdep will complain. +*/ +static inline struct srcu_ctr __percpu *srcu_read_lock_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +struct srcu_ctr __percpu *retval; + +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); + retval =__srcu_read_lock_fast (ssp); +rcu_try_lock_acquire (&ssp->dep_map); +return retval; +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader.
@@ -356,6 +384,21 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)__srcu_read_unlock (ssp, idx); } +/** +* srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit a light-weight SRCU read-side critical section. +*/ +static inline void srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +srcu_lock_release (&ssp->dep_map); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader.
include/linux/srcutiny.h +22 -0
@@ -71,6 +71,28 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)return idx; } +struct srcu_ctr ; + +static inline bool __srcu_ptr_to_ctr (struct srcu_struct *ssp,struct srcu_ctr __percpu *scpp) +{ +return (int )(intptr_t )(struct srcu_ctr __force __kernel *)scpp; +} + +static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr (struct srcu_struct *ssp,int idx) +{ +return (struct srcu_ctr __percpu *)(intptr_t )idx; +} + +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +return __srcu_ctr_to_ptr (ssp,__srcu_read_lock (ssp)); +} + +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +__srcu_read_unlock (ssp,__srcu_ptr_to_ctr (ssp, scp)); +} +#define __srcu_read_lock_lite __srcu_read_lock#define __srcu_read_unlock_lite __srcu_read_unlock
include/linux/srcutree.h +38 -0
@@ -226,6 +226,44 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ssreturn &ssp->sda->srcu_ctrs[idx]; } +/* +* Counts the new reader in the appropriate per-CPU element of the +* srcu_struct. Returns a pointer that must be passed to the matching +* srcu_read_unlock_fast(). +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) +{ +struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp); + +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +barrier ();/* Avoid leaking the critical section. */ +return scp; +} + +/* +* Removes the count for the old reader from the appropriate +* per-CPU element of the srcu_struct. Note that this may well be a +* different CPU than that which was incremented by the corresponding +* srcu_read_lock_fast(), but it must be within the same task. +* +* Note that this_cpu_inc() is an RCU read-side critical section either +* because it disables interrupts, because it is a single instruction, +* or because it is a read-modify-write atomic operation, depending on +* the whims of the architecture. +*/ +static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +{ +barrier ();/* Avoid leaking the critical section. */ +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); +} +/* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct. Returns an index that must be passed to the matching
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 14/20] rcutorture: Add ability to test srcu_read_{,un}lock_fast()
In reply to Paul E. McKenneyThis commit permits rcutorture to test srcu_read_{,un}lock_fast(), which is specified by the rcutorture.reader_flavor=0x8 kernel boot parameter.
--- kernel/rcu/rcutorture.c | 9 +++++++++ 1 file changed, 9 insertions(+)
kernel/rcu/rcutorture.c +9 -0
@@ -677,6 +677,7 @@ static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)static int srcu_torture_read_lock (void ) {int idx; +struct srcu_ctr __percpu *scp;int ret =0 ;if ((reader_flavor &SRCU_READ_FLAVOR_NORMAL ) || !(reader_flavor &SRCU_READ_FLAVOR_ALL )) {
@@ -694,6 +695,12 @@ static int srcu_torture_read_lock(void)WARN_ON_ONCE (idx & ~0x1 ); ret += idx <<2 ; } +if (reader_flavor &SRCU_READ_FLAVOR_FAST ) { + scp =srcu_read_lock_fast (srcu_ctlp); + idx =__srcu_ptr_to_ctr (srcu_ctlp, scp); +WARN_ON_ONCE (idx & ~0x1 ); + ret += idx <<3 ; + }return ret; }
@@ -719,6 +726,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)static void srcu_torture_read_unlock (int idx) {WARN_ON_ONCE ((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1 ))); +if (reader_flavor &SRCU_READ_FLAVOR_FAST ) +srcu_read_unlock_fast (srcu_ctlp,__srcu_ctr_to_ptr (srcu_ctlp, (idx &0x8 ) >>3 ));if (reader_flavor &SRCU_READ_FLAVOR_LITE )srcu_read_unlock_lite (srcu_ctlp, (idx &0x4 ) >>2 );if (reader_flavor &SRCU_READ_FLAVOR_NMI )
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 15/20] refscale: Add srcu_read_lock_fast() support using "srcu-fast"
In reply to Paul E. McKenneyThis commit creates a new srcu-fast option for the refscale.scale_type module parameter that selects srcu_read_lock_fast() and srcu_read_unlock_fast().
--- kernel/rcu/refscale.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-)
kernel/rcu/refscale.c +31 -1
@@ -216,6 +216,36 @@ static const struct ref_scale_ops srcu_ops = { .name = "srcu" }; +static void srcu_fast_ref_scale_read_section (const int nloops) +{ +int i; +struct srcu_ctr __percpu *scp; + +for (i = nloops; i >=0 ; i--) { + scp =srcu_read_lock_fast (srcu_ctlp); +srcu_read_unlock_fast (srcu_ctlp, scp); + } +} + +static void srcu_fast_ref_scale_delay_section (const int nloops,const int udl,const int ndl) +{ +int i; +struct srcu_ctr __percpu *scp; + +for (i = nloops; i >=0 ; i--) { + scp =srcu_read_lock_fast (srcu_ctlp); +un_delay (udl, ndl); +srcu_read_unlock_fast (srcu_ctlp, scp); + } +} + +static const struct ref_scale_ops srcu_fast_ops = { + .init = rcu_sync_scale_init, + .readsection = srcu_fast_ref_scale_read_section, + .delaysection = srcu_fast_ref_scale_delay_section, + .name ="srcu-fast" +}; +static void srcu_lite_ref_scale_read_section (const int nloops) {int i;
@@ -1163,7 +1193,7 @@ ref_scale_init(void)long i;int firsterr =0 ;static const struct ref_scale_ops *scale_ops[] = { - &rcu_ops, &srcu_ops, &srcu_lite_ops, RCU_TRACE_OPS RCU_TASKS_OPS + &rcu_ops, &srcu_ops, &srcu_fast_ops, &srcu_lite_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 16/20] rcutorture: Make scenario SRCU-P use srcu_read_lock_fast()
In reply to Paul E. McKenneyThis commit causes the rcutorture SRCU-P scenario use the srcu_read_lock_fast() and srcu_read_unlock_fast() functions. This will cause these two functions to be regularly tested by several developers (myself included), for example, those who use torture.sh as an RCU acceptance test.
--- tools/testing/selftests/rcutorture/configs/rcu/SRCU-P.boot | 1 + 1 file changed, 1 insertion(+)
tools/testing/selftests/rcutorture/configs/rcu/SRCU-P.boot +1 -0
@@ -2,3 +2,4 @@ rcutorture.torture_type=srcud rcupdate.rcu_self_test=1 rcutorture.fwd_progress=3 srcutree.big_cpu_lim=5 +rcutorture.reader_flavor=0x8
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 17/20] srcu: Fix srcu_read_unlock_{lite,nmisafe}() kernel-doc
In reply to Paul E. McKenneyThe srcu_read_unlock_lite() and srcu_read_unlock_nmisafe() both say that their idx parameters must come from srcu_read_lock(). This would be bad, because a given srcu_struct structure may be used only with one flavor of SRCU reader. This commit therefore updates the srcu_read_unlock_lite() kernel-doc header to say that its idx parameter must be obtained from srcu_read_lock_lite() and the srcu_read_unlock_nmisafe() kernel-doc header to say that its idx parameter must be obtained from srcu_read_lock_nmisafe().
--- include/linux/srcu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/srcu.h +2 -2
@@ -402,7 +402,7 @@ static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ct/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. -* @idx: return value from corresponding srcu_read_lock(). +* @idx: return value from corresponding srcu_read_lock_lite(). * * Exit a light-weight SRCU read-side critical section. */
@@ -418,7 +418,7 @@ static inline void srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)/** * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. -* @idx: return value from corresponding srcu_read_lock(). +* @idx: return value from corresponding srcu_read_lock_nmisafe(). * * Exit an SRCU read-side critical section, but in an NMI-safe manner. */
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 18/20] srcu: Document that srcu_{read_lock,down_read}() can share srcu_struct
In reply to Paul E. McKenneyThis commit adds a sentence to the srcu_down_read() function's kernel-doc header noting that it is permissible to use srcu_down_read() and srcu_read_lock() on the same srcu_struct, even concurrently.
--- include/linux/srcu.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/linux/srcu.h +2 -1
@@ -359,7 +359,8 @@ srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)* srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler. * * Calls to srcu_down_read() may be nested, similar to the manner in -* which calls to down_read() may be nested. +* which calls to down_read() may be nested. The same srcu_struct may be +* used concurrently by srcu_down_read() and srcu_read_lock(). */ static inline int srcu_down_read (struct srcu_struct *ssp)__acquires (ssp) {
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 19/20] srcu: Add srcu_down_read_fast() and srcu_up_read_fast()
In reply to Paul E. McKenneyA pair of matching srcu_read_lock_fast() and srcu_read_unlock_fast() invocations must take place within the same context, for example, within the same task. Otherwise, lockdep complains, as is the right thing to do for most use cases.
However, there are use cases involving tracing (for example, uretprobes) in which an SRCU reader needs to begin in one task and end in a timer handler, which might interrupt some other task. This commit therefore supplies the semaphore-like srcu_down_read_fast() and srcu_up_read_fast() functions, which act like srcu_read_lock_fast() and srcu_read_unlock_fast(), but permitting srcu_up_read_fast() to be invoked in a different context than was the matching srcu_down_read_fast().
Neither srcu_down_read_fast() nor srcu_up_read_fast() may be invoked from an NMI handler.
--- include/linux/srcu.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)
include/linux/srcu.h +34 -0
@@ -281,6 +281,24 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *return retval; } +/** +* srcu_down_read_fast - register a new reader for an SRCU-protected structure. +* @ssp: srcu_struct in which to register the new reader. +* +* Enter a semaphore-like SRCU read-side critical section, but for +* a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and +* srcu_down_read() for more information. +* +* The same srcu_struct may be used concurrently by srcu_down_read_fast() +* and srcu_read_lock_fast(). +*/ +static inline struct srcu_ctr __percpu *srcu_down_read_fast (struct srcu_struct *ssp)__acquires (ssp) +{ +WARN_ON_ONCE (IS_ENABLED (CONFIG_PROVE_RCU ) &&in_nmi ()); +srcu_check_read_flavor_force (ssp,SRCU_READ_FLAVOR_FAST ); +return __srcu_read_lock_fast (ssp); +} +/** * srcu_read_lock_lite - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader.
@@ -400,6 +418,22 @@ static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ct__srcu_read_unlock_fast (ssp, scp); } +/** +* srcu_up_read_fast - unregister a old reader from an SRCU-protected structure. +* @ssp: srcu_struct in which to unregister the old reader. +* @scp: return value from corresponding srcu_read_lock_fast(). +* +* Exit an SRCU read-side critical section, but not necessarily from +* the same context as the maching srcu_down_read_fast(). +*/ +static inline void srcu_up_read_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) +__releases (ssp) +{ +WARN_ON_ONCE (IS_ENABLED (CONFIG_PROVE_RCU ) &&in_nmi ()); +srcu_check_read_flavor (ssp,SRCU_READ_FLAVOR_FAST ); +__srcu_read_unlock_fast (ssp, scp); +} +/** * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader.
Signature · 1 line
2.40.1
Paul E. McKenney
maintainerPATCH v2 1/1 20/20] srcu: Make SRCU-fast also be NMI-safe
In reply to Paul E. McKenneyBPF uses rcu_read_lock_trace() in NMI context, so srcu_read_lock_fast() must be NMI-safe if it is to have any chance of addressing RCU Tasks Trace use cases. This commit therefore causes srcu_read_lock_fast() and srcu_read_unlock_fast() to use atomic_long_inc() instead of this_cpu_inc() on architectures that support NMIs but do not have NMI-safe implementations of this_cpu_inc(). Note that both x86 and arm64 have NMI-safe implementations of this_cpu_inc(), and thus do not pay the performance penalty inherent in atomic_inc_long().
It is tempting to use this trick to fold srcu_read_lock_nmisafe() into srcu_read_lock(), but this would need careful thought, review, and performance analysis. Though those smp_mb() calls might well make performance a non-issue.
--- include/linux/srcutree.h | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-)
include/linux/srcutree.h +24 -10
@@ -231,17 +231,24 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss* srcu_struct. Returns a pointer that must be passed to the matching * srcu_read_unlock_fast(). * -* Note that this_cpu_inc() is an RCU read-side critical section either -* because it disables interrupts, because it is a single instruction, -* or because it is a read-modify-write atomic operation, depending on -* the whims of the architecture. +* Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side +* critical sections either because they disables interrupts, because they +* are a single instruction, or because they are a read-modify-write atomic +* operation, depending on the whims of the architecture. +* +* This means that __srcu_read_lock_fast() is not all that fast +* on architectures that support NMIs but do not supply NMI-safe +* implementations of this_cpu_inc(). */ static inline struct srcu_ctr __percpu *__srcu_read_lock_fast (struct srcu_struct *ssp) {struct srcu_ctr __percpu *scp =READ_ONCE (ssp->srcu_ctrp);RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_lock_fast()." ); -this_cpu_inc (scp->srcu_locks.counter);/* Y */ +if (!IS_ENABLED (CONFIG_NEED_SRCU_NMI_SAFE )) +this_cpu_inc (scp->srcu_locks.counter);/* Y */ +else +atomic_long_inc (raw_cpu_ptr (&scp->srcu_locks));/* Z */ barrier ();/* Avoid leaking the critical section. */ return scp; }
@@ -252,15 +259,22 @@ static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct* different CPU than that which was incremented by the corresponding * srcu_read_lock_fast(), but it must be within the same task. * -* Note that this_cpu_inc() is an RCU read-side critical section either -* because it disables interrupts, because it is a single instruction, -* or because it is a read-modify-write atomic operation, depending on -* the whims of the architecture. +* Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side +* critical sections either because they disables interrupts, because they +* are a single instruction, or because they are a read-modify-write atomic +* operation, depending on the whims of the architecture. +* +* This means that __srcu_read_unlock_fast() is not all that fast +* on architectures that support NMIs but do not supply NMI-safe +* implementations of this_cpu_inc(). */ static inline void __srcu_read_unlock_fast (struct srcu_struct *ssp,struct srcu_ctr __percpu *scp) {barrier ();/* Avoid leaking the critical section. */ -this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +if (!IS_ENABLED (CONFIG_NEED_SRCU_NMI_SAFE )) +this_cpu_inc (scp->srcu_unlocks.counter);/* Z */ +else +atomic_long_inc (raw_cpu_ptr (&scp->srcu_unlocks));/* Z */ RCU_LOCKDEP_WARN (!rcu_is_watching (),"RCU must be watching srcu_read_unlock_fast()." ); }
Signature · 1 line
2.40.1