Polled grace-period updates for v6.1
Subsystems (2)
Paul E. McKenney
maintainercover Polled grace-period updates for v6.1
Hello!
This series adds support for full-state polled RCU grace periods, both normal and expedited:
1. Add full-sized polling for get_completed*() and poll_state*().
2. Add full-sized polling for get_state().
3. Abstract synchronous and polled API testing.
4. Allow per-RCU-flavor polled double-GP check.
5. Verify RCU reader prevents full polling from completing.
6. Remove redundant RTWS_DEF_FREE check.
7. Verify long-running reader prevents full polling from completing.
8. Add full-sized polling for start_poll().
9. Add full-sized polling for start_poll_expedited().
10. Remove blank line from poll_state_synchronize_rcu() docbook
header.
11. Add full-sized polling for cond_sync_full().
12. Add full-sized polling for cond_sync_exp_full().
13. Disable run-time single-CPU grace-period optimization.
14. Set rcu_data structures' initial ->gpwrap value to true.
15. rcu-tasks: Remove grace-period fast-path rcu-tasks helper.
16. Make synchronize_rcu() fast path update ->gp_seq counters.
17. Remove expedited grace-period fast-path forward-progress helper.
18. Make synchronize_rcu_expedited() fast path update
.expedited_sequence.
19. Remove ->rgos_polled field from rcu_gp_oldstate structure.
20. Adjust rcu_poll_need_2gp() for rcu_gp_oldstate field removal.
21. Make synchronize_rcu() fastpath update only boot-CPU counters.
22. Use 1-suffixed variable in rcu_torture_write_types() check.
23. Expand rcu_torture_write_types() first "if" statement.
24. Add functions to compare grace-period state values.
25. Limit read-side polling-API testing.
Thanx, Paul
------------------------------------------------------------------------
b/include/linux/rcupdate.h | 3 b/include/linux/rcutiny.h | 9 b/include/linux/rcutree.h | 8 b/kernel/rcu/rcutorture.c | 9 b/kernel/rcu/tiny.c | 10 + b/kernel/rcu/tree.c | 76 +++++++- b/kernel/rcu/tree_exp.h | 18 + include/linux/rcupdate.h | 21 ++ include/linux/rcutiny.h | 41 ++++ include/linux/rcutree.h | 34 +++ kernel/rcu/rcutorture.c | 412 +++++++++++++++++++++++++++++++++------------ kernel/rcu/tree.c | 230 +++++++++++++++++-------- kernel/rcu/tree_exp.h | 35 +++ 13 files changed, 720 insertions(+), 186 deletions(-)
Paul E. McKenney
maintainerPATCH 1/25 rcu: Add full-sized polling for get_completed*() and poll_state*()
The get_completed_synchronize_rcu() and poll_state_synchronize_rcu() APIs compress the combined expedited and normal grace-period states into a single unsigned long, which conserves storage, but can miss grace periods in certain cases involving overlapping normal and expedited grace periods. Missing the occasional grace period is usually not a problem, but there are use cases that care about each and every grace period.
This commit therefore adds the first members of the full-state RCU grace-period polling API, namely the get_completed_synchronize_rcu_full() and poll_state_synchronize_rcu_full() functions. These use up to three times the storage (rcu_gp_oldstate structure instead of unsigned long), but which are guaranteed not to miss grace periods, at least in situations where the single-CPU grace-period optimization does not apply.
--- include/linux/rcupdate.h | 3 ++ include/linux/rcutiny.h | 9 +++++ include/linux/rcutree.h | 8 +++++ kernel/rcu/rcutorture.c | 9 +++++ kernel/rcu/tiny.c | 10 ++++++ kernel/rcu/tree.c | 76 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 111 insertions(+), 4 deletions(-)
include/linux/rcupdate.h +3 -0
@@ -42,7 +42,10 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func);void rcu_barrier_tasks (void );void rcu_barrier_tasks_rude (void );void synchronize_rcu (void ); + +struct rcu_gp_oldstate ;unsigned long get_completed_synchronize_rcu (void ); +void get_completed_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);#ifdef CONFIG_PREEMPT_RCU
include/linux/rcutiny.h +9 -0
@@ -14,10 +14,19 @@#include <asm/param.h> /* for HZ */ +struct rcu_gp_oldstate { +unsigned long rgos_norm; +}; +unsigned long get_state_synchronize_rcu (void );unsigned long start_poll_synchronize_rcu (void );bool poll_state_synchronize_rcu (unsigned long oldstate); +static inline bool poll_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ +return poll_state_synchronize_rcu (rgosp->rgos_norm); +} +static inline void cond_synchronize_rcu (unsigned long oldstate) {might_sleep ();
include/linux/rcutree.h +8 -0
@@ -40,11 +40,19 @@ bool rcu_eqs_special_set(int cpu);void rcu_momentary_dyntick_idle (void );void kfree_rcu_scheduler_running (void );bool rcu_gp_might_be_stalled (void ); + +struct rcu_gp_oldstate { +unsigned long rgos_norm; +unsigned long rgos_exp; +unsigned long rgos_polled; +}; +unsigned long start_poll_synchronize_rcu_expedited (void );void cond_synchronize_rcu_expedited (unsigned long oldstate);unsigned long get_state_synchronize_rcu (void );unsigned long start_poll_synchronize_rcu (void );bool poll_state_synchronize_rcu (unsigned long oldstate); +bool poll_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);void cond_synchronize_rcu (unsigned long oldstate);bool rcu_is_idle_cpu (int cpu);
kernel/rcu/rcutorture.c +9 -0
@@ -336,8 +336,10 @@ struct rcu_torture_ops {void (*cond_sync_exp)(unsigned long oldstate);unsigned long (*get_gp_state)(void );unsigned long (*get_gp_completed)(void ); +void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);unsigned long (*start_gp_poll)(void );bool (*poll_gp_state)(unsigned long oldstate); +bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);void (*cond_sync)(unsigned long oldstate);call_rcu_func_t call;void (*cb_barrier)(void );
@@ -503,8 +505,10 @@ static struct rcu_torture_ops rcu_ops = { .exp_sync = synchronize_rcu_expedited, .get_gp_state = get_state_synchronize_rcu, .get_gp_completed = get_completed_synchronize_rcu, + .get_gp_completed_full = get_completed_synchronize_rcu_full, .start_gp_poll = start_poll_synchronize_rcu, .poll_gp_state = poll_state_synchronize_rcu, + .poll_gp_state_full = poll_state_synchronize_rcu_full, .cond_sync = cond_synchronize_rcu, .get_gp_state_exp = get_state_synchronize_rcu, .start_gp_poll_exp = start_poll_synchronize_rcu_expedited,
@@ -1212,6 +1216,7 @@ rcu_torture_writer(void *arg)bool boot_ended;bool can_expedite = !rcu_gp_is_expedited () && !rcu_gp_is_normal ();unsigned long cookie; +struct rcu_gp_oldstate cookie_full;int expediting =0 ;unsigned long gp_snap;int i;
@@ -1277,6 +1282,10 @@ rcu_torture_writer(void *arg) } cur_ops->readunlock (idx); } +if (cur_ops->get_gp_completed_full && cur_ops->poll_gp_state_full) { + cur_ops->get_gp_completed_full (&cookie_full); +WARN_ON_ONCE (!cur_ops->poll_gp_state_full (&cookie_full)); + }switch (synctype[torture_random (&rand) % nsynctypes]) {case RTWS_DEF_FREE : rcu_torture_writer_state =RTWS_DEF_FREE ;
kernel/rcu/tiny.c +10 -0
@@ -183,6 +183,16 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) }EXPORT_SYMBOL_GPL (call_rcu); +/* +* Store a grace-period-counter "cookie". For more information, +* see the Tree RCU header comment. +*/ +void get_completed_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm =RCU_GET_STATE_COMPLETED ; +} +EXPORT_SYMBOL_GPL (get_completed_synchronize_rcu_full); +/* * Return a grace-period-counter "cookie". For more information, * see the Tree RCU header comment.
kernel/rcu/tree.c +72 -4
100 lines not on this page · raw message
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 2/25 rcu: Add full-sized polling for get_state()
In reply to Paul E. McKenneyThe get_state_synchronize_rcu() API compresses the combined expedited and normal grace-period states into a single unsigned long, which conserves storage, but can miss grace periods in certain cases involving overlapping normal and expedited grace periods. Missing the occasional grace period is usually not a problem, but there are use cases that care about each and every grace period.
This commit therefore adds the next member of the full-state RCU grace-period polling API, namely the get_state_synchronize_rcu_full() function. This uses up to three times the storage (rcu_gp_oldstate structure instead of unsigned long), but is guaranteed not to miss grace periods.
--- include/linux/rcutiny.h | 6 ++++++ include/linux/rcutree.h | 1 + kernel/rcu/rcutorture.c | 10 ++++++---- kernel/rcu/tree.c | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-)
include/linux/rcutiny.h +6 -0
@@ -19,6 +19,12 @@ struct rcu_gp_oldstate { };unsigned long get_state_synchronize_rcu (void ); + +static inline void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm =get_state_synchronize_rcu (); +} +unsigned long start_poll_synchronize_rcu (void );bool poll_state_synchronize_rcu (unsigned long oldstate);
include/linux/rcutree.h +1 -0
@@ -50,6 +50,7 @@ struct rcu_gp_oldstate {unsigned long start_poll_synchronize_rcu_expedited (void );void cond_synchronize_rcu_expedited (unsigned long oldstate);unsigned long get_state_synchronize_rcu (void ); +void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);unsigned long start_poll_synchronize_rcu (void );bool poll_state_synchronize_rcu (unsigned long oldstate);bool poll_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);
kernel/rcu/rcutorture.c +6 -4
@@ -335,6 +335,7 @@ struct rcu_torture_ops {bool (*poll_gp_state_exp)(unsigned long oldstate);void (*cond_sync_exp)(unsigned long oldstate);unsigned long (*get_gp_state)(void ); +void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);unsigned long (*get_gp_completed)(void );void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);unsigned long (*start_gp_poll)(void );
@@ -504,6 +505,7 @@ static struct rcu_torture_ops rcu_ops = { .sync = synchronize_rcu, .exp_sync = synchronize_rcu_expedited, .get_gp_state = get_state_synchronize_rcu, + .get_gp_state_full = get_state_synchronize_rcu_full, .get_gp_completed = get_completed_synchronize_rcu, .get_gp_completed_full = get_completed_synchronize_rcu_full, .start_gp_poll = start_poll_synchronize_rcu,
@@ -1293,12 +1295,12 @@ rcu_torture_writer(void *arg)break ;case RTWS_EXP_SYNC : rcu_torture_writer_state =RTWS_EXP_SYNC ; -if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - cookie = cur_ops->get_gp_state (); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) + cur_ops->get_gp_state_full (&cookie_full); cur_ops->exp_sync (); cur_ops->exp_sync (); -if (cur_ops->get_gp_state && cur_ops->poll_gp_state) -WARN_ON_ONCE (!cur_ops->poll_gp_state (cookie)); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) +WARN_ON_ONCE (!cur_ops->poll_gp_state_full (&cookie_full));rcu_torture_pipe_update (old_rp);break ;case RTWS_COND_GET :
kernel/rcu/tree.c +33 -0
@@ -1755,6 +1755,8 @@ static noinline void rcu_gp_cleanup(void)dump_blkd_tasks (rnp,10 );WARN_ON_ONCE (rnp->qsmask);WRITE_ONCE (rnp->gp_seq, new_gp_seq); +if (!rnp->parent) +smp_mb ();// Order against failing poll_state_synchronize_rcu_full(). rdp =this_cpu_ptr (&rcu_data);if (rnp == rdp->mynode) needgp =__note_gp_changes (rnp, rdp) || needgp;
@@ -3556,6 +3558,37 @@ unsigned long get_state_synchronize_rcu(void) }EXPORT_SYMBOL_GPL (get_state_synchronize_rcu); +/** +* get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited +* @rgosp: location to place combined normal/expedited grace-period state +* +* Places the normal and expedited grace-period states in @rgosp. This +* state value can be passed to a later call to cond_synchronize_rcu_full() +* or poll_state_synchronize_rcu_full() to determine whether or not a +* grace period (whether normal or expedited) has elapsed in the meantime. +* The rcu_gp_oldstate structure takes up twice the memory of an unsigned +* long, but is guaranteed to see all grace periods. In contrast, the +* combined state occupies less memory, but can sometimes fail to take +* grace periods into account. +* +* This does not guarantee that the needed grace period will actually +* start. +*/ +void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ +struct rcu_node *rnp =rcu_get_root (); + +/* +* Any prior manipulation of RCU-protected data must happen +* before the loads from ->gp_seq and ->expedited_sequence. +*/ +smp_mb ();/* ^^^ */ + rgosp->rgos_norm =rcu_seq_snap (&rnp->gp_seq); + rgosp->rgos_exp =rcu_seq_snap (&rcu_state.expedited_sequence); + rgosp->rgos_polled =rcu_seq_snap (&rcu_state.gp_seq_polled); +} +EXPORT_SYMBOL_GPL (get_state_synchronize_rcu_full); +/** * start_poll_synchronize_rcu - Snapshot and start RCU grace period *
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 3/25 rcutorture: Abstract synchronous and polled API testing
In reply to Paul E. McKenneyThis commit abstracts a do_rtws_sync() function that does synchronous grace-period testing, but also testing the polled API 25% of the time each for the normal and full-state variants of the polled API.
--- kernel/rcu/rcutorture.c | 48 ++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 12 deletions(-)
kernel/rcu/rcutorture.c +36 -12
@@ -1207,6 +1207,40 @@ static void rcu_torture_write_types(void) } } +/* +* Do the specified rcu_torture_writer() synchronous grace period, +* while also testing out the polled APIs. Note well that the single-CPU +* grace-period optimizations must be accounted for. +*/ +static void do_rtws_sync (struct torture_random_state *trsp,void (*sync)(void )) +{ +unsigned long cookie; +struct rcu_gp_oldstate cookie_full; +bool dopoll; +bool dopoll_full; +unsigned long r =torture_random (trsp); + + dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r &0x300 ); + dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r &0xc00 ); +if (dopoll || dopoll_full) +cpus_read_lock (); +if (dopoll) + cookie = cur_ops->get_gp_state (); +if (dopoll_full) + cur_ops->get_gp_state_full (&cookie_full); +if (dopoll || (!IS_ENABLED (CONFIG_TINY_RCU ) && dopoll_full &&num_online_cpus () <=1 )) +sync (); +sync (); +WARN_ONCE (dopoll && !cur_ops->poll_gp_state (cookie), +"%s: Cookie check 3 failed %pS() online %*pbl." , + __func__, sync,cpumask_pr_args (cpu_online_mask)); +WARN_ONCE (dopoll_full && !cur_ops->poll_gp_state_full (&cookie_full), +"%s: Cookie check 4 failed %pS() online %*pbl" , + __func__, sync,cpumask_pr_args (cpu_online_mask)); +if (dopoll || dopoll_full) +cpus_read_unlock (); +} +/* * RCU torture writer kthread. Repeatedly substitutes a new structure * for that pointed to by rcu_torture_current, freeing the old structure
@@ -1295,12 +1329,7 @@ rcu_torture_writer(void *arg)break ;case RTWS_EXP_SYNC : rcu_torture_writer_state =RTWS_EXP_SYNC ; -if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) - cur_ops->get_gp_state_full (&cookie_full); - cur_ops->exp_sync (); - cur_ops->exp_sync (); -if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) -WARN_ON_ONCE (!cur_ops->poll_gp_state_full (&cookie_full)); +do_rtws_sync (&rand, cur_ops->exp_sync);rcu_torture_pipe_update (old_rp);break ;case RTWS_COND_GET :
@@ -1339,12 +1368,7 @@ rcu_torture_writer(void *arg)break ;case RTWS_SYNC : rcu_torture_writer_state =RTWS_SYNC ; -if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - cookie = cur_ops->get_gp_state (); - cur_ops->sync (); - cur_ops->sync (); -if (cur_ops->get_gp_state && cur_ops->poll_gp_state) -WARN_ON_ONCE (!cur_ops->poll_gp_state (cookie)); +do_rtws_sync (&rand, cur_ops->sync);rcu_torture_pipe_update (old_rp);break ;default :
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 4/25 rcutorture: Allow per-RCU-flavor polled double-GP check
In reply to Paul E. McKenneyOnly vanilla RCU needs a double grace period for its compressed polled grace-period old-state cookie. This commit therefore adds an rcu_torture_ops per-flavor function ->poll_need_2gp to allow this check to be adapted to the RCU flavor under test. A NULL pointer for this function says that doubled grace periods are never needed.
--- kernel/rcu/rcutorture.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
kernel/rcu/rcutorture.c +8 -1
@@ -341,6 +341,7 @@ struct rcu_torture_ops {unsigned long (*start_gp_poll)(void );bool (*poll_gp_state)(unsigned long oldstate);bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); +bool (*poll_need_2gp)(bool poll,bool poll_full);void (*cond_sync)(unsigned long oldstate);call_rcu_func_t call;void (*cb_barrier)(void );
@@ -492,6 +493,11 @@ static void rcu_sync_torture_init(void)INIT_LIST_HEAD (&rcu_torture_removed); } +static bool rcu_poll_need_2gp (bool poll,bool poll_full) +{ +return poll || (!IS_ENABLED (CONFIG_TINY_RCU ) && poll_full &&num_online_cpus () <=1 ); +} +static struct rcu_torture_ops rcu_ops = { .ttype =RCU_FLAVOR , .init = rcu_sync_torture_init,
@@ -511,6 +517,7 @@ static struct rcu_torture_ops rcu_ops = { .start_gp_poll = start_poll_synchronize_rcu, .poll_gp_state = poll_state_synchronize_rcu, .poll_gp_state_full = poll_state_synchronize_rcu_full, + .poll_need_2gp = rcu_poll_need_2gp, .cond_sync = cond_synchronize_rcu, .get_gp_state_exp = get_state_synchronize_rcu, .start_gp_poll_exp = start_poll_synchronize_rcu_expedited,
@@ -1228,7 +1235,7 @@ static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) cookie = cur_ops->get_gp_state ();if (dopoll_full) cur_ops->get_gp_state_full (&cookie_full); -if (dopoll || (!IS_ENABLED (CONFIG_TINY_RCU ) && dopoll_full &&num_online_cpus () <=1 )) +if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp (dopoll, dopoll_full))sync ();sync (); WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 5/25 rcutorture: Verify RCU reader prevents full polling from completing
In reply to Paul E. McKenneyThis commit adds a test to rcu_torture_writer() that verifies that a ->get_gp_state_full() and ->poll_gp_state_full() polled grace-period sequence does not claim that a grace period elapsed within the confines of the corresponding read-side critical section.
--- kernel/rcu/rcutorture.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-)
kernel/rcu/rcutorture.c +16 -3
@@ -1309,6 +1309,8 @@ rcu_torture_writer(void *arg)atomic_inc (&rcu_torture_wcount[i]);WRITE_ONCE (old_rp->rtort_pipe_count, old_rp->rtort_pipe_count +1 ); + +// Make sure readers block polled grace periods. if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { idx = cur_ops->readlock (); cookie = cur_ops->get_gp_state ();
@@ -1325,9 +1327,20 @@ rcu_torture_writer(void *arg) } cur_ops->readunlock (idx); } -if (cur_ops->get_gp_completed_full && cur_ops->poll_gp_state_full) { - cur_ops->get_gp_completed_full (&cookie_full); -WARN_ON_ONCE (!cur_ops->poll_gp_state_full (&cookie_full)); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { + idx = cur_ops->readlock (); + cur_ops->get_gp_state_full (&cookie_full); +WARN_ONCE (cur_ops->poll_gp_state_full (&cookie_full), +"%s: Cookie check 5 failed %s(%d) online %*pbl\n" , + __func__, +rcu_torture_writer_state_getname (), + rcu_torture_writer_state, +cpumask_pr_args (cpu_online_mask)); +if (cur_ops->get_gp_completed_full) { + cur_ops->get_gp_completed_full (&cookie_full); +WARN_ON_ONCE (!cur_ops->poll_gp_state_full (&cookie_full)); + } + cur_ops->readunlock (idx); }switch (synctype[torture_random (&rand) % nsynctypes]) {case RTWS_DEF_FREE :
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 6/25 rcutorture: Remove redundant RTWS_DEF_FREE check
In reply to Paul E. McKenneyThis check does nothing because the state at this point in the code because the rcu_torture_writer_state value is guaranteed to instead be RTWS_REPLACE. This commit therefore removes this check.
--- kernel/rcu/rcutorture.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
kernel/rcu/rcutorture.c +1 -2
@@ -1314,8 +1314,7 @@ rcu_torture_writer(void *arg)if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { idx = cur_ops->readlock (); cookie = cur_ops->get_gp_state (); - WARN_ONCE(rcu_torture_writer_state != RTWS_DEF_FREE && - cur_ops->poll_gp_state(cookie), + WARN_ONCE(cur_ops->poll_gp_state(cookie), "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", __func__, rcu_torture_writer_state_getname(),
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 7/25 rcutorture: Verify long-running reader prevents full polling from completing
In reply to Paul E. McKenneyThis commit adds full-state polling checks to accompany the old-style polling checks in the rcu_torture_one_read() function. If a polling cycle within an RCU reader completes, a WARN_ONCE() is triggered.
--- kernel/rcu/rcutorture.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
kernel/rcu/rcutorture.c +10 -0
@@ -1770,6 +1770,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp,static bool rcu_torture_one_read (struct torture_random_state *trsp,long myid) {unsigned long cookie; +struct rcu_gp_oldstate cookie_full;int i;unsigned long started;unsigned long completed;
@@ -1787,6 +1788,8 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)rcutorture_one_extend (&readstate, newstate, trsp, rtrsp++);if (cur_ops->get_gp_state && cur_ops->poll_gp_state) cookie = cur_ops->get_gp_state (); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) + cur_ops->get_gp_state_full (&cookie_full); started = cur_ops->get_gp_seq (); ts =rcu_trace_clock_local (); p = rcu_dereference_check(rcu_torture_current,
@@ -1827,6 +1830,13 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)rcu_torture_writer_state_getname (), rcu_torture_writer_state, cookie, cur_ops->get_gp_state()); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) +WARN_ONCE (cur_ops->poll_gp_state_full (&cookie_full), +"%s: Cookie check 6 failed %s(%d) online %*pbl\n" , + __func__, +rcu_torture_writer_state_getname (), + rcu_torture_writer_state, +cpumask_pr_args (cpu_online_mask));rcutorture_one_extend (&readstate,0 , trsp, rtrsp);WARN_ON_ONCE (readstate);// This next splat is expected behavior if leakpointer, especially
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 8/25 rcu: Add full-sized polling for start_poll()
In reply to Paul E. McKenneyThe start_poll_synchronize_rcu() API compresses the combined expedited and normal grace-period states into a single unsigned long, which conserves storage, but can miss grace periods in certain cases involving overlapping normal and expedited grace periods. Missing the occasional grace period is usually not a problem, but there are use cases that care about each and every grace period.
This commit therefore adds the next member of the full-state RCU grace-period polling API, namely the start_poll_synchronize_rcu_full() function. This uses up to three times the storage (rcu_gp_oldstate structure instead of unsigned long), but is guaranteed not to miss grace periods.
--- include/linux/rcutiny.h | 6 +++++ include/linux/rcutree.h | 1 + kernel/rcu/rcutorture.c | 49 +++++++++++++++++++++++++++------- kernel/rcu/tree.c | 58 ++++++++++++++++++++++++++++++++--------- 4 files changed, 92 insertions(+), 22 deletions(-)
include/linux/rcutiny.h +6 -0
@@ -26,6 +26,12 @@ static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) }unsigned long start_poll_synchronize_rcu (void ); + +static inline void start_poll_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm =start_poll_synchronize_rcu (); +} +bool poll_state_synchronize_rcu (unsigned long oldstate);static inline bool poll_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp)
include/linux/rcutree.h +1 -0
@@ -52,6 +52,7 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate);unsigned long get_state_synchronize_rcu (void );void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);unsigned long start_poll_synchronize_rcu (void ); +void start_poll_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);bool poll_state_synchronize_rcu (unsigned long oldstate);bool poll_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);void cond_synchronize_rcu (unsigned long oldstate);
kernel/rcu/rcutorture.c +40 -9
121 lines not on this page · raw message
kernel/rcu/tree.c +45 -13
77 lines not on this page · raw message
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 9/25 rcu: Add full-sized polling for start_poll_expedited()
In reply to Paul E. McKenneyThe start_poll_synchronize_rcu_expedited() API compresses the combined expedited and normal grace-period states into a single unsigned long, which conserves storage, but can miss grace periods in certain cases involving overlapping normal and expedited grace periods. Missing the occasional grace period is usually not a problem, but there are use cases that care about each and every grace period.
This commit therefore adds yet another member of the full-state RCU grace-period polling API, which is the start_poll_synchronize_rcu_expedited_full() function. This uses up to three times the storage (rcu_gp_oldstate structure instead of unsigned long), but is guaranteed not to miss grace periods.
[ paulmck: Apply feedback from kernel test robot and Julia Lawall. ]
--- include/linux/rcutiny.h | 5 ++++ include/linux/rcutree.h | 1 + kernel/rcu/rcutorture.c | 51 +++++++++++++++++++++++++++++++++-------- kernel/rcu/tree_exp.h | 18 +++++++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-)
include/linux/rcutiny.h +5 -0
@@ -49,6 +49,11 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void)return start_poll_synchronize_rcu (); } +static inline void start_poll_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm =start_poll_synchronize_rcu_expedited (); +} +static inline void cond_synchronize_rcu_expedited (unsigned long oldstate) {cond_synchronize_rcu (oldstate);
include/linux/rcutree.h +1 -0
@@ -48,6 +48,7 @@ struct rcu_gp_oldstate { };unsigned long start_poll_synchronize_rcu_expedited (void ); +void start_poll_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp);void cond_synchronize_rcu_expedited (unsigned long oldstate);unsigned long get_state_synchronize_rcu (void );void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);
kernel/rcu/rcutorture.c +41 -10
111 lines not on this page · raw message
kernel/rcu/tree_exp.h +18 -0
@@ -1027,6 +1027,24 @@ unsigned long start_poll_synchronize_rcu_expedited(void) }EXPORT_SYMBOL_GPL (start_poll_synchronize_rcu_expedited); +/** +* start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period +* @rgosp: Place to put snapshot of grace-period state +* +* Places the normal and expedited grace-period states in rgosp. This +* state value can be passed to a later call to cond_synchronize_rcu_full() +* or poll_state_synchronize_rcu_full() to determine whether or not a +* grace period (whether normal or expedited) has elapsed in the meantime. +* If the needed expedited grace period is not already slated to start, +* initiates that grace period. +*/ +void start_poll_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp) +{ +get_state_synchronize_rcu_full (rgosp); + (void )start_poll_synchronize_rcu_expedited (); +} +EXPORT_SYMBOL_GPL (start_poll_synchronize_rcu_expedited_full); +/** * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period *
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 10/25 rcu: Remove blank line from poll_state_synchronize_rcu() docbook header
In reply to Paul E. McKenneyThis commit removes the blank line preceding the oldstate parameter to the docbook header for the poll_state_synchronize_rcu() function and marks uses of this parameter later in that header.
--- kernel/rcu/tree.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
kernel/rcu/tree.c +2 -3
@@ -3662,11 +3662,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full);/** * poll_state_synchronize_rcu - Has the specified RCU grace period completed? -* * @oldstate: value from get_state_synchronize_rcu() or start_poll_synchronize_rcu() * * If a full RCU grace period has elapsed since the earlier call from -* which oldstate was obtained, return @true, otherwise return @false. +* which @oldstate was obtained, return @true, otherwise return @false. * If @false is returned, it is the caller's responsibility to invoke this * function later on until it does return @true. Alternatively, the caller * can explicitly wait for a grace period, for example, by passing @oldstate
@@ -3675,7 +3674,7 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full);* Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited for * more than a billion grace periods (and way more on a 64-bit system!). -* Those needing to keep oldstate values for very long time periods +* Those needing to keep old state values for very long time periods * (many hours even on 32-bit systems) should check them occasionally and * either refresh them or set a flag indicating that the grace period has * completed. Alternatively, they can use get_completed_synchronize_rcu()
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 11/25 rcu: Add full-sized polling for cond_sync_full()
In reply to Paul E. McKenneyThe cond_synchronize_rcu() API compresses the combined expedited and normal grace-period states into a single unsigned long, which conserves storage, but can miss grace periods in certain cases involving overlapping normal and expedited grace periods. Missing the occasional grace period is usually not a problem, but there are use cases that care about each and every grace period.
This commit therefore adds yet another member of the full-state RCU grace-period polling API, which is the cond_synchronize_rcu_full() function. This uses up to three times the storage (rcu_gp_oldstate structure instead of unsigned long), but is guaranteed not to miss grace periods.
[ paulmck: Apply feedback from kernel test robot and Julia Lawall. ]
--- include/linux/rcutiny.h | 5 +++ include/linux/rcutree.h | 1 + kernel/rcu/rcutorture.c | 67 +++++++++++++++++++++++++++++------------ kernel/rcu/tree.c | 28 ++++++++++++++++- 4 files changed, 80 insertions(+), 21 deletions(-)
include/linux/rcutiny.h +5 -0
@@ -44,6 +44,11 @@ static inline void cond_synchronize_rcu(unsigned long oldstate)might_sleep (); } +static inline void cond_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ +cond_synchronize_rcu (rgosp->rgos_norm); +} +static inline unsigned long start_poll_synchronize_rcu_expedited (void ) {return start_poll_synchronize_rcu ();
include/linux/rcutree.h +1 -0
@@ -57,6 +57,7 @@ void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);bool poll_state_synchronize_rcu (unsigned long oldstate);bool poll_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);void cond_synchronize_rcu (unsigned long oldstate); +void cond_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);bool rcu_is_idle_cpu (int cpu);
kernel/rcu/rcutorture.c +47 -20
127 lines not on this page · raw message
kernel/rcu/tree.c +27 -1
@@ -3749,7 +3749,6 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu_full);/** * cond_synchronize_rcu - Conditionally wait for an RCU grace period -* * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() * * If a full RCU grace period has elapsed since the earlier call to
@@ -3773,6 +3772,33 @@ void cond_synchronize_rcu(unsigned long oldstate) }EXPORT_SYMBOL_GPL (cond_synchronize_rcu); +/** +* cond_synchronize_rcu_full - Conditionally wait for an RCU grace period +* @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() +* +* If a full RCU grace period has elapsed since the call to +* get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), +* or start_poll_synchronize_rcu_expedited_full() from which @rgosp was +* obtained, just return. Otherwise, invoke synchronize_rcu() to wait +* for a full grace period. +* +* Yes, this function does not take counter wrap into account. +* But counter wrap is harmless. If the counter wraps, we have waited for +* more than 2 billion grace periods (and way more on a 64-bit system!), +* so waiting for a couple of additional grace periods should be just fine. +* +* This function provides the same memory-ordering guarantees that +* would be provided by a synchronize_rcu() that was invoked at the call +* to the function that provided @rgosp and that returned at the end of +* this function. +*/ +void cond_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp) +{ +if (!poll_state_synchronize_rcu_full (rgosp)) +synchronize_rcu (); +} +EXPORT_SYMBOL_GPL (cond_synchronize_rcu_full); +/* * Check to see if there is any immediate RCU-related work to be done by * the current CPU, returning 1 if so and zero otherwise. The checks are
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 12/25 rcu: Add full-sized polling for cond_sync_exp_full()
In reply to Paul E. McKenneyThe cond_synchronize_rcu_expedited() API compresses the combined expedited and normal grace-period states into a single unsigned long, which conserves storage, but can miss grace periods in certain cases involving overlapping normal and expedited grace periods. Missing the occasional grace period is usually not a problem, but there are use cases that care about each and every grace period.
This commit therefore adds yet another member of the full-state RCU grace-period polling API, which is the cond_synchronize_rcu_exp_full() function. This uses up to three times the storage (rcu_gp_oldstate structure instead of unsigned long), but is guaranteed not to miss grace periods.
--- include/linux/rcutiny.h | 5 +++ include/linux/rcutree.h | 1 + kernel/rcu/rcutorture.c | 72 ++++++++++++++++++++++++++++------------- kernel/rcu/tree_exp.h | 27 ++++++++++++++++ 4 files changed, 83 insertions(+), 22 deletions(-)
include/linux/rcutiny.h +5 -0
@@ -64,6 +64,11 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate)cond_synchronize_rcu (oldstate); } +static inline void cond_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp) +{ +cond_synchronize_rcu_expedited (rgosp->rgos_norm); +} +extern void rcu_barrier (void );static inline void synchronize_rcu_expedited (void )
include/linux/rcutree.h +1 -0
@@ -50,6 +50,7 @@ struct rcu_gp_oldstate {unsigned long start_poll_synchronize_rcu_expedited (void );void start_poll_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp);void cond_synchronize_rcu_expedited (unsigned long oldstate); +void cond_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp);unsigned long get_state_synchronize_rcu (void );void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp);unsigned long start_poll_synchronize_rcu (void );
kernel/rcu/rcutorture.c +50 -22
125 lines not on this page · raw message
kernel/rcu/tree_exp.h +27 -0
@@ -1071,3 +1071,30 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate)synchronize_rcu_expedited (); }EXPORT_SYMBOL_GPL (cond_synchronize_rcu_expedited); + +/** +* cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period +* @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() +* +* If a full RCU grace period has elapsed since the call to +* get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), +* or start_poll_synchronize_rcu_expedited_full() from which @rgosp was +* obtained, just return. Otherwise, invoke synchronize_rcu_expedited() +* to wait for a full grace period. +* +* Yes, this function does not take counter wrap into account. +* But counter wrap is harmless. If the counter wraps, we have waited for +* more than 2 billion grace periods (and way more on a 64-bit system!), +* so waiting for a couple of additional grace periods should be just fine. +* +* This function provides the same memory-ordering guarantees that +* would be provided by a synchronize_rcu() that was invoked at the call +* to the function that provided @rgosp and that returned at the end of +* this function. +*/ +void cond_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp) +{ +if (!poll_state_synchronize_rcu_full (rgosp)) +synchronize_rcu_expedited (); +} +EXPORT_SYMBOL_GPL (cond_synchronize_rcu_expedited_full);
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 13/25 rcu: Disable run-time single-CPU grace-period optimization
In reply to Paul E. McKenneyThe run-time single-CPU grace-period optimization applies only to kernels built with CONFIG_SMP=y && CONFIG_PREEMPTION=y that are running on a single-CPU system. But a kernel intended for a single-CPU system should instead be built with CONFIG_SMP=n, and in any case, single-CPU systems running Linux no longer appear to be the common case. Plus this optimization results in the rcu_gp_oldstate structure being half again larger than it needs to be.
This commit therefore disables the run-time single-CPU grace-period optimization, so that this optimization applies only during the pre-scheduler portion of the boot sequence.
--- kernel/rcu/tree.c | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-)
kernel/rcu/tree.c +9 -31
@@ -3423,42 +3423,20 @@ void __init kfree_rcu_scheduler_running(void)/* * During early boot, any blocking grace-period wait automatically -* implies a grace period. Later on, this is never the case for PREEMPTION. +* implies a grace period. * -* However, because a context switch is a grace period for !PREEMPTION, any -* blocking grace-period wait automatically implies a grace period if -* there is only one CPU online at any point time during execution of -* either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to -* occasionally incorrectly indicate that there are multiple CPUs online -* when there was in fact only one the whole time, as this just adds some -* overhead: RCU still operates correctly. +* Later on, this could in theory be the case for kernels built with +* CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this +* is not a common case. Furthermore, this optimization would cause +* the rcu_gp_oldstate structure to expand by 50%, so this potential +* grace-period optimization is ignored once the scheduler is running. */ static int rcu_blocking_is_gp (void ) { -int ret; - -// Invoking preempt_model_*() too early gets a splat. -if (rcu_scheduler_active ==RCU_SCHEDULER_INACTIVE || -preempt_model_full () ||preempt_model_rt ()) -return rcu_scheduler_active ==RCU_SCHEDULER_INACTIVE ; +if (rcu_scheduler_active !=RCU_SCHEDULER_INACTIVE ) +return false;might_sleep ();/* Check for RCU read-side critical section. */ -preempt_disable (); -/* -* If the rcu_state.n_online_cpus counter is equal to one, -* there is only one CPU, and that CPU sees all prior accesses -* made by any CPU that was online at the time of its access. -* Furthermore, if this counter is equal to one, its value cannot -* change until after the preempt_enable() below. -* -* Furthermore, if rcu_state.n_online_cpus is equal to one here, -* all later CPUs (both this one and any that come online later -* on) are guaranteed to see all accesses prior to this point -* in the code, without the need for additional memory barriers. -* Those memory barriers are provided by CPU-hotplug code. -*/ - ret =READ_ONCE (rcu_state.n_online_cpus) <=1 ; -preempt_enable (); -return ret; +return true; }/**
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 14/25 rcu: Set rcu_data structures' initial ->gpwrap value to true
In reply to Paul E. McKenneyIt would be good do reduce the size of the rcu_gp_oldstate structure from three unsigned long instances to two, but this requires that the boot-time optimized grace periods update the various ->gp_seq fields. Updating these fields in the rcu_state structure and in all of the rcu_node structures is at least semi-reasonable, but updating them in all of the rcu_data structures is a bridge too far. This means that if there are too many early boot-time grace periods, the ->gp_seq field in the rcu_data structure cannot be trusted. This commit therefore sets each rcu_data structure's ->gpwrap field to provide the necessary impetus for a suitable level of distrust.
--- kernel/rcu/tree.c | 1 + 1 file changed, 1 insertion(+)
kernel/rcu/tree.c +1 -0
@@ -76,6 +76,7 @@/* Data structures. */ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { + .gpwrap = true, #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags =SEGCBLIST_RCU_CORE , #endif
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 15/25 rcu-tasks: Remove grace-period fast-path rcu-tasks helper
In reply to Paul E. McKenneyNow that the grace-period fast path can only happen during the pre-scheduler portion of early boot, this fast path can no longer block run-time RCU Tasks and RCU Tasks Trace grace periods. This commit therefore removes the conditional cond_resched_tasks_rcu_qs() invocation.
--- kernel/rcu/tree.c | 2 -- 1 file changed, 2 deletions(-)
kernel/rcu/tree.c +0 -2
@@ -3492,8 +3492,6 @@ void synchronize_rcu(void)// which allows reuse of ->gp_seq_polled_snap. rcu_poll_gp_seq_start_unlocked (&rcu_state.gp_seq_polled_snap);rcu_poll_gp_seq_end_unlocked (&rcu_state.gp_seq_polled_snap); -if (rcu_init_invoked ()) -cond_resched_tasks_rcu_qs ();return ;// Context allows vacuous grace periods. }if (rcu_gp_is_expedited ())
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 16/25 rcu: Make synchronize_rcu() fast path update ->gp_seq counters
In reply to Paul E. McKenneyThis commit causes the early boot single-CPU synchronize_rcu() fastpath to update the rcu_state and rcu_node structures' ->gp_seq and ->gp_seq_needed counters. This will allow the full-state polled grace-period APIs to detect all normal grace periods without the need to track the special combined polling-only counter, which is a step towards removing the ->rgos_polled field from the rcu_gp_oldstate, thereby reducing its size by one third.
--- kernel/rcu/tree.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-)
kernel/rcu/tree.c +26 -13
@@ -3480,24 +3480,37 @@ static int rcu_blocking_is_gp(void)*/ void synchronize_rcu (void ) { +unsigned long flags; +struct rcu_node *rnp; +RCU_LOCKDEP_WARN (lock_is_held (&rcu_bh_lock_map) ||lock_is_held (&rcu_lock_map) ||lock_is_held (&rcu_sched_lock_map),"Illegal synchronize_rcu() in RCU read-side critical section" ); -if (rcu_blocking_is_gp ()) { -// Note well that this code runs with !PREEMPT && !SMP. -// In addition, all code that advances grace periods runs at -// process level. Therefore, this normal GP overlaps with -// other normal GPs only by being fully nested within them, -// which allows reuse of ->gp_seq_polled_snap. -rcu_poll_gp_seq_start_unlocked (&rcu_state.gp_seq_polled_snap); -rcu_poll_gp_seq_end_unlocked (&rcu_state.gp_seq_polled_snap); -return ;// Context allows vacuous grace periods. +if (!rcu_blocking_is_gp ()) { +if (rcu_gp_is_expedited ()) +synchronize_rcu_expedited (); +else +wait_rcu_gp (call_rcu); +return ; } -if (rcu_gp_is_expedited ()) -synchronize_rcu_expedited (); -else -wait_rcu_gp (call_rcu); + +// Context allows vacuous grace periods. +// Note well that this code runs with !PREEMPT && !SMP. +// In addition, all code that advances grace periods runs at +// process level. Therefore, this normal GP overlaps with other +// normal GPs only by being fully nested within them, which allows +// reuse of ->gp_seq_polled_snap. +rcu_poll_gp_seq_start_unlocked (&rcu_state.gp_seq_polled_snap); +rcu_poll_gp_seq_end_unlocked (&rcu_state.gp_seq_polled_snap); + +// Update normal grace-period counters to record grace period. +local_irq_save (flags); +WARN_ON_ONCE (num_online_cpus () >1 ); + rcu_state.gp_seq += (1 <<RCU_SEQ_CTR_SHIFT ); +rcu_for_each_node_breadth_first (rnp) + rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; +local_irq_restore (flags); }EXPORT_SYMBOL_GPL (synchronize_rcu);
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 17/25 rcu: Remove expedited grace-period fast-path forward-progress helper
In reply to Paul E. McKenneyNow that the expedited grace-period fast path can only happen during the pre-scheduler portion of early boot, this fast path can no longer block run-time RCU Trace grace periods. This commit therefore removes the conditional cond_resched() invocation.
--- kernel/rcu/tree_exp.h | 2 -- 1 file changed, 2 deletions(-)
kernel/rcu/tree_exp.h +0 -2
@@ -924,8 +924,6 @@ void synchronize_rcu_expedited(void)// them, which allows reuse of ->gp_seq_polled_exp_snap. rcu_poll_gp_seq_start_unlocked (&rcu_state.gp_seq_polled_exp_snap);rcu_poll_gp_seq_end_unlocked (&rcu_state.gp_seq_polled_exp_snap); -if (rcu_init_invoked ()) -cond_resched ();return ;// Context allows vacuous grace periods. }
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 18/25 rcu: Make synchronize_rcu_expedited() fast path update .expedited_sequence
In reply to Paul E. McKenneyThis commit causes the early boot single-CPU synchronize_rcu_expedited() fastpath to update the rcu_state structure's ->expedited_sequence counter. This will allow the full-state polled grace-period APIs to detect all expedited grace periods without the need to track the special combined polling-only counter, which is another step towards removing the ->rgos_polled field from the rcu_gp_oldstate, thereby reducing its size by one third.
--- kernel/rcu/tree_exp.h | 6 ++++++ 1 file changed, 6 insertions(+)
kernel/rcu/tree_exp.h +6 -0
@@ -906,6 +906,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp)void synchronize_rcu_expedited (void ) {bool boottime = (rcu_scheduler_active ==RCU_SCHEDULER_INIT ); +unsigned long flags;struct rcu_exp_work rew;struct rcu_node *rnp;unsigned long s;
@@ -924,6 +925,11 @@ void synchronize_rcu_expedited(void)// them, which allows reuse of ->gp_seq_polled_exp_snap. rcu_poll_gp_seq_start_unlocked (&rcu_state.gp_seq_polled_exp_snap);rcu_poll_gp_seq_end_unlocked (&rcu_state.gp_seq_polled_exp_snap); + +local_irq_save (flags); +WARN_ON_ONCE (num_online_cpus () >1 ); + rcu_state.expedited_sequence += (1 <<RCU_SEQ_CTR_SHIFT ); +local_irq_restore (flags);return ;// Context allows vacuous grace periods. }
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 19/25 rcu: Remove ->rgos_polled field from rcu_gp_oldstate structure
In reply to Paul E. McKenneyBecause both normal and expedited grace periods increment their respective counters on their pre-scheduler early boot fastpaths, the rcu_gp_oldstate structure no longer needs its ->rgos_polled field. This commit therefore removes this field, shrinking this structure so that it is the same size as an rcu_head structure.
--- include/linux/rcutree.h | 1 - kernel/rcu/tree.c | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-)
include/linux/rcutree.h +0 -1
@@ -44,7 +44,6 @@ bool rcu_gp_might_be_stalled(void);struct rcu_gp_oldstate {unsigned long rgos_norm;unsigned long rgos_exp; -unsigned long rgos_polled; };unsigned long start_poll_synchronize_rcu_expedited (void );
kernel/rcu/tree.c +1 -5
@@ -3526,7 +3526,6 @@ void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) { rgosp->rgos_norm =RCU_GET_STATE_COMPLETED ; rgosp->rgos_exp =RCU_GET_STATE_COMPLETED ; - rgosp->rgos_polled =RCU_GET_STATE_COMPLETED ; }EXPORT_SYMBOL_GPL (get_completed_synchronize_rcu_full);
@@ -3575,7 +3574,6 @@ void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)smp_mb ();/* ^^^ */ rgosp->rgos_norm =rcu_seq_snap (&rnp->gp_seq); rgosp->rgos_exp =rcu_seq_snap (&rcu_state.expedited_sequence); - rgosp->rgos_polled =rcu_seq_snap (&rcu_state.gp_seq_polled); }EXPORT_SYMBOL_GPL (get_state_synchronize_rcu_full);
@@ -3727,9 +3725,7 @@ bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)if (rgosp->rgos_norm ==RCU_GET_STATE_COMPLETED ||rcu_seq_done_exact (&rnp->gp_seq, rgosp->rgos_norm) || rgosp->rgos_exp ==RCU_GET_STATE_COMPLETED || -rcu_seq_done_exact (&rcu_state.expedited_sequence, rgosp->rgos_exp) || - rgosp->rgos_polled ==RCU_GET_STATE_COMPLETED || -rcu_seq_done_exact (&rcu_state.gp_seq_polled, rgosp->rgos_polled)) { +rcu_seq_done_exact (&rcu_state.expedited_sequence, rgosp->rgos_exp)) {smp_mb ();/* Ensure GP ends before subsequent accesses. */ return true; }
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 20/25 rcutorture: Adjust rcu_poll_need_2gp() for rcu_gp_oldstate field removal
In reply to Paul E. McKenneyNow that rcu_gp_oldstate can accurately track both normal and expedited grace periods regardless of system state, rcutorture's rcu_poll_need_2gp() function need only call for a second grace period for the old single-unsigned-long grace-period polling APIs This commit therefore adjusts rcu_poll_need_2gp() accordingly.
--- kernel/rcu/rcutorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/rcutorture.c +1 -1
@@ -520,7 +520,7 @@ static void rcu_sync_torture_init(void)static bool rcu_poll_need_2gp (bool poll,bool poll_full) { -return poll || (!IS_ENABLED (CONFIG_TINY_RCU ) && poll_full &&num_online_cpus () <=1 ); +return poll; }static struct rcu_torture_ops rcu_ops = {
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 21/25 rcu: Make synchronize_rcu() fastpath update only boot-CPU counters
In reply to Paul E. McKenneyLarge systems can have hundreds of rcu_node structures, and updating counters in each of them might slow down booting. This commit therefore updates only the counters in those rcu_node structures corresponding to the boot CPU, up to and including the root rcu_node structure.
The counters for the remaining rcu_node structures are updated by the rcu_scheduler_starting() function, which executes just before the first non-boot kthread is spawned.
--- kernel/rcu/tree.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)
kernel/rcu/tree.c +16 -2
@@ -3504,11 +3504,14 @@ void synchronize_rcu(void)rcu_poll_gp_seq_start_unlocked (&rcu_state.gp_seq_polled_snap);rcu_poll_gp_seq_end_unlocked (&rcu_state.gp_seq_polled_snap); -// Update normal grace-period counters to record grace period. +// Update the normal grace-period counters to record +// this grace period, but only those used by the boot CPU. +// The rcu_scheduler_starting() will take care of the rest of +// these counters. local_irq_save (flags);WARN_ON_ONCE (num_online_cpus () >1 ); rcu_state.gp_seq += (1 <<RCU_SEQ_CTR_SHIFT ); -rcu_for_each_node_breadth_first (rnp) +for (rnp =this_cpu_ptr (&rcu_data)->mynode; rnp; rnp = rnp->parent) rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;local_irq_restore (flags); }
@@ -4456,9 +4459,20 @@ early_initcall(rcu_spawn_gp_kthread);*/ void rcu_scheduler_starting (void ) { +unsigned long flags; +struct rcu_node *rnp; +WARN_ON (num_online_cpus () !=1 );WARN_ON (nr_context_switches () >0 );rcu_test_sync_prims (); + +// Fix up the ->gp_seq counters. +local_irq_save (flags); +rcu_for_each_node_breadth_first (rnp) + rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; +local_irq_restore (flags); + +// Switch out of early boot mode. rcu_scheduler_active =RCU_SCHEDULER_INIT ;rcu_test_sync_prims (); }
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 22/25 rcutorture: Use 1-suffixed variable in rcu_torture_write_types() check
In reply to Paul E. McKenneyThis commit changes the use of gp_poll_exp to gp_poll_exp1 in the first check in rcu_torture_write_types(). No functional effect, but consistency is a good thing.
--- kernel/rcu/rcutorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/rcu/rcutorture.c +1 -1
@@ -1196,7 +1196,7 @@ static void rcu_torture_write_types(void)/* Initialize synctype[] array. If none set, take default. */ if (!gp_cond1 && !gp_cond_exp1 && !gp_cond_full1 && !gp_cond_exp_full1 && !gp_exp1 && - !gp_poll_exp && !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 && + !gp_poll_exp1 && !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 && !gp_sync1) gp_cond1 = gp_cond_exp1 = gp_cond_full1 = gp_cond_exp_full1 = gp_exp1 = gp_poll_exp1 = gp_poll_exp_full1 = gp_normal1 = gp_poll1 =
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 23/25 rcutorture: Expand rcu_torture_write_types() first "if" statement
In reply to Paul E. McKenneyThis commit expands the rcu_torture_write_types() function's first "if" condition and body, placing one element per line, in order to make the compiler's error messages more helpful.
--- kernel/rcu/rcutorture.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-)
kernel/rcu/rcutorture.c +23 -6
@@ -1195,12 +1195,29 @@ static void rcu_torture_write_types(void)bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;/* Initialize synctype[] array. If none set, take default. */ -if (!gp_cond1 && !gp_cond_exp1 && !gp_cond_full1 && !gp_cond_exp_full1 && !gp_exp1 && - !gp_poll_exp1 && !gp_poll_exp_full1 && !gp_normal1 && !gp_poll1 && !gp_poll_full1 && - !gp_sync1) - gp_cond1 = gp_cond_exp1 = gp_cond_full1 = gp_cond_exp_full1 = gp_exp1 = - gp_poll_exp1 = gp_poll_exp_full1 = gp_normal1 = gp_poll1 = - gp_poll_full1 = gp_sync1 = true; +if (!gp_cond1 && + !gp_cond_exp1 && + !gp_cond_full1 && + !gp_cond_exp_full1 && + !gp_exp1 && + !gp_poll_exp1 && + !gp_poll_exp_full1 && + !gp_normal1 && + !gp_poll1 && + !gp_poll_full1 && + !gp_sync1) { + gp_cond1 = true; + gp_cond_exp1 = true; + gp_cond_full1 = true; + gp_cond_exp_full1 = true; + gp_exp1 = true; + gp_poll_exp1 = true; + gp_poll_exp_full1 = true; + gp_normal1 = true; + gp_poll1 = true; + gp_poll_full1 = true; + gp_sync1 = true; + }if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { synctype[nsynctypes++] =RTWS_COND_GET ;pr_info ("%s: Testing conditional GPs.\n" , __func__);
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 24/25 rcu: Add functions to compare grace-period state values
In reply to Paul E. McKenneyThis commit adds same_state_synchronize_rcu() and same_state_synchronize_rcu_full() functions to compare grace-period state values, for example, those obtained from get_state_synchronize_rcu() and get_state_synchronize_rcu_full(). These functions allow small structures to omit these state values by placing them in list headers for lists containing structures with the same token value. Presumably the per-structure list pointers are the same ones used to link the structures into whatever reader-accessible data structure was used.
This commit also adds both NUM_ACTIVE_RCU_POLL_OLDSTATE and NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE, which define the maximum number of distinct unsigned long values and rcu_gp_oldstate values, respectively, corresponding to not-yet-completed grace periods. These values can be used to size arrays of the list headers described above.
--- include/linux/rcupdate.h | 21 +++++++++++++++++++++ include/linux/rcutiny.h | 14 ++++++++++++++ include/linux/rcutree.h | 28 ++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+)
include/linux/rcupdate.h +21 -0
@@ -47,6 +47,27 @@ struct rcu_gp_oldstate;unsigned long get_completed_synchronize_rcu (void );void get_completed_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp); +// Maximum number of unsigned long values corresponding to +// not-yet-completed RCU grace periods. +#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2 + +/** +* same_state_synchronize_rcu - Are two old-state values identical? +* @oldstate1: First old-state value. +* @oldstate2: Second old-state value. +* +* The two old-state values must have been obtained from either +* get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or +* get_completed_synchronize_rcu(). Returns @true if the two values are +* identical and @false otherwise. This allows structures whose lifetimes +* are tracked by old-state values to push these values to a list header, +* allowing those structures to be slightly smaller. +*/ +static inline bool same_state_synchronize_rcu (unsigned long oldstate1,unsigned long oldstate2) +{ +return oldstate1 == oldstate2; +} +#ifdef CONFIG_PREEMPT_RCU void __rcu_read_lock (void );
include/linux/rcutiny.h +14 -0
@@ -18,6 +18,20 @@ struct rcu_gp_oldstate {unsigned long rgos_norm; }; +// Maximum number of rcu_gp_oldstate values corresponding to +// not-yet-completed RCU grace periods. +#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 + +/* +* Are the two oldstate values the same? See the Tree RCU version for +* docbook header. +*/ +static inline bool same_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp1, +struct rcu_gp_oldstate *rgosp2) +{ +return rgosp1->rgos_norm == rgosp2->rgos_norm; +} +unsigned long get_state_synchronize_rcu (void );static inline void get_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp)
include/linux/rcutree.h +28 -0
@@ -46,6 +46,34 @@ struct rcu_gp_oldstate {unsigned long rgos_exp; }; +// Maximum number of rcu_gp_oldstate values corresponding to +// not-yet-completed RCU grace periods. +#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 + +/** +* same_state_synchronize_rcu_full - Are two old-state values identical? +* @rgosp1: First old-state value. +* @rgosp2: Second old-state value. +* +* The two old-state values must have been obtained from either +* get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), +* or get_completed_synchronize_rcu_full(). Returns @true if the two +* values are identical and @false otherwise. This allows structures +* whose lifetimes are tracked by old-state values to push these values +* to a list header, allowing those structures to be slightly smaller. +* +* Note that equality is judged on a bitwise basis, so that an +* @rcu_gp_oldstate structure with an already-completed state in one field +* will compare not-equal to a structure with an already-completed state +* in the other field. After all, the @rcu_gp_oldstate structure is opaque +* so how did such a situation come to pass in the first place? +*/ +static inline bool same_state_synchronize_rcu_full (struct rcu_gp_oldstate *rgosp1, +struct rcu_gp_oldstate *rgosp2) +{ +return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; +} +unsigned long start_poll_synchronize_rcu_expedited (void );void start_poll_synchronize_rcu_expedited_full (struct rcu_gp_oldstate *rgosp);void cond_synchronize_rcu_expedited (unsigned long oldstate);
Signature · 1 line
2.31.1.189.g2e36527f23
Paul E. McKenney
maintainerPATCH 25/25 rcutorture: Limit read-side polling-API testing
In reply to Paul E. McKenneyRCU's polled grace-period API is reasonably lightweight, but still contains heavyweight memory barriers. This commit therefore limits testing of this API from rcutorture's readers in order to avoid the false negatives that these heavyweight operations could provoke.
--- kernel/rcu/rcutorture.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-)
kernel/rcu/rcutorture.c +23 -18
@@ -1903,6 +1903,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp,*/ static bool rcu_torture_one_read (struct torture_random_state *trsp,long myid) { +bool checkpolling = !(torture_random (trsp) &0xfff );unsigned long cookie;struct rcu_gp_oldstate cookie_full;int i;
@@ -1920,10 +1921,12 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)WARN_ON_ONCE (!rcu_is_watching ()); newstate =rcutorture_extend_mask (readstate, trsp);rcutorture_one_extend (&readstate, newstate, trsp, rtrsp++); -if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - cookie = cur_ops->get_gp_state (); -if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) - cur_ops->get_gp_state_full (&cookie_full); +if (checkpolling) { +if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + cookie = cur_ops->get_gp_state (); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) + cur_ops->get_gp_state_full (&cookie_full); + } started = cur_ops->get_gp_seq (); ts =rcu_trace_clock_local (); p = rcu_dereference_check(rcu_torture_current,
@@ -1957,20 +1960,22 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) }__this_cpu_inc (rcu_torture_batch[completed]);preempt_enable (); -if (cur_ops->get_gp_state && cur_ops->poll_gp_state) -WARN_ONCE (cur_ops->poll_gp_state (cookie), -"%s: Cookie check 2 failed %s(%d) %lu->%lu\n" , - __func__, -rcu_torture_writer_state_getname (), - rcu_torture_writer_state, - cookie, cur_ops->get_gp_state ()); -if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) -WARN_ONCE (cur_ops->poll_gp_state_full (&cookie_full), -"%s: Cookie check 6 failed %s(%d) online %*pbl\n" , - __func__, -rcu_torture_writer_state_getname (), - rcu_torture_writer_state, -cpumask_pr_args (cpu_online_mask)); +if (checkpolling) { +if (cur_ops->get_gp_state && cur_ops->poll_gp_state) +WARN_ONCE (cur_ops->poll_gp_state (cookie), +"%s: Cookie check 2 failed %s(%d) %lu->%lu\n" , + __func__, +rcu_torture_writer_state_getname (), + rcu_torture_writer_state, + cookie, cur_ops->get_gp_state ()); +if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) +WARN_ONCE (cur_ops->poll_gp_state_full (&cookie_full), +"%s: Cookie check 6 failed %s(%d) online %*pbl\n" , + __func__, +rcu_torture_writer_state_getname (), + rcu_torture_writer_state, +cpumask_pr_args (cpu_online_mask)); + }rcutorture_one_extend (&readstate,0 , trsp, rtrsp);WARN_ON_ONCE (readstate);// This next splat is expected behavior if leakpointer, especially
Signature · 1 line
2.31.1.189.g2e36527f23