Torture-test updates for v4.21/v5.0
Subsystems (3)
cover Torture-test updates for v4.21/v5.0
Hello!
This series contains torture-test updates:
1. Add call_rcu() flooding forward-progress tests. If people
are going to be seeing forward-progress issues with RCU, then rcutorture needs to up its game.
2. Bring any extra CPUs online during kernel startup.
3. Remove cbflood facility due to its being obsoleted by flooding
forward-progress tests.
4. Break up too-long rcu_torture_fwd_prog() function.
5. Affinity forward-progress test to avoid housekeeping CPUs.
6. Remove unnecessary "ret" variables, courtesy of Pierce Griffiths.
7. Prepare for asynchronous access to rcu_fwd_startat.
8. Dump grace-period diagnostics upon forward-progress OOM.
9. Account for nocb-CPU callback counts in RCU CPU stall warnings.
10. Print per-CPU callback counts for forward-progress failures.
11. Print GP age upon forward-progress failure.
12. Print histogram of CB invocation at OOM time.
13. Print time since GP end upon forward-progress failure.
14. Print forward-progress test age upon failure.
15. Recover from OOM during forward-progress tests.
16. Use 100ms buckets for forward-progress callback histograms.
17. Don't do forward-progress testing of known-bad "RCU" variants.
Thanx, Paul
------------------------------------------------------------------------
Documentation/admin-guide/kernel-parameters.txt | 18 kernel/rcu/rcu.h | 4 kernel/rcu/rcutorture.c | 603 ++++++++++++++---------- kernel/rcu/tree.c | 64 ++ kernel/rcu/tree.h | 3 kernel/rcu/tree_plugin.h | 35 + kernel/torture.c | 34 - 7 files changed, 484 insertions(+), 277 deletions(-)
PATCH 1/17 rcutorture: Add call_rcu() flooding forward-progress tests
From: "Paul E. McKenney" <[email protected]>
This commit adds a call_rcu() flooding loop to the forward-progress test. This emulates tight userspace loops that force call_rcu() invocations, for example, the infamous loop containing close(open()) that instigated the addition of blimit. If RCU does not make sufficient forward progress in invoking the resulting flood of callbacks, rcutorture emits a warning.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 129 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-)
kernel/rcu/rcutorture.c +127 -2
@@ -259,6 +259,8 @@ static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */static wait_queue_head_t *barrier_cbs_wq;/* Coordinate barrier testing. */ static DECLARE_WAIT_QUEUE_HEAD (barrier_wq); +static bool rcu_fwd_cb_nodelay;/* Short rcu_torture_delay() delays. */ +/* * Allocate an element from the rcu_tortures pool. */
@@ -348,7 +350,8 @@ rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)* period, and we want a long delay occasionally to trigger * force_quiescent_state. */ -if (!(torture_random (rrsp) % (nrealreaders *2000 * longdelay_ms))) { +if (!rcu_fwd_cb_nodelay && + !(torture_random (rrsp) % (nrealreaders *2000 * longdelay_ms))) { started = cur_ops->get_gp_seq (); ts =rcu_trace_clock_local ();if (preempt_count () & (SOFTIRQ_MASK |HARDIRQ_MASK ))
@@ -1674,6 +1677,43 @@ static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) cur_ops->call (&fcsp->rh, rcu_torture_fwd_prog_cb); } +/* State for continuous-flood RCU callbacks. */ +struct rcu_fwd_cb { +struct rcu_head rh; +struct rcu_fwd_cb *rfc_next; +int rfc_gps; +}; +static DEFINE_SPINLOCK (rcu_fwd_lock); +static struct rcu_fwd_cb *rcu_fwd_cb_head; +static struct rcu_fwd_cb **rcu_fwd_cb_tail = &rcu_fwd_cb_head; +static long n_launders_cb; +static unsigned long rcu_fwd_startat; +#define MAX_FWD_CB_JIFFIES (8 * HZ)/* Maximum CB test duration. */ +#define MIN_FWD_CB_LAUNDERS 3/* This many CB invocations to count. */ +#define MIN_FWD_CBS_LAUNDERED 100/* Number of counted CBs. */ +static long n_launders_hist[2 *MAX_FWD_CB_JIFFIES /HZ ]; + +/* Callback function for continuous-flood RCU callbacks. */ +static void rcu_torture_fwd_cb_cr (struct rcu_head *rhp) +{ +int i; + struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); +struct rcu_fwd_cb **rfcpp; + + rfcp->rfc_next =NULL ; + rfcp->rfc_gps++; +spin_lock (&rcu_fwd_lock); + rfcpp = rcu_fwd_cb_tail; + rcu_fwd_cb_tail = &rfcp->rfc_next; +WRITE_ONCE (*rfcpp, rfcp); +WRITE_ONCE (n_launders_cb, n_launders_cb +1 ); + i = ((jiffies - rcu_fwd_startat) /HZ ); +if (i >=ARRAY_SIZE (n_launders_hist)) + i =ARRAY_SIZE (n_launders_hist) -1 ; + n_launders_hist[i]++; +spin_unlock (&rcu_fwd_lock); +} +/* Carry out grace-period forward-progress testing. */ static int rcu_torture_fwd_prog (void *args) {
@@ -1681,11 +1721,21 @@ static int rcu_torture_fwd_prog(void *args)unsigned long dur;struct fwd_cb_state fcs;unsigned long gps; +int i;int idx; +int j; +long n_launders; +long n_launders_cb_snap; +long n_launders_sa; +long n_max_cbs; +long n_max_gps; +struct rcu_fwd_cb *rfcp; +struct rcu_fwd_cb *rfcpn;int sd;int sd4;bool selfpropcb = false;unsigned long stopat; +unsigned long stoppedat;int tested =0 ;int tested_tries =0 ;static DEFINE_TORTURE_RANDOM (trs);
@@ -1699,6 +1749,8 @@ static int rcu_torture_fwd_prog(void *args) } do { schedule_timeout_interruptible(fwd_progress_holdoff * HZ); + + /* Tight loop containing cond_resched(). */ if (selfpropcb) { WRITE_ONCE(fcs.stop, 0); cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
@@ -1708,7 +1760,8 @@ static int rcu_torture_fwd_prog(void *args) sd = cur_ops->stall_dur () +1 ; sd4 = (sd + fwd_progress_div -1 ) / fwd_progress_div; dur = sd4 +torture_random (&trs) % (sd - sd4); - stopat = jiffies + dur; + rcu_fwd_startat = jiffies; + stopat = rcu_fwd_startat + dur;while (time_before (jiffies, stopat) && !torture_must_stop ()) { idx = cur_ops->readlock ();udelay (10 );
@@ -1729,6 +1782,78 @@ static int rcu_torture_fwd_prog(void *args) cur_ops->sync ();/* Wait for running CB to complete. */ cur_ops->cb_barrier ();/* Wait for queued callbacks. */ } + +/* Loop continuously posting RCU callbacks. */ +WRITE_ONCE (rcu_fwd_cb_nodelay, true); + cur_ops->sync ();/* Later readers see above write. */ + rcu_fwd_startat = jiffies; + stopat = rcu_fwd_startat +MAX_FWD_CB_JIFFIES ; + n_launders =0 ; + n_launders_cb =0 ; + n_launders_sa =0 ; + n_max_cbs =0 ; + n_max_gps =0 ; +for (i =0 ; i <ARRAY_SIZE (n_launders_hist); i++) + n_launders_hist[i] =0 ; + cver =READ_ONCE (rcu_torture_current_version); + gps = cur_ops->get_gp_seq (); +while (time_before (jiffies, stopat) && !torture_must_stop ()) { + rfcp =READ_ONCE (rcu_fwd_cb_head); + rfcpn =NULL ; +if (rfcp) + rfcpn =READ_ONCE (rfcp->rfc_next); +if (rfcpn) { +if (rfcp->rfc_gps >=MIN_FWD_CB_LAUNDERS && + ++n_max_gps >=MIN_FWD_CBS_LAUNDERED ) +break ; + rcu_fwd_cb_head = rfcpn; + n_launders++; + n_launders_sa++; + }else { + rfcp =kmalloc (sizeof (*rfcp),GFP_KERNEL ); +if (WARN_ON_ONCE (!rfcp)) { +schedule_timeout_interruptible (1 ); +continue ; + } + n_max_cbs++; + n_launders_sa =0 ; + rfcp->rfc_gps =0 ; + } + cur_ops->call (&rfcp->rh, rcu_torture_fwd_cb_cr); +cond_resched (); + } + stoppedat = jiffies; + n_launders_cb_snap =READ_ONCE (n_launders_cb); + cver =READ_ONCE (rcu_torture_current_version) - cver; + gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps); + cur_ops->cb_barrier ();/* Wait for callbacks to be invoked. */ +for (;;) { + rfcp = rcu_fwd_cb_head; +if (!rfcp) +break ; + rcu_fwd_cb_head = rfcp->rfc_next; +kfree (rfcp); + } + rcu_fwd_cb_tail = &rcu_fwd_cb_head; +WRITE_ONCE (rcu_fwd_cb_nodelay, false); +if (!torture_must_stop ()) { +WARN_ON (n_max_gps <MIN_FWD_CBS_LAUNDERED ); +pr_alert ("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n" , + __func__, + stoppedat - rcu_fwd_startat, + jiffies - stoppedat, + n_launders + n_max_cbs - n_launders_cb_snap, + n_launders, n_launders_sa, + n_max_gps, n_max_cbs, cver, gps); +for (i =ARRAY_SIZE (n_launders_hist) -1 ; i >0 ; i--) +if (n_launders_hist[i] >0 ) +break ; +pr_alert ("Callback-invocation histogram:" ); +for (j =0 ; j <= i; j++) +pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]); +pr_cont ("\n" ); + } +/* Avoid slow periods, better to test when busy. */ stutter_wait ("rcu_torture_fwd_prog" ); }while (!torture_must_stop ());
Signature · 1 line
2.17.1
PATCH 2/17 torture: Bring any extra CPUs online during kernel startup
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
Currently, the torture scripts rely on the initrd/init script to bring any extra CPUs online, for example, in the case where the kernel and qemu have different ideas about how many CPUs are present. This works, but is an unnecessary dependency on initrd, which needs to vary depending on the distro. This commit therefore causes torture_onoff() to check for additional CPUs, attempting to bring any found online. Errors are ignored, just as they are by the initrd/init script.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/torture.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
kernel/torture.c +12 -0
@@ -194,11 +194,23 @@ torture_onoff(void *arg)int cpu;int maxcpu =-1 ;DEFINE_TORTURE_RANDOM (rand); +int ret;VERBOSE_TOROUT_STRING ("torture_onoff task started" );for_each_online_cpu (cpu) maxcpu = cpu;WARN_ON (maxcpu <0 ); +if (!IS_MODULE (CONFIG_TORTURE_TEST )) +for_each_possible_cpu (cpu) { +if (cpu_online (cpu)) +continue ; + ret =cpu_up (cpu); +if (ret && verbose) { +pr_alert ("%s" TORTURE_FLAG +"%s: Initial online %d: errno %d\n" , + __func__, torture_type, cpu, ret); + } + }if (maxcpu ==0 ) {VERBOSE_TOROUT_STRING ("Only one CPU, so CPU-hotplug testing is disabled" );
Signature · 1 line
2.17.1
PATCH 3/17 rcutorture: Remove cbflood facility
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
Now that the forward-progress code does a full-bore continuous callback flood lasting multiple seconds, there is little point in also posting a mere 60,000 callbacks every second or so. This commit therefore removes the old cbflood testing. Over time, it may be desirable to concurrently do full-bore continuous callback floods on all CPUs simultaneously, but one dragon at a time.
Signed-off-by Paul E. McKenney <[email protected]>
--- .../admin-guide/kernel-parameters.txt | 18 ---- kernel/rcu/rcutorture.c | 86 +------------------ 2 files changed, 1 insertion(+), 103 deletions(-)
Documentation/admin-guide/kernel-parameters.txt +0 -18
@@ -3743,24 +3743,6 @@ in microseconds. The default of zero says no holdoff. - rcutorture.cbflood_inter_holdoff= [KNL] - Set holdoff time (jiffies) between successive - callback-flood tests. - - rcutorture.cbflood_intra_holdoff= [KNL] - Set holdoff time (jiffies) between successive - bursts of callbacks within a given callback-flood - test. - - rcutorture.cbflood_n_burst= [KNL] - Set the number of bursts making up a given - callback-flood test. Set this to zero to - disable callback-flood testing. - - rcutorture.cbflood_n_per_burst= [KNL] - Set the number of callbacks to be registered - in a given burst of a callback-flood test. - rcutorture.fqs_duration= [KNL] Set duration of force_quiescent_state bursts in microseconds.
kernel/rcu/rcutorture.c +1 -85
135 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 4/17 rcutorture: Break up too-long rcu_torture_fwd_prog() function
In reply to Paul E. McKenneyFrom: "Paul E. McKenney" <[email protected]>
This commit splits rcu_torture_fwd_prog_nr() and rcu_torture_fwd_prog_cr() functions out of rcu_torture_fwd_prog() in order to reduce indentation pain and because rcu_torture_fwd_prog() was getting a bit too long. In addition, this will enable easier conditional execution of the rcu_torture_fwd_prog_cr() function, which can give false-positive failures in some NO_HZ_FULL configurations due to overloading the housekeeping CPUs.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 254 +++++++++++++++++++++------------------- 1 file changed, 135 insertions(+), 119 deletions(-)
kernel/rcu/rcutorture.c +135 -119
@@ -1650,15 +1650,70 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)spin_unlock (&rcu_fwd_lock); } -/* Carry out grace-period forward-progress testing. */ -static int rcu_torture_fwd_prog (void *args) +/* Carry out need_resched()/cond_resched() forward-progress testing. */ +static void rcu_torture_fwd_prog_nr (int *tested,int *tested_tries) {unsigned long cver;unsigned long dur;struct fwd_cb_state fcs;unsigned long gps; -int i;int idx; +int sd; +int sd4; +bool selfpropcb = false; +unsigned long stopat; +static DEFINE_TORTURE_RANDOM (trs); + +if (cur_ops->call && cur_ops->sync && cur_ops->cb_barrier) { +init_rcu_head_on_stack (&fcs.rh); + selfpropcb = true; + } + +/* Tight loop containing cond_resched(). */ +if (selfpropcb) { +WRITE_ONCE (fcs.stop,0 ); + cur_ops->call (&fcs.rh, rcu_torture_fwd_prog_cb); + } + cver =READ_ONCE (rcu_torture_current_version); + gps = cur_ops->get_gp_seq (); + sd = cur_ops->stall_dur () +1 ; + sd4 = (sd + fwd_progress_div -1 ) / fwd_progress_div; + dur = sd4 +torture_random (&trs) % (sd - sd4); + rcu_fwd_startat = jiffies; + stopat = rcu_fwd_startat + dur; +while (time_before (jiffies, stopat) && !torture_must_stop ()) { + idx = cur_ops->readlock (); +udelay (10 ); + cur_ops->readunlock (idx); +if (!fwd_progress_need_resched ||need_resched ()) +cond_resched (); + } + (*tested_tries)++; +if (!time_before (jiffies, stopat) && !torture_must_stop ()) { + (*tested)++; + cver =READ_ONCE (rcu_torture_current_version) - cver; + gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps); +WARN_ON (!cver && gps <2 ); +pr_alert ("%s: Duration %ld cver %ld gps %ld\n" , __func__, dur, cver, gps); + } +if (selfpropcb) { +WRITE_ONCE (fcs.stop,1 ); + cur_ops->sync ();/* Wait for running CB to complete. */ + cur_ops->cb_barrier ();/* Wait for queued callbacks. */ + } + +if (selfpropcb) { +WARN_ON (READ_ONCE (fcs.stop) !=2 ); +destroy_rcu_head_on_stack (&fcs.rh); + } +} + +/* Carry out call_rcu() forward-progress testing. */ +static void rcu_torture_fwd_prog_cr (void ) +{ +unsigned long cver; +unsigned long gps; +int i;int j;long n_launders;long n_launders_cb_snap;
@@ -1667,136 +1722,97 @@ static int rcu_torture_fwd_prog(void *args)long n_max_gps;struct rcu_fwd_cb *rfcp;struct rcu_fwd_cb *rfcpn; -int sd; -int sd4; -bool selfpropcb = false;unsigned long stopat;unsigned long stoppedat; + +/* Loop continuously posting RCU callbacks. */ +WRITE_ONCE (rcu_fwd_cb_nodelay, true); + cur_ops->sync ();/* Later readers see above write. */ + rcu_fwd_startat = jiffies; + stopat = rcu_fwd_startat +MAX_FWD_CB_JIFFIES ; + n_launders =0 ; + n_launders_cb =0 ; + n_launders_sa =0 ; + n_max_cbs =0 ; + n_max_gps =0 ; +for (i =0 ; i <ARRAY_SIZE (n_launders_hist); i++) + n_launders_hist[i] =0 ; + cver =READ_ONCE (rcu_torture_current_version); + gps = cur_ops->get_gp_seq (); +while (time_before (jiffies, stopat) && !torture_must_stop ()) { + rfcp =READ_ONCE (rcu_fwd_cb_head); + rfcpn =NULL ; +if (rfcp) + rfcpn =READ_ONCE (rfcp->rfc_next); +if (rfcpn) { +if (rfcp->rfc_gps >=MIN_FWD_CB_LAUNDERS && + ++n_max_gps >=MIN_FWD_CBS_LAUNDERED ) +break ; + rcu_fwd_cb_head = rfcpn; + n_launders++; + n_launders_sa++; + }else { + rfcp =kmalloc (sizeof (*rfcp),GFP_KERNEL ); +if (WARN_ON_ONCE (!rfcp)) { +schedule_timeout_interruptible (1 ); +continue ; + } + n_max_cbs++; + n_launders_sa =0 ; + rfcp->rfc_gps =0 ; + } + cur_ops->call (&rfcp->rh, rcu_torture_fwd_cb_cr); +cond_resched (); + } + stoppedat = jiffies; + n_launders_cb_snap =READ_ONCE (n_launders_cb); + cver =READ_ONCE (rcu_torture_current_version) - cver; + gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps); + cur_ops->cb_barrier ();/* Wait for callbacks to be invoked. */ +for (;;) { + rfcp = rcu_fwd_cb_head; +if (!rfcp) +break ; + rcu_fwd_cb_head = rfcp->rfc_next; +kfree (rfcp); + } + rcu_fwd_cb_tail = &rcu_fwd_cb_head; +WRITE_ONCE (rcu_fwd_cb_nodelay, false); +if (!torture_must_stop ()) { +WARN_ON (n_max_gps <MIN_FWD_CBS_LAUNDERED ); +pr_alert ("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n" , + __func__, + stoppedat - rcu_fwd_startat, jiffies - stoppedat, + n_launders + n_max_cbs - n_launders_cb_snap, + n_launders, n_launders_sa, + n_max_gps, n_max_cbs, cver, gps); +for (i =ARRAY_SIZE (n_launders_hist) -1 ; i >0 ; i--) +if (n_launders_hist[i] >0 ) +break ; +pr_alert ("Callback-invocation histogram:" ); +for (j =0 ; j <= i; j++) +pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]); +pr_cont ("\n" ); + } +} + +/* Carry out grace-period forward-progress testing. */ +static int rcu_torture_fwd_prog (void *args) +{int tested =0 ;int tested_tries =0 ; -static DEFINE_TORTURE_RANDOM (trs);VERBOSE_TOROUT_STRING ("rcu_torture_fwd_progress task started" );if (!IS_ENABLED (CONFIG_SMP ) || !IS_ENABLED (CONFIG_RCU_BOOST ))set_user_nice (current,MAX_NICE ); -if (cur_ops->call && cur_ops->sync && cur_ops->cb_barrier) { -init_rcu_head_on_stack (&fcs.rh); - selfpropcb = true; - }do {schedule_timeout_interruptible (fwd_progress_holdoff *HZ ); - -/* Tight loop containing cond_resched(). */ -if (selfpropcb) { -WRITE_ONCE (fcs.stop,0 ); - cur_ops->call (&fcs.rh, rcu_torture_fwd_prog_cb); - } - cver =READ_ONCE (rcu_torture_current_version); - gps = cur_ops->get_gp_seq (); - sd = cur_ops->stall_dur () +1 ; - sd4 = (sd + fwd_progress_div -1 ) / fwd_progress_div; - dur = sd4 +torture_random (&trs) % (sd - sd4); - rcu_fwd_startat = jiffies; - stopat = rcu_fwd_startat + dur; -while (time_before (jiffies, stopat) && !torture_must_stop ()) { - idx = cur_ops->readlock (); -udelay (10 ); - cur_ops->readunlock (idx); -if (!fwd_progress_need_resched ||need_resched ()) -cond_resched (); - } - tested_tries++; -if (!time_before (jiffies, stopat) && !torture_must_stop ()) { - tested++; - cver =READ_ONCE (rcu_torture_current_version) - cver; - gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps); -WARN_ON (!cver && gps <2 ); -pr_alert ("%s: Duration %ld cver %ld gps %ld\n" , __func__, dur, cver, gps); - } -if (selfpropcb) { -WRITE_ONCE (fcs.stop,1 ); - cur_ops->sync ();/* Wait for running CB to complete. */ - cur_ops->cb_barrier ();/* Wait for queued callbacks. */ - } - -/* Loop continuously posting RCU callbacks. */ -WRITE_ONCE (rcu_fwd_cb_nodelay, true); - cur_ops->sync ();/* Later readers see above write. */ - rcu_fwd_startat = jiffies; - stopat = rcu_fwd_startat +MAX_FWD_CB_JIFFIES ; - n_launders =0 ; - n_launders_cb =0 ; - n_launders_sa =0 ; - n_max_cbs =0 ; - n_max_gps =0 ; -for (i =0 ; i <ARRAY_SIZE (n_launders_hist); i++) - n_launders_hist[i] =0 ; - cver =READ_ONCE (rcu_torture_current_version); - gps = cur_ops->get_gp_seq (); -while (time_before (jiffies, stopat) && !torture_must_stop ()) { - rfcp =READ_ONCE (rcu_fwd_cb_head); - rfcpn =NULL ; -if (rfcp) - rfcpn =READ_ONCE (rfcp->rfc_next); -if (rfcpn) { -if (rfcp->rfc_gps >=MIN_FWD_CB_LAUNDERS && - ++n_max_gps >=MIN_FWD_CBS_LAUNDERED ) -break ; - rcu_fwd_cb_head = rfcpn; - n_launders++; - n_launders_sa++; - }else { - rfcp =kmalloc (sizeof (*rfcp),GFP_KERNEL ); -if (WARN_ON_ONCE (!rfcp)) { -schedule_timeout_interruptible (1 ); -continue ; - } - n_max_cbs++; - n_launders_sa =0 ; - rfcp->rfc_gps =0 ; - } - cur_ops->call (&rfcp->rh, rcu_torture_fwd_cb_cr); -cond_resched (); - } - stoppedat = jiffies; - n_launders_cb_snap =READ_ONCE (n_launders_cb); - cver =READ_ONCE (rcu_torture_current_version) - cver; - gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps); - cur_ops->cb_barrier ();/* Wait for callbacks to be invoked. */ -for (;;) { - rfcp = rcu_fwd_cb_head; -if (!rfcp) -break ; - rcu_fwd_cb_head = rfcp->rfc_next; -kfree (rfcp); - } - rcu_fwd_cb_tail = &rcu_fwd_cb_head; -WRITE_ONCE (rcu_fwd_cb_nodelay, false); -if (!torture_must_stop ()) { -WARN_ON (n_max_gps <MIN_FWD_CBS_LAUNDERED ); -pr_alert ("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n" , - __func__, - stoppedat - rcu_fwd_startat, - jiffies - stoppedat, - n_launders + n_max_cbs - n_launders_cb_snap, - n_launders, n_launders_sa, - n_max_gps, n_max_cbs, cver, gps); -for (i =ARRAY_SIZE (n_launders_hist) -1 ; i >0 ; i--) -if (n_launders_hist[i] >0 ) -break ; -pr_alert ("Callback-invocation histogram:" ); -for (j =0 ; j <= i; j++) -pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]); -pr_cont ("\n" ); - } +rcu_torture_fwd_prog_nr (&tested, &tested_tries); +rcu_torture_fwd_prog_cr ();/* Avoid slow periods, better to test when busy. */ stutter_wait ("rcu_torture_fwd_prog" ); }while (!torture_must_stop ()); -if (selfpropcb) { -WARN_ON (READ_ONCE (fcs.stop) !=2 ); -destroy_rcu_head_on_stack (&fcs.rh); - }/* Short runs might not contain a valid forward-progress attempt. */ WARN_ON (!tested && tested_tries >=5 );pr_alert ("%s: tested %d tested_tries %d\n" , __func__, tested, tested_tries);
Signature · 1 line
2.17.1
PATCH 5/17 rcutorture: Affinity forward-progress test to avoid housekeeping CPUs
In reply to Paul E. McKenneyThis commit affinities the forward-progress tests to avoid hogging a housekeeping CPU on the theory that the offloaded callbacks will be running on those housekeeping CPUs.
Signed-off-by Paul E. McKenney <[email protected]>
[ paulmck: Fix NULL-pointer issue located by kbuild test robot. ]
Tested-by Rong Chen <[email protected]>
--- kernel/rcu/rcu.h | 2 ++ kernel/rcu/rcutorture.c | 1 + kernel/rcu/tree_plugin.h | 11 +++++++++++ 3 files changed, 14 insertions(+)
kernel/rcu/rcu.h +2 -0
@@ -539,8 +539,10 @@ extern struct workqueue_struct *rcu_par_gp_wq;#ifdef CONFIG_RCU_NOCB_CPU bool rcu_is_nocb_cpu (int cpu); +void rcu_bind_current_to_nocb (void );#else static inline bool rcu_is_nocb_cpu (int cpu) {return false; } +static inline void rcu_bind_current_to_nocb (void ) { }#endif #endif /* __LINUX_RCU_H */
kernel/rcu/rcutorture.c +1 -0
@@ -1803,6 +1803,7 @@ static int rcu_torture_fwd_prog(void *args)int tested_tries =0 ;VERBOSE_TOROUT_STRING ("rcu_torture_fwd_progress task started" ); +rcu_bind_current_to_nocb ();if (!IS_ENABLED (CONFIG_SMP ) || !IS_ENABLED (CONFIG_RCU_BOOST ))set_user_nice (current,MAX_NICE ); do {
kernel/rcu/tree_plugin.h +11 -0
@@ -2587,6 +2587,17 @@ static bool init_nocb_callback_list(struct rcu_data *rdp) return true; } +/* +* Bind the current task to the offloaded CPUs. If there are no offloaded +* CPUs, leave the task unbound. Splat if the bind attempt fails. +*/ +void rcu_bind_current_to_nocb (void ) +{ +if (cpumask_available (rcu_nocb_mask) &&cpumask_weight (rcu_nocb_mask)) +WARN_ON (sched_setaffinity (current->pid, rcu_nocb_mask)); +} +EXPORT_SYMBOL_GPL (rcu_bind_current_to_nocb); +#else /* #ifdef CONFIG_RCU_NOCB_CPU */ static bool rcu_nocb_cpu_needs_barrier (int cpu)
Signature · 1 line
2.17.1
PATCH 6/17 torture: Remove unnecessary "ret" variables
In reply to Paul E. McKenneyFrom: Pierce Griffiths <[email protected]>
Remove return variables (declared as "ret") in cases where, depending on whether a condition evaluates as true, the result of a function call can be immediately returned instead of storing the result in the return variable. When the condition evaluates as false, the constant initially stored in the return variable at declaration is returned instead.
Signed-off-by Pierce Griffiths <[email protected]>
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/torture.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-)
kernel/torture.c +8 -14
@@ -245,16 +245,15 @@ torture_onoff(void *arg)*/ int torture_onoff_init (long ooholdoff,long oointerval) { -int ret =0 ; -#ifdef CONFIG_HOTPLUG_CPU onoff_holdoff = ooholdoff; onoff_interval = oointerval;if (onoff_interval <=0 )return 0 ; - ret =torture_create_kthread (torture_onoff,NULL , onoff_task); -#endif /* #ifdef CONFIG_HOTPLUG_CPU */ -return ret; +return torture_create_kthread (torture_onoff,NULL , onoff_task); +#else /* #ifdef CONFIG_HOTPLUG_CPU */ +return 0 ; +#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ }EXPORT_SYMBOL_GPL (torture_onoff_init);
@@ -525,15 +524,13 @@ static int torture_shutdown(void *arg)*/ int torture_shutdown_init (int ssecs,void (*cleanup)(void )) { -int ret =0 ; - torture_shutdown_hook = cleanup;if (ssecs >0 ) { shutdown_time =ktime_add (ktime_get (),ktime_set (ssecs,0 )); - ret =torture_create_kthread (torture_shutdown,NULL , +return torture_create_kthread (torture_shutdown,NULL , shutdown_task); } -return ret; +return 0 ; }EXPORT_SYMBOL_GPL (torture_shutdown_init);
@@ -632,13 +629,10 @@ static int torture_stutter(void *arg)/* * Initialize and kick off the torture_stutter kthread. */ -int torture_stutter_init (int s) +int torture_stutter_init (const int s) { -int ret; - stutter = s; - ret =torture_create_kthread (torture_stutter,NULL , stutter_task); -return ret; +return torture_create_kthread (torture_stutter,NULL , stutter_task); }EXPORT_SYMBOL_GPL (torture_stutter_init);
Signature · 1 line
2.17.1
PATCH 7/17 rcutorture: Prepare for asynchronous access to rcu_fwd_startat
In reply to Paul E. McKenneyBecause rcutorture's forward-progress checking will trigger from an OOM notifier, this notifier will introduce asynchronous concurrent access to the rcu_fwd_startat variable. This commit therefore prepares for this by converting updates to WRITE_ONCE().
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/rcu/rcutorture.c +2 -2
@@ -1679,7 +1679,7 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries) sd = cur_ops->stall_dur () +1 ; sd4 = (sd + fwd_progress_div -1 ) / fwd_progress_div; dur = sd4 +torture_random (&trs) % (sd - sd4); - rcu_fwd_startat = jiffies; +WRITE_ONCE (rcu_fwd_startat, jiffies); stopat = rcu_fwd_startat + dur;while (time_before (jiffies, stopat) && !torture_must_stop ()) { idx = cur_ops->readlock ();
@@ -1728,7 +1728,7 @@ static void rcu_torture_fwd_prog_cr(void)/* Loop continuously posting RCU callbacks. */ WRITE_ONCE (rcu_fwd_cb_nodelay, true); cur_ops->sync ();/* Later readers see above write. */ - rcu_fwd_startat = jiffies; +WRITE_ONCE (rcu_fwd_startat, jiffies); stopat = rcu_fwd_startat +MAX_FWD_CB_JIFFIES ; n_launders =0 ; n_launders_cb =0 ;
Signature · 1 line
2.17.1
PATCH 8/17 rcutorture: Dump grace-period diagnostics upon forward-progress OOM
In reply to Paul E. McKenneyThis commit adds an OOM notifier during rcutorture forward-progress testing. If this notifier is invoked, it dumps out some grace-period state to help debug the forward-progress problem.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcu.h | 2 ++ kernel/rcu/rcutorture.c | 31 ++++++++++++++++++++++++++++--- kernel/rcu/tree.c | 20 ++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-)
kernel/rcu/rcu.h +2 -0
@@ -526,12 +526,14 @@ srcu_batches_completed(struct srcu_struct *sp) { return 0; }static inline void rcu_force_quiescent_state (void ) { }static inline void show_rcu_gp_kthreads (void ) { }static inline int rcu_get_gp_kthreads_prio (void ) {return 0 ; } +static inline void rcu_fwd_progress_check (unsigned long j) { }#else /* #ifdef CONFIG_TINY_RCU */ unsigned long rcu_get_gp_seq (void );unsigned long rcu_exp_batches_completed (void );unsigned long srcu_batches_completed (struct srcu_struct *sp);void show_rcu_gp_kthreads (void );int rcu_get_gp_kthreads_prio (void ); +void rcu_fwd_progress_check (unsigned long j);void rcu_force_quiescent_state (void );extern struct workqueue_struct *rcu_gp_wq;extern struct workqueue_struct *rcu_par_gp_wq;
kernel/rcu/rcutorture.c +28 -3
@@ -56,6 +56,7 @@#include <linux/vmalloc.h> #include <linux/sched/debug.h> #include <linux/sched/sysctl.h> +#include <linux/oom.h> #include "rcu.h"
@@ -1624,6 +1625,7 @@ static struct rcu_fwd_cb *rcu_fwd_cb_head;static struct rcu_fwd_cb **rcu_fwd_cb_tail = &rcu_fwd_cb_head;static long n_launders_cb;static unsigned long rcu_fwd_startat; +static bool rcu_fwd_emergency_stop;#define MAX_FWD_CB_JIFFIES (8 * HZ)/* Maximum CB test duration. */ #define MIN_FWD_CB_LAUNDERS 3/* This many CB invocations to count. */ #define MIN_FWD_CBS_LAUNDERED 100/* Number of counted CBs. */
@@ -1681,7 +1683,8 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries) dur = sd4 +torture_random (&trs) % (sd - sd4);WRITE_ONCE (rcu_fwd_startat, jiffies); stopat = rcu_fwd_startat + dur; -while (time_before (jiffies, stopat) && !torture_must_stop ()) { +while (time_before (jiffies, stopat) && + !READ_ONCE (rcu_fwd_emergency_stop) && !torture_must_stop ()) { idx = cur_ops->readlock ();udelay (10 ); cur_ops->readunlock (idx);
@@ -1689,7 +1692,8 @@ static void rcu_torture_fwd_prog_nr(int *tested, int *tested_tries)cond_resched (); } (*tested_tries)++; -if (!time_before (jiffies, stopat) && !torture_must_stop ()) { +if (!time_before (jiffies, stopat) && + !READ_ONCE (rcu_fwd_emergency_stop) && !torture_must_stop ()) { (*tested)++; cver =READ_ONCE (rcu_torture_current_version) - cver; gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps);
@@ -1739,7 +1743,8 @@ static void rcu_torture_fwd_prog_cr(void) n_launders_hist[i] =0 ; cver =READ_ONCE (rcu_torture_current_version); gps = cur_ops->get_gp_seq (); -while (time_before (jiffies, stopat) && !torture_must_stop ()) { +while (time_before (jiffies, stopat) && + !READ_ONCE (rcu_fwd_emergency_stop) && !torture_must_stop ()) { rfcp =READ_ONCE (rcu_fwd_cb_head); rfcpn =NULL ;if (rfcp)
@@ -1796,6 +1801,23 @@ static void rcu_torture_fwd_prog_cr(void) } } + +/* +* OOM notifier, but this only prints diagnostic information for the +* current forward-progress test. +*/ +static int rcutorture_oom_notify (struct notifier_block *self, +unsigned long notused,void *nfreed) +{ +rcu_fwd_progress_check (1 + (jiffies -READ_ONCE (rcu_fwd_startat) /2 )); +WRITE_ONCE (rcu_fwd_emergency_stop, true); +return NOTIFY_OK ; +} + +static struct notifier_block rcutorture_oom_nb = { + .notifier_call = rcutorture_oom_notify +}; +/* Carry out grace-period forward-progress testing. */ static int rcu_torture_fwd_prog (void *args) {
@@ -1808,8 +1830,11 @@ static int rcu_torture_fwd_prog(void *args)set_user_nice (current,MAX_NICE ); do { schedule_timeout_interruptible(fwd_progress_holdoff * HZ); + WRITE_ONCE(rcu_fwd_emergency_stop, false); + register_oom_notifier(&rcutorture_oom_nb); rcu_torture_fwd_prog_nr(&tested, &tested_tries); rcu_torture_fwd_prog_cr(); + unregister_oom_notifier(&rcutorture_oom_nb); /* Avoid slow periods, better to test when busy. */ stutter_wait("rcu_torture_fwd_prog");
kernel/rcu/tree.c +20 -0
@@ -2654,6 +2654,26 @@ rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp)raw_spin_unlock_irqrestore_rcu_node (rnp, flags); } +/* +* Do a forward-progress check for rcutorture. This is normally invoked +* due to an OOM event. The argument "j" gives the time period during +* which rcutorture would like progress to have been made. +*/ +void rcu_fwd_progress_check (unsigned long j) +{ +struct rcu_data *rdp; + +if (rcu_gp_in_progress ()) { +show_rcu_gp_kthreads (); + }else { +preempt_disable (); + rdp =this_cpu_ptr (&rcu_data); +rcu_check_gp_start_stall (rdp->mynode, rdp, j); +preempt_enable (); + } +} +EXPORT_SYMBOL_GPL (rcu_fwd_progress_check); +/* * This does the RCU core processing work for the specified rcu_data * structures. This may be called only from the CPU to whom the rdp
Signature · 1 line
2.17.1
PATCH 9/17 rcu: Account for nocb-CPU callback counts in RCU CPU stall warnings
In reply to Paul E. McKenneyThe RCU CPU stall warnings print an estimate of the total number of RCU callbacks queued in the system, but this estimate leaves out the callbacks queued for nocbs CPUs. This commit therefore introduces rcu_get_n_cbs_cpu(), which gives an accurate callback estimate for both nocbs and normal CPUs, and uses this new function as needed.
This commit also introduces a rcu_get_n_cbs_nocb_cpu() helper function that returns the number of callbacks for nocbs CPUs or zero otherwise, and also uses this function in place of direct access to ->nocb_q_count while in the area (fewer characters, you see).
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/tree.c | 19 +++++++++++++++---- kernel/rcu/tree.h | 1 + kernel/rcu/tree_plugin.h | 24 +++++++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-)
kernel/rcu/tree.c +15 -4
@@ -207,6 +207,19 @@ static int rcu_gp_in_progress(void)return rcu_seq_state (rcu_seq_current (&rcu_state.gp_seq)); } +/* +* Return the number of callbacks queued on the specified CPU. +* Handles both the nocbs and normal cases. +*/ +static long rcu_get_n_cbs_cpu (int cpu) +{ +struct rcu_data *rdp =per_cpu_ptr (&rcu_data, cpu); + +if (rcu_segcblist_is_enabled (&rdp->cblist))/* Online normal CPU? */ +return rcu_segcblist_n_cbs (&rdp->cblist); +return rcu_get_n_cbs_nocb_cpu (rdp);/* Works for offline, too. */ +} +void rcu_softirq_qs (void ) {rcu_qs ();
@@ -1262,8 +1275,7 @@ static void print_other_cpu_stall(unsigned long gp_seq)print_cpu_stall_info_end ();for_each_possible_cpu (cpu) - totqlen +=rcu_segcblist_n_cbs (&per_cpu_ptr (&rcu_data, - cpu)->cblist); + totqlen +=rcu_get_n_cbs_cpu (cpu);pr_cont ("(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n" ,smp_processor_id (), (long )(jiffies - rcu_state.gp_start), (long )rcu_seq_current (&rcu_state.gp_seq), totqlen);
@@ -1323,8 +1335,7 @@ static void print_cpu_stall(void)raw_spin_unlock_irqrestore_rcu_node (rdp->mynode, flags);print_cpu_stall_info_end ();for_each_possible_cpu (cpu) - totqlen +=rcu_segcblist_n_cbs (&per_cpu_ptr (&rcu_data, - cpu)->cblist); + totqlen +=rcu_get_n_cbs_cpu (cpu);pr_cont (" (t=%lu jiffies g=%ld q=%lu)\n" , jiffies - rcu_state.gp_start, (long )rcu_seq_current (&rcu_state.gp_seq), totqlen);
kernel/rcu/tree.h +1 -0
@@ -466,6 +466,7 @@ static void __init rcu_spawn_nocb_kthreads(void);static void __init rcu_organize_nocb_kthreads (void );#endif /* #ifdef CONFIG_RCU_NOCB_CPU */ static bool init_nocb_callback_list (struct rcu_data *rdp); +static unsigned long rcu_get_n_cbs_nocb_cpu (struct rcu_data *rdp);static void rcu_bind_gp_kthread (void );static bool rcu_nohz_full_cpu (void );static void rcu_dynticks_task_enter (void );
kernel/rcu/tree_plugin.h +19 -5
@@ -1997,7 +1997,7 @@ static bool rcu_nocb_cpu_needs_barrier(int cpu)* (if a callback is in fact needed). This is associated with an * atomic_inc() in the caller. */ - ret =atomic_long_read (&rdp->nocb_q_count); + ret =rcu_get_n_cbs_nocb_cpu (rdp);#ifdef CONFIG_PROVE_RCU rhp =READ_ONCE (rdp->nocb_head);
@@ -2052,7 +2052,7 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp, TPS("WakeNotPoll"));return ; } - len =atomic_long_read (&rdp->nocb_q_count); + len =rcu_get_n_cbs_nocb_cpu (rdp);if (old_rhpp == &rdp->nocb_head) {if (!irqs_disabled_flags (flags)) {/* ... if queue was empty ... */
@@ -2101,11 +2101,11 @@ static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,trace_rcu_kfree_callback (rcu_state.name, rhp, (unsigned long )rhp->func, -atomic_long_read (&rdp->nocb_q_count_lazy), - -atomic_long_read (&rdp->nocb_q_count)); + -rcu_get_n_cbs_nocb_cpu (rdp)); else trace_rcu_callback(rcu_state.name, rhp, -atomic_long_read(&rdp->nocb_q_count_lazy), - -atomic_long_read(&rdp->nocb_q_count)); + -rcu_get_n_cbs_nocb_cpu(rdp));/* * If called from an extended quiescent state with interrupts
@@ -2328,7 +2328,7 @@ static int rcu_nocb_kthread(void *arg)/* Each pass through the following loop invokes a callback. */ trace_rcu_batch_start (rcu_state.name,atomic_long_read (&rdp->nocb_q_count_lazy), -atomic_long_read (&rdp->nocb_q_count),-1 ); +rcu_get_n_cbs_nocb_cpu (rdp),-1 ); c = cl =0 ;while (list) { next = list->next;
@@ -2598,6 +2598,15 @@ void rcu_bind_current_to_nocb(void) }EXPORT_SYMBOL_GPL (rcu_bind_current_to_nocb); +/* +* Return the number of RCU callbacks still queued from the specified +* CPU, which must be a nocbs CPU. +*/ +static unsigned long rcu_get_n_cbs_nocb_cpu (struct rcu_data *rdp) +{ +return atomic_long_read (&rdp->nocb_q_count); +} +#else /* #ifdef CONFIG_RCU_NOCB_CPU */ static bool rcu_nocb_cpu_needs_barrier (int cpu)
@@ -2658,6 +2667,11 @@ static bool init_nocb_callback_list(struct rcu_data *rdp) return false; } +static unsigned long rcu_get_n_cbs_nocb_cpu (struct rcu_data *rdp) +{ +return 0 ; +} +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ /*
Signature · 1 line
2.17.1
PATCH 10/17 rcu: Print per-CPU callback counts for forward-progress failures
In reply to Paul E. McKenneyThis commit prints out the non-zero per-CPU callback counts when a forware-progress error (OOM event) occurs.
Signed-off-by Paul E. McKenney <[email protected]>
[ paulmck: Fix a pair of uninitialized locals spotted by kbuild test robot. ]
--- kernel/rcu/tree.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
kernel/rcu/tree.c +18 -0
@@ -2672,6 +2672,10 @@ rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp)*/ void rcu_fwd_progress_check (unsigned long j) { +unsigned long cbs; +int cpu; +unsigned long max_cbs =0 ; +int max_cpu =-1 ;struct rcu_data *rdp;if (rcu_gp_in_progress ()) {
@@ -2682,6 +2686,20 @@ void rcu_fwd_progress_check(unsigned long j)rcu_check_gp_start_stall (rdp->mynode, rdp, j);preempt_enable (); } +for_each_possible_cpu (cpu) { + cbs =rcu_get_n_cbs_cpu (cpu); +if (!cbs) +continue ; +if (max_cpu <0 ) +pr_info ("%s: callbacks" , __func__); +pr_cont (" %d: %lu" , cpu, cbs); +if (cbs <= max_cbs) +continue ; + max_cbs = cbs; + max_cpu = cpu; + } +if (max_cpu >=0 ) +pr_cont ("\n" ); }EXPORT_SYMBOL_GPL (rcu_fwd_progress_check);
Signature · 1 line
2.17.1
PATCH 11/17 rcutorture: Print GP age upon forward-progress failure
In reply to Paul E. McKenneySigned-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/tree.c | 2 ++ 1 file changed, 2 insertions(+)
kernel/rcu/tree.c +2 -0
@@ -2679,6 +2679,8 @@ void rcu_fwd_progress_check(unsigned long j)struct rcu_data *rdp;if (rcu_gp_in_progress ()) { +pr_info ("%s: GP age %lu jiffies\n" , + __func__, jiffies - rcu_state.gp_start);show_rcu_gp_kthreads (); }else {preempt_disable ();
Signature · 1 line
2.17.1
PATCH 12/17 rcutorture: Print histogram of CB invocation at OOM time
In reply to Paul E. McKenneyOne reason why a forward-progress test might fail would be if something prevented or delayed callback invocation. This commit therefore adds a callback-invocation histogram printout when OOM is reported to rcutorture.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-)
kernel/rcu/rcutorture.c +16 -8
@@ -1631,6 +1631,20 @@ static bool rcu_fwd_emergency_stop;#define MIN_FWD_CBS_LAUNDERED 100/* Number of counted CBs. */ static long n_launders_hist[2 *MAX_FWD_CB_JIFFIES /HZ ]; +static void rcu_torture_fwd_cb_hist (void ) +{ +int i; +int j; + +for (i =ARRAY_SIZE (n_launders_hist) -1 ; i >0 ; i--) +if (n_launders_hist[i] >0 ) +break ; +pr_alert ("%s: Callback-invocation histogram:" , __func__); +for (j =0 ; j <= i; j++) +pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]); +pr_cont ("\n" ); +} +/* Callback function for continuous-flood RCU callbacks. */ static void rcu_torture_fwd_cb_cr (struct rcu_head *rhp) {
@@ -1718,7 +1732,6 @@ static void rcu_torture_fwd_prog_cr(void)unsigned long cver;unsigned long gps;int i; -int j;long n_launders;long n_launders_cb_snap;long n_launders_sa;
@@ -1791,13 +1804,7 @@ static void rcu_torture_fwd_prog_cr(void) n_launders + n_max_cbs - n_launders_cb_snap, n_launders, n_launders_sa, n_max_gps, n_max_cbs, cver, gps); -for (i =ARRAY_SIZE (n_launders_hist) -1 ; i >0 ; i--) -if (n_launders_hist[i] >0 ) -break ; -pr_alert ("Callback-invocation histogram:" ); -for (j =0 ; j <= i; j++) -pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]); -pr_cont ("\n" ); +rcu_torture_fwd_cb_hist (); } }
@@ -1809,6 +1816,7 @@ static void rcu_torture_fwd_prog_cr(void)static int rcutorture_oom_notify (struct notifier_block *self,unsigned long notused,void *nfreed) { +rcu_torture_fwd_cb_hist ();rcu_fwd_progress_check (1 + (jiffies -READ_ONCE (rcu_fwd_startat) /2 ));WRITE_ONCE (rcu_fwd_emergency_stop, true);return NOTIFY_OK ;
Signature · 1 line
2.17.1
PATCH 13/17 rcutorture: Print time since GP end upon forward-progress failure
In reply to Paul E. McKenneyIf rcutorture's forward-progress tests fail while a grace period is not in progress, it is useful to print the time since the last grace period ended as a way to detect failure to launch a new grace period. This commit therefore makes this change.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/tree.c | 5 ++++- kernel/rcu/tree.h | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-)
kernel/rcu/tree.c +4 -1
@@ -1997,7 +1997,8 @@ static void rcu_gp_cleanup(void)WRITE_ONCE (rcu_state.gp_activity, jiffies);raw_spin_lock_irq_rcu_node (rnp); - gp_duration = jiffies - rcu_state.gp_start; + rcu_state.gp_end = jiffies; + gp_duration = rcu_state.gp_end - rcu_state.gp_start;if (gp_duration > rcu_state.gp_max) rcu_state.gp_max = gp_duration;
@@ -2683,6 +2684,8 @@ void rcu_fwd_progress_check(unsigned long j) __func__, jiffies - rcu_state.gp_start);show_rcu_gp_kthreads (); }else { +pr_info ("%s: Last GP end %lu jiffies ago\n" , + __func__, jiffies - rcu_state.gp_end);preempt_disable (); rdp =this_cpu_ptr (&rcu_data);rcu_check_gp_start_stall (rdp->mynode, rdp, j);
kernel/rcu/tree.h +2 -0
@@ -328,6 +328,8 @@ struct rcu_state {/* force_quiescent_state(). */ unsigned long gp_start;/* Time at which GP started, */ /* but in jiffies. */ +unsigned long gp_end;/* Time last GP ended, again */ +/* in jiffies. */ unsigned long gp_activity;/* Time of last GP kthread */ /* activity in jiffies. */ unsigned long gp_req_activity;/* Time of last GP request */
Signature · 1 line
2.17.1
PATCH 14/17 rcutorture: Print forward-progress test age upon failure
In reply to Paul E. McKenneyThis commit prints the age of the forward-progress test in jiffies, in order to allow better interpretation of the callback-invocation histograms.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/rcu/rcutorture.c +2 -1
@@ -1639,7 +1639,8 @@ static void rcu_torture_fwd_cb_hist(void)for (i =ARRAY_SIZE (n_launders_hist) -1 ; i >0 ; i--)if (n_launders_hist[i] >0 )break ; -pr_alert ("%s: Callback-invocation histogram:" , __func__); +pr_alert ("%s: Callback-invocation histogram (duration %lu jiffies):" , + __func__, jiffies - rcu_fwd_startat);for (j =0 ; j <= i; j++)pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]);pr_cont ("\n" );
Signature · 1 line
2.17.1
PATCH 15/17 rcutorture: Recover from OOM during forward-progress tests
In reply to Paul E. McKenneyThis commit causes the OOM handler to do rcu_barrier() calls and to free up forward-progress callbacks in order to recover from OOM events. The current test is terminated, but subsequent forward-progress tests can proceed. This allows a long test to result in multiple forward-progress failures, greatly reducing the required testing time.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 60 +++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 11 deletions(-)
kernel/rcu/rcutorture.c +49 -11
@@ -1649,13 +1649,14 @@ static void rcu_torture_fwd_cb_hist(void)/* Callback function for continuous-flood RCU callbacks. */ static void rcu_torture_fwd_cb_cr (struct rcu_head *rhp) { +unsigned long flags;int i; struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh);struct rcu_fwd_cb **rfcpp; rfcp->rfc_next =NULL ; rfcp->rfc_gps++; -spin_lock (&rcu_fwd_lock); +spin_lock_irqsave (&rcu_fwd_lock, flags); rfcpp = rcu_fwd_cb_tail; rcu_fwd_cb_tail = &rfcp->rfc_next;WRITE_ONCE (*rfcpp, rfcp);
@@ -1664,7 +1665,33 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)if (i >=ARRAY_SIZE (n_launders_hist)) i =ARRAY_SIZE (n_launders_hist) -1 ; n_launders_hist[i]++; -spin_unlock (&rcu_fwd_lock); +spin_unlock_irqrestore (&rcu_fwd_lock, flags); +} + +/* +* Free all callbacks on the rcu_fwd_cb_head list, either because the +* test is over or because we hit an OOM event. +*/ +static unsigned long rcu_torture_fwd_prog_cbfree (void ) +{ +unsigned long flags; +unsigned long freed =0 ; +struct rcu_fwd_cb *rfcp; + +for (;;) { +spin_lock_irqsave (&rcu_fwd_lock, flags); + rfcp = rcu_fwd_cb_head; +if (!rfcp) +break ; + rcu_fwd_cb_head = rfcp->rfc_next; +if (!rcu_fwd_cb_head) + rcu_fwd_cb_tail = &rcu_fwd_cb_head; +spin_unlock_irqrestore (&rcu_fwd_lock, flags); +kfree (rfcp); + freed++; + } +spin_unlock_irqrestore (&rcu_fwd_lock, flags); +return freed; }/* Carry out need_resched()/cond_resched() forward-progress testing. */
@@ -1743,6 +1770,9 @@ static void rcu_torture_fwd_prog_cr(void)unsigned long stopat;unsigned long stoppedat; +if (READ_ONCE (rcu_fwd_emergency_stop)) +return ;/* Get out of the way quickly, no GP wait! */ +/* Loop continuously posting RCU callbacks. */ WRITE_ONCE (rcu_fwd_cb_nodelay, true); cur_ops->sync ();/* Later readers see above write. */
@@ -1788,16 +1818,10 @@ static void rcu_torture_fwd_prog_cr(void) cver =READ_ONCE (rcu_torture_current_version) - cver; gps =rcutorture_seq_diff (cur_ops->get_gp_seq (), gps); cur_ops->cb_barrier ();/* Wait for callbacks to be invoked. */ -for (;;) { - rfcp = rcu_fwd_cb_head; -if (!rfcp) -break ; - rcu_fwd_cb_head = rfcp->rfc_next; -kfree (rfcp); - } - rcu_fwd_cb_tail = &rcu_fwd_cb_head; + (void )rcu_torture_fwd_prog_cbfree (); +WRITE_ONCE (rcu_fwd_cb_nodelay, false); -if (!torture_must_stop ()) { +if (!torture_must_stop () && !READ_ONCE (rcu_fwd_emergency_stop)) {WARN_ON (n_max_gps <MIN_FWD_CBS_LAUNDERED ); pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n", __func__,
@@ -1817,9 +1841,23 @@ static void rcu_torture_fwd_prog_cr(void)static int rcutorture_oom_notify (struct notifier_block *self,unsigned long notused,void *nfreed) { +WARN (1 ,"%s invoked upon OOM during forward-progress testing.\n" , + __func__);rcu_torture_fwd_cb_hist ();rcu_fwd_progress_check (1 + (jiffies -READ_ONCE (rcu_fwd_startat) /2 ));WRITE_ONCE (rcu_fwd_emergency_stop, true); +smp_mb ();/* Emergency stop before free and wait to avoid hangs. */ +pr_info ("%s: Freed %lu RCU callbacks.\n" , + __func__,rcu_torture_fwd_prog_cbfree ()); +rcu_barrier (); +pr_info ("%s: Freed %lu RCU callbacks.\n" , + __func__,rcu_torture_fwd_prog_cbfree ()); +rcu_barrier (); +pr_info ("%s: Freed %lu RCU callbacks.\n" , + __func__,rcu_torture_fwd_prog_cbfree ()); +smp_mb ();/* Frees before return to avoid redoing OOM. */ + (*(unsigned long *)nfreed)++;/* Forward progress CBs freed! */ +pr_info ("%s returning after OOM processing.\n" , __func__);return NOTIFY_OK ; }
Signature · 1 line
2.17.1
PATCH 16/17 rcutorture: Use 100ms buckets for forward-progress callback histograms
In reply to Paul E. McKenneyThis commit narrows the scope of each bucket of the forward-progress callback-invocation histograms from one second to 100 milliseconds, which aids debugging of forward-progress problems by making shorter-duration callback-invocation stalls visible.
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
kernel/rcu/rcutorture.c +5 -3
@@ -1629,7 +1629,8 @@ static bool rcu_fwd_emergency_stop;#define MAX_FWD_CB_JIFFIES (8 * HZ)/* Maximum CB test duration. */ #define MIN_FWD_CB_LAUNDERS 3/* This many CB invocations to count. */ #define MIN_FWD_CBS_LAUNDERED 100/* Number of counted CBs. */ -static long n_launders_hist[2 *MAX_FWD_CB_JIFFIES /HZ ]; +#define FWD_CBS_HIST_DIV 10/* Histogram buckets/second. */ +static long n_launders_hist[2 *MAX_FWD_CB_JIFFIES / (HZ /FWD_CBS_HIST_DIV )];static void rcu_torture_fwd_cb_hist (void ) {
@@ -1642,7 +1643,8 @@ static void rcu_torture_fwd_cb_hist(void)pr_alert ("%s: Callback-invocation histogram (duration %lu jiffies):" , __func__, jiffies - rcu_fwd_startat);for (j =0 ; j <= i; j++) -pr_cont (" %ds: %ld" , j +1 , n_launders_hist[j]); +pr_cont (" %ds/%d: %ld" , + j +1 ,FWD_CBS_HIST_DIV , n_launders_hist[j]);pr_cont ("\n" ); }
@@ -1661,7 +1663,7 @@ static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) rcu_fwd_cb_tail = &rfcp->rfc_next;WRITE_ONCE (*rfcpp, rfcp);WRITE_ONCE (n_launders_cb, n_launders_cb +1 ); - i = ((jiffies - rcu_fwd_startat) /HZ ); + i = ((jiffies - rcu_fwd_startat) / (HZ /FWD_CBS_HIST_DIV ));if (i >=ARRAY_SIZE (n_launders_hist)) i =ARRAY_SIZE (n_launders_hist) -1 ; n_launders_hist[i]++;
Signature · 1 line
2.17.1
PATCH 17/17 rcutorture: Don't do busted forward-progress testing
In reply to Paul E. McKenneyThe "busted" rcutorture type is an intentionally broken implementation of RCU. Doing forward-progress testing on this implementation is not particularly meaningful on the one hand and can result in fatal abuse of the memory allocator on the other. This commit therefore disables forward-progress testing of the "busted" rcutorture type.
Reported-by kernel test robot <[email protected]>
Signed-off-by Paul E. McKenney <[email protected]>
--- kernel/rcu/rcutorture.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/rcu/rcutorture.c +2 -1
@@ -1900,7 +1900,8 @@ static int __init rcu_torture_fwd_prog_init(void) {if (!fwd_progress)return 0 ;/* Not requested, so don't do it. */ -if (!cur_ops->stall_dur || cur_ops->stall_dur () <=0 ) { +if (!cur_ops->stall_dur || cur_ops->stall_dur () <=0 || + cur_ops == &rcu_busted_ops) {VERBOSE_TOROUT_STRING ("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test" );return 0 ; }
Signature · 1 line
2.17.1