Skip to contentSkip to conversation

Reader end deboost testing

RFC PATCH In review Joel Fernandes 2 participants 7 messages Jun 16 – Jun 17

Subsystems (2)
lore
Share & tools
lore

Joel Fernandes

author maintainer
#0 lore raw

cover Reader end deboost testing

This series adds rcutorture coverage for the RCU priority-boosting "tardy deboost" issue. The first 3 patches are Paul's and the next 2 are fixes I had to make to the test, so that another patch series (I haven't posted yet) can correctly be tested. I am posting just the test patches first in advance, which can go in independently if we agree on the changes.

Joel Fernandes (2):
rcutorture: tighten boost-WARN to exclude any implicit-reader context
rcutorture: give async deboost mechanisms up to 500us before WARN

Paul E. McKenney (3):
rcutorture: Abstract reader-segment dump into
rcu_torture_dump_read_segs()
rcutorture: Check for immediate deboosting at reader end
rcutorture: Test RCU readers from hardware interrupt handlers

 kernel/rcu/rcu.h         |   7 ++
 kernel/rcu/rcutorture.c  | 257 +++++++++++++++++++++++++++------------
 kernel/rcu/tree_plugin.h |  32 ++++-
 3 files changed, 218 insertions(+), 78 deletions(-)
Signature · 1 line
2.34.1
author maintainer
#1 lore raw

RFC PATCH 1/5 rcutorture: Abstract reader-segment dump into rcu_torture_dump_read_segs()

From: "Paul E. McKenney" <[email protected]>

This commit abstracts the open-coded dumping of reader segments in the rcu_torture_cleanup() function into a new rcu_torture_dump_read_segs() function. This abstraction will allow reader segments to be dumped for other purposes.

---
 kernel/rcu/rcutorture.c | 148 +++++++++++++++++++++-------------------
 1 file changed, 76 insertions(+), 72 deletions(-)
kernel/rcu/rcutorture.c +76 -72
index 5f2848b828dc..42f94c62c18a 100644
@@ -2378,6 +2378,80 @@ struct rcu_torture_one_read_state {
 	unsigned long long ts;
 };
 
+static void rcu_torture_dump_read_segs(struct rt_read_seg *rrsp, int nsegs)
+{
+	bool firsttime;
+	int i;
+	int j;
+
+	firsttime = 1;
+	for (i = 0; i < nsegs; i++) {
+		if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP))
+			pr_alert("\t%lluus ", div64_u64(rrsp[i].rt_ts, 1000ULL));
+		else
+			pr_alert("\t");
+		pr_cont("%d: %#4x", i, rrsp[i].rt_readstate);
+		if (rrsp[i].rt_delay_jiffies != 0) {
+			pr_cont("%s%ldjiffies", firsttime ? "" : "+",
+				rrsp[i].rt_delay_jiffies);
+			firsttime = 0;
+		}
+		if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) {
+			pr_cont(" CPU %2d", rrsp[i].rt_cpu);
+			if (rrsp[i].rt_cpu != rrsp[i].rt_end_cpu)
+				pr_cont("->%-2d", rrsp[i].rt_end_cpu);
+			else
+				pr_cont(" ...");
+		}
+		if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) &&
+		    cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) {
+			char buf1[20+1];
+			char buf2[20+1];
+			char sepchar = '-';
+
+			cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq, buf1, ARRAY_SIZE(buf1));
+			cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq_end, buf2, ARRAY_SIZE(buf2));
+			if (rrsp[i].rt_gp_seq == rrsp[i].rt_gp_seq_end) {
+				if (buf2[0]) {
+					for (j = 0; buf2[j]; j++)
+						buf2[j] = '.';
+					if (j)
+						buf2[j - 1] = ' ';
+				}
+				sepchar = ' ';
+			}
+			pr_cont(" %s%c%s", buf1, sepchar, buf2);
+		}
+		if (rrsp[i].rt_delay_ms != 0) {
+			pr_cont(" %s%ldms", firsttime ? "" : "+", rrsp[i].rt_delay_ms);
+			firsttime = 0;
+		}
+		if (rrsp[i].rt_delay_us != 0) {
+			pr_cont(" %s%ldus", firsttime ? "" : "+", rrsp[i].rt_delay_us);
+			firsttime = 0;
+		}
+		pr_cont("%s", rrsp[i].rt_preempted ? " preempted" : "");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_BH)
+			pr_cont(" BH");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_IRQ)
+			pr_cont(" IRQ");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_PREEMPT)
+			pr_cont(" PREEMPT");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RBH)
+			pr_cont(" RBH");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_SCHED)
+			pr_cont(" SCHED");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_1)
+			pr_cont(" RCU_1");
+		if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_2)
+			pr_cont(" RCU_2");
+		pr_cont("\n");
+
+	}
+	if (rt_read_preempted)
+		pr_alert("\tReader was preempted.\n");
+}
+
 static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp,
 					    struct torture_random_state *trsp)
 {
@@ -4071,11 +4145,9 @@ static void rcu_gpwrap_lag_cleanup(void)
 static void
 rcu_torture_cleanup(void)
 {
-	int firsttime;
 	int flags = 0;
 	unsigned long gp_seq = 0;
 	int i;
-	int j;
 
 	if (torture_cleanup_begin()) {
 		if (cur_ops->cb_barrier != NULL) {
@@ -4160,76 +4232,8 @@ rcu_torture_cleanup(void)
 		pr_alert("Failure/close-call rcutorture reader segments:\n");
 		if (rt_read_nsegs == 0)
 			pr_alert("\t: No segments recorded!!!\n");
-		firsttime = 1;
-		for (i = 0; i < rt_read_nsegs; i++) {
-			if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP))
-				pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL));
-			else
-				pr_alert("\t");
-			pr_cont("%d: %#4x", i, err_segs[i].rt_readstate);
-			if (err_segs[i].rt_delay_jiffies != 0) {
-				pr_cont("%s%ldjiffies", firsttime ? "" : "+",
-					err_segs[i].rt_delay_jiffies);
-				firsttime = 0;
-			}
-			if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) {
-				pr_cont(" CPU %2d", err_segs[i].rt_cpu);
-				if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu)
-					pr_cont("->%-2d", err_segs[i].rt_end_cpu);
-				else
-					pr_cont(" ...");
-			}
-			if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) &&
-			    cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) {
-				char buf1[20+1];
-				char buf2[20+1];
-				char sepchar = '-';
-
-				cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq,
-							buf1, ARRAY_SIZE(buf1));
-				cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end,
-							buf2, ARRAY_SIZE(buf2));
-				if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) {
-					if (buf2[0]) {
-						for (j = 0; buf2[j]; j++)
-							buf2[j] = '.';
-						if (j)
-							buf2[j - 1] = ' ';
-					}
-					sepchar = ' ';
-				}
-				pr_cont(" %s%c%s", buf1, sepchar, buf2);
-			}
-			if (err_segs[i].rt_delay_ms != 0) {
-				pr_cont(" %s%ldms", firsttime ? "" : "+",
-					err_segs[i].rt_delay_ms);
-				firsttime = 0;
-			}
-			if (err_segs[i].rt_delay_us != 0) {
-				pr_cont(" %s%ldus", firsttime ? "" : "+",
-					err_segs[i].rt_delay_us);
-				firsttime = 0;
-			}
-			pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : "");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH)
-				pr_cont(" BH");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ)
-				pr_cont(" IRQ");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT)
-				pr_cont(" PREEMPT");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH)
-				pr_cont(" RBH");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED)
-				pr_cont(" SCHED");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1)
-				pr_cont(" RCU_1");
-			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2)
-				pr_cont(" RCU_2");
-			pr_cont("\n");
-
-		}
-		if (rt_read_preempted)
-			pr_alert("\tReader was preempted.\n");
+		else
+			rcu_torture_dump_read_segs(err_segs, rt_read_nsegs);
 	}
 	if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
 		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
Signature · 1 line
2.34.1
author maintainer
#2 lore raw

RFC PATCH 2/5 rcutorture: Check for immediate deboosting at reader end

In reply to Joel Fernandes

From: "Paul E. McKenney" <[email protected]>

This commit adds a check for failure to have fully deboosted a multi-segmented RCU reader at the end of the full read-side critical section. This check only happens for fully task-level readers, because a a handler might have interrupted an already-boosted task-level RCU reader, and a reader in that handler could then cause false positives. The first failed check (due to an RCU reader that was not immediately deboosted) causes a splat and a list of the segments making up that RCU reader. Subsequent failures fail silently, all in the name of keeping console output down to a dull roar.

Although most uses of RCU priority boosting serve as debugging aids, this might change, and in fact might already have changed. And allowing (for example) RCU priority boosting to persist until the next scheduler tick could cause an aggressively real-time system to miss sub-millisecond deadlines. So we do need to find this sort of problem during testing, and preferably not in the field.

The name and type of the newly added rcu_torture_ops function pointer (named "->is_task_rcu_boosted()") may need to change should other end-of-reader checks be needed. But let's start simple.

---
 kernel/rcu/rcu.h         |  7 +++++++
 kernel/rcu/rcutorture.c  | 22 ++++++++++++++++++++++
 kernel/rcu/tree_plugin.h | 32 +++++++++++++++++++++++++++++++-
 3 files changed, 60 insertions(+), 1 deletion(-)
kernel/rcu/rcu.h +7 -0
index fa6d30ce73d1..14faa11ef23c 100644
@@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re
 void synchronize_rcu_trivial_preempt(void);
 #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU
 
+#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST)
+bool rcu_is_task_rcu_boosted(void);
+#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST)
+static inline bool rcu_is_task_rcu_boosted(void) { return false; }
+#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST)
+
+
 #endif /* __LINUX_RCU_H */
kernel/rcu/rcutorture.c +22 -0
index 42f94c62c18a..04c36fcde5cd 100644
@@ -426,6 +426,7 @@ struct rcu_torture_ops {
 	void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len);
 	void (*set_gpwrap_lag)(unsigned long lag);
 	int (*get_gpwrap_count)(int cpu);
+	bool (*is_task_rcu_boosted)(void);
 	long cbflood_max;
 	int irq_capable;
 	int can_boost;
@@ -635,6 +636,7 @@ static struct rcu_torture_ops rcu_ops = {
 	.format_gp_seqs		= rcutorture_format_gp_seqs,
 	.set_gpwrap_lag		= rcu_set_gpwrap_lag,
 	.get_gpwrap_count	= rcu_get_gpwrap_count,
+	.is_task_rcu_boosted	= rcu_is_task_rcu_boosted,
 	.irq_capable		= 1,
 	.can_boost		= IS_ENABLED(CONFIG_RCU_BOOST),
 	.extendables		= RCUTORTURE_MAX_EXTEND,
@@ -2565,7 +2567,9 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp,
  */
 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 {
+	static int firsttime = 1;
 	int newstate;
+	unsigned int nsegs;
 	struct rcu_torture_one_read_state rtors;
 
 	WARN_ON_ONCE(!rcu_is_watching());
@@ -2577,6 +2581,24 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 		return false;
 	rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp);
 	rcu_torture_one_read_end(&rtors, trsp);
+	// This splat will happen on systems built with CONFIG_IRQ_WORK=n
+	// and on systems where arch_irq_work_has_interrupt() returns false.
+	// It might also happen on systems using a short-duration clock
+	// interrupt instead of a self-IPI (powerpc, s390) or that use
+	// neither a self-IPI nor a short-duration clock interrupts
+	// (all architectures using the generic implementation
+	// of arch_irq_work_raise()).  On such systems, RCU cannot
+	// guarantee to immediately deboost RCU readers when the outermost
+	// rcu_read_unlock() does not end the full segmented RCU read-side
+	// critical section.
+	if (WARN_ON_ONCE(cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() &&
+			 !in_serving_softirq() && !in_hardirq() && !in_nmi()) &&
+	    READ_ONCE(firsttime) && xchg(&firsttime, 0)) {
+		nsegs = rtors.rtrsp - rtors.rtseg;
+		nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS);
+		pr_alert("Slow-deboost rcutorture reader segments:\n");
+		rcu_torture_dump_read_segs(rtors.rtseg, nsegs);
+	}
 	return true;
 }
 
kernel/rcu/tree_plugin.h +31 -1
index fbbabcf2c3a7..d495747e143f 100644
@@ -1093,7 +1093,7 @@ void rcu_read_unlock_strict(void)
 	 *
 	 * The in_atomic_preempt_off() check ensures that we come here holding
 	 * the last preempt_count (which will get dropped once we return to
-	 * __rcu_read_unlock().
+	 * __rcu_read_unlock()).
 	 */
 	rdp = this_cpu_ptr(&rcu_data);
 	rdp->cpu_no_qs.b.norm = false;
@@ -1490,6 +1490,36 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)
 	wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
 }
 
+#ifdef CONFIG_RCU_TORTURE_TEST
+
+/*
+ * Is the current task RCU priority boosted?  This is used by
+ * rcutorture to check that tasks are always deboosted once then exit
+ * an RCU read-side critical section, no matter how many overlapping
+ * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(),
+ * or local_irq_disable() made up that reader.
+ *
+ * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex)
+ * are safe because tasks release ->boost_mtx when they own it, they
+ * cannot be boosted unless current->rcu_blocked_node is non-NULL,
+ * current->rcu_blocked_node is modified only by the current task,
+ * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner
+ * switching among other tasks cannot force an equality comparison.
+ */
+bool rcu_is_task_rcu_boosted(void)
+{
+	struct rcu_node *rnp;
+	struct task_struct *t = current;
+
+	rnp = t->rcu_blocked_node;
+	if (!rnp)
+		return false;
+	return rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t;
+}
+EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted);
+
+#endif // #ifdef CONFIG_RCU_TORTURE_TEST
+
 #else /* #ifdef CONFIG_RCU_BOOST */
 
 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
Signature · 1 line
2.34.1
author maintainer
#3 lore raw

RFC PATCH 3/5 rcutorture: Test RCU readers from hardware interrupt handlers

In reply to Joel Fernandes

From: "Paul E. McKenney" <[email protected]>

Although rcutorture has long had the irqreader module parameter, this parameter results only in RCU readers in softirq handlers, specifically, timers. This commit therefore uses smp_call_function_single() to test RCU readers in real hardware interrupt handlers, thus providing the full effect from the irqreader module parameter.

However, consistency/debug checks must account for the possibility that the smp_call_function_single() handler function is directly invoked from the idle loop, in which case, for example, in_hardirq() will return false. This commit uses a per-CPU variable to record being in the rcu_torture_irq() smp_call_function_single() handler function.

---
 kernel/rcu/rcutorture.c | 75 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 70 insertions(+), 5 deletions(-)
kernel/rcu/rcutorture.c +70 -5
index 04c36fcde5cd..13e0eec1e1ce 100644
@@ -212,6 +212,7 @@ static long n_rcu_torture_boost_ktrerror;
 static long n_rcu_torture_boost_failure;
 static long n_rcu_torture_boosts;
 static atomic_long_t n_rcu_torture_timers;
+static atomic_long_t n_rcu_torture_irqs;
 static long n_barrier_attempts;
 static long n_barrier_successes; /* did rcu_barrier test succeed? */
 static unsigned long n_read_exits;
@@ -2103,6 +2104,8 @@ static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp,
 	smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign.
 }
 
+static DEFINE_PER_CPU(bool, torture_in_scf_handler);
+
 // Verify the specified RCUTORTURE_RDR* state.
 #define ROEC_ARGS "%s %s: Current %#x  To add %#x  To remove %#x  preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count()
 static void rcutorture_one_extend_check(char *s, int curstate, int new, int old)
@@ -2129,7 +2132,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old)
 
 	// Interrupt handlers have all sorts of stuff disabled, so ignore
 	// unintended disabling.
-	if (in_serving_softirq() || in_hardirq())
+	if (in_serving_softirq() || in_hardirq() || this_cpu_read(torture_in_scf_handler))
 		return;
 
 	WARN_ONCE(cur_ops->extendables &&
@@ -2321,12 +2324,19 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp)
 			mask |= RCUTORTURE_RDR_RCU_1;
 	}
 
+	/*
+	 * Don't mess with interrupt masking in interrupt handlers.
+	 */
+	if (in_hardirq() || this_cpu_read(torture_in_scf_handler))
+		mask &= ~(preempts_irq | bhs);
+
 	/*
 	 * Can't enable bh w/irq disabled.
 	 */
 	if (mask & RCUTORTURE_RDR_IRQ)
 		mask |= oldmask & bhs;
 
+
 	/*
 	 * Ideally these sequences would be detected in debug builds
 	 * (regardless of RT), but until then don't stop testing
@@ -2581,6 +2591,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 		return false;
 	rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp);
 	rcu_torture_one_read_end(&rtors, trsp);
+
 	// This splat will happen on systems built with CONFIG_IRQ_WORK=n
 	// and on systems where arch_irq_work_has_interrupt() returns false.
 	// It might also happen on systems using a short-duration clock
@@ -2618,7 +2629,7 @@ static void rcu_torture_timer(struct timer_list *unused)
 	atomic_long_inc(&n_rcu_torture_timers);
 	(void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1);
 
-	/* Test call_rcu() invocation from interrupt handler. */
+	/* Test call_rcu() invocation from softirq handler. */
 	if (cur_ops->call) {
 		struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT);
 
@@ -2627,6 +2638,41 @@ static void rcu_torture_timer(struct timer_list *unused)
 	}
 }
 
+static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_irq_rand);
+
+/*
+ * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
+ * incrementing the corresponding element of the pipeline array.  The
+ * counter in the element should never be greater than 1, otherwise, the
+ * RCU implementation is broken.
+ *
+ * Note that on some systems, "interrupts" from idle are direct calls
+ * rather than interrupts.  The torture_in_scf_handler per-CPU variable
+ * accounts for this case.
+ */
+static void rcu_torture_irq(void *unused)
+{
+	WARN_ON_ONCE(in_nmi());
+	lockdep_assert_irqs_disabled();
+	atomic_long_inc(&n_rcu_torture_irqs);
+	this_cpu_write(torture_in_scf_handler, true);
+	(void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_irq_rand), -1);
+	this_cpu_write(torture_in_scf_handler, false);
+
+	// Test call_rcu() invocation from interrupt handler.  Interrupts
+	// will always be disabled here, even in CONFIG_PREEMPT_RT=y kernels.
+	// The "right" thing to do would be to create a special-purpose
+	// lockless or raw-spinlock-protected allocator, but in the meantime,
+	// skip testing call_rcu() from interrupt handlers in kernels built
+	// with either CONFIG_PREEMPT_RT=y or CONFIG_PROVE_LOCKING=y.
+	if (cur_ops->call && !IS_ENABLED(CONFIG_PROVE_LOCKING) && !IS_ENABLED(CONFIG_PREEMPT_RT)) {
+		struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT);
+
+		if (rhp)
+			cur_ops->call(rhp, rcu_torture_timer_cb);
+	}
+}
+
 /*
  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
  * incrementing the corresponding element of the pipeline array.  The
@@ -2636,6 +2682,7 @@ static void rcu_torture_timer(struct timer_list *unused)
 static int
 rcu_torture_reader(void *arg)
 {
+	unsigned long lastscf = jiffies;
 	unsigned long lastsleep = jiffies;
 	long myid = (long)arg;
 	int mynumonline = myid;
@@ -2649,8 +2696,25 @@ rcu_torture_reader(void *arg)
 	tick_dep_set_task(current, TICK_DEP_BIT_RCU);  // CPU bound, so need tick.
 	do {
 		if (irqreader && cur_ops->irq_capable) {
-			if (!timer_pending(&t))
+			if (!timer_pending(&t)) {
+				int cpu;
+
 				mod_timer(&t, jiffies + 1);
+				preempt_disable();
+				cpu = torture_random(&rand) % nr_cpu_ids;
+				if (!cpu_online(cpu)) {
+					cpu = cpumask_next(cpu, cpu_online_mask);
+					if (cpu >= nr_cpu_ids)
+						cpu = cpumask_next(-1, cpu_online_mask);
+				}
+				// An smp_call_function_single() to self is not an interrupt!
+				if (cpu != smp_processor_id() &&
+				    time_after(jiffies, lastscf + HZ * nrealreaders / 50)) {
+					smp_call_function_single(cpu, rcu_torture_irq, NULL, 0);
+					lastscf = jiffies;
+				}
+				preempt_enable();
+			}
 		}
 		if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop())
 			schedule_timeout_interruptible(HZ);
@@ -2926,10 +2990,11 @@ rcu_torture_stats_print(void)
 		atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries),
 		n_rcu_torture_barrier_error,
 		n_rcu_torture_boost_ktrerror);
-	pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
+	pr_cont("rtbf: %ld rtb: %ld nt: %ld ni: %ld ",
 		n_rcu_torture_boost_failure,
 		n_rcu_torture_boosts,
-		atomic_long_read(&n_rcu_torture_timers));
+		atomic_long_read(&n_rcu_torture_timers),
+		atomic_long_read(&n_rcu_torture_irqs));
 	if (updownreaders)
 		pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired,  nmigrates);
 	torture_onoff_stats();
Signature · 1 line
2.34.1
author maintainer
#4 lore raw

RFC PATCH 4/5 rcutorture: tighten boost-WARN to exclude any implicit-reader context

In reply to Joel Fernandes

The Slow-deboost WARN_ON_ONCE in rcu_torture_one_read() currently uses only the handler-context predicates to decide whether the calling context is "safe" to assert immediate deboosting:

    !in_serving_softirq() && !in_hardirq() && !in_nmi()

They do NOT exclude several other contexts in which the task is still holding an implicit RCU read-side critical section.

1. local_bh_disable()-but-not-in-softirq.

2. preempt_disable()-held.

3. irqs_disabled().

4. rcu_preempt_depth() > 0.

The current predicate set lets the WARN fire while these implicit readers are still active, which is a false positive: the user-visible "tardy deboost" finding becomes ambiguous between an actual deferred-QS bug and a valid caller still inside an implicit-reader section.

Replace the three handler-only predicates with the following which covers everything.

    preempt_count() == 0 && !irqs_disabled() && rcu_preempt_depth() == 0

Note that we only check in contexts where rcu_read_unlock_special() deboosts synchronously, i.e. where it is not inside an implicit RCU read-side critical section. This matches rcu_read_unlock_special()'s own gate and rcu_flavor_sched_clock_irq()'s:
"rcu_preempt_depth() > 0 || (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))".

Also note, in PREEMPT_RT, softirqs always run under local_bh_disable(), which itself takes rcu_read_lock() to satisfy RCU's bottom-half requirement (see softirq.c) — so rcu_preempt_depth() is non-zero throughout any RT softirq thus making in_serving_softirq() redundant.

---
 kernel/rcu/rcutorture.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/rcu/rcutorture.c +2 -1
index 13e0eec1e1ce..e0bea4039c42 100644
@@ -2603,7 +2603,8 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 	// rcu_read_unlock() does not end the full segmented RCU read-side
 	// critical section.
 	if (WARN_ON_ONCE(cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() &&
-			 !in_serving_softirq() && !in_hardirq() && !in_nmi()) &&
+			 preempt_count() == 0 && !irqs_disabled() &&
+			 rcu_preempt_depth() == 0) &&
 	    READ_ONCE(firsttime) && xchg(&firsttime, 0)) {
 		nsegs = rtors.rtrsp - rtors.rtseg;
 		nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS);
Signature · 1 line
2.34.1
author maintainer
#5 lore raw

RFC PATCH 5/5 rcutorture: give async deboost mechanisms up to 500us before WARN

In reply to Joel Fernandes

The Slow-deboost detector in rcu_torture_one_read() checks for a still- boosted task immediately after the outer rcu_read_unlock() returns. Sub-microsecond checking is unrealistic for any async deboost mechanism (irq_work, softirq dispatch). As, even with a delay in deboosting of just 50us, the check beats it.

Therefore, poll the boost state for up to 500us before declaring the deboost tardy.

---
 kernel/rcu/rcutorture.c | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)
kernel/rcu/rcutorture.c +20 -9
index e0bea4039c42..12813b5381d6 100644
@@ -2601,15 +2601,26 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 	// of arch_irq_work_raise()).  On such systems, RCU cannot
 	// guarantee to immediately deboost RCU readers when the outermost
 	// rcu_read_unlock() does not end the full segmented RCU read-side
-	// critical section.
-	if (WARN_ON_ONCE(cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() &&
-			 preempt_count() == 0 && !irqs_disabled() &&
-			 rcu_preempt_depth() == 0) &&
-	    READ_ONCE(firsttime) && xchg(&firsttime, 0)) {
-		nsegs = rtors.rtrsp - rtors.rtseg;
-		nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS);
-		pr_alert("Slow-deboost rcutorture reader segments:\n");
-		rcu_torture_dump_read_segs(rtors.rtseg, nsegs);
+	// critical section.  Allow up to 500us for an async deboost
+	// mechanism (e.g. irq work, timers etc) to fire before
+	// declaring the deboost tardy.
+	if (cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() &&
+	    preempt_count() == 0 && !irqs_disabled() &&
+	    rcu_preempt_depth() == 0) {
+		int wait_us;
+
+		for (wait_us = 0; wait_us < 500; wait_us += 10) {
+			udelay(10);
+			if (!cur_ops->is_task_rcu_boosted())
+				break;
+		}
+		if (WARN_ON_ONCE(cur_ops->is_task_rcu_boosted()) &&
+		    READ_ONCE(firsttime) && xchg(&firsttime, 0)) {
+			nsegs = rtors.rtrsp - rtors.rtseg;
+			nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS);
+			pr_alert("Slow-deboost rcutorture reader segments:\n");
+			rcu_torture_dump_read_segs(rtors.rtseg, nsegs);
+		}
 	}
 	return true;
 }
Signature · 1 line
2.34.1
maintainer
#6 lore raw
In reply to Joel Fernandes
Quoted 5 lines · Joel Fernandes

This series adds rcutorture coverage for the RCU priority-boosting "tardy deboost" issue. The first 3 patches are Paul's and the next 2 are fixes I had to make to the test, so that another patch series (I haven't posted yet) can correctly be tested. I am posting just the test patches first in advance, which can go in independently if we agree on the changes.

OK, I have recently gotten distracted with hazptrtorture, and am likely to be more distracted with call_rcu*_nolock().

So if you are ready, then you should go first.

Quoted 3 lines

Joel Fernandes (2):
rcutorture: tighten boost-WARN to exclude any implicit-reader context
rcutorture: give async deboost mechanisms up to 500us before WARN

You need these, but once I get the immediate-deboosting going, they should become optional. But that is OK, I can adjust later.

Thanx, Paul

Quoted 14 lines

Paul E. McKenney (3):
rcutorture: Abstract reader-segment dump into
rcu_torture_dump_read_segs()
rcutorture: Check for immediate deboosting at reader end
rcutorture: Test RCU readers from hardware interrupt handlers

 kernel/rcu/rcu.h         |   7 ++
 kernel/rcu/rcutorture.c  | 257 +++++++++++++++++++++++++++------------
 kernel/rcu/tree_plugin.h |  32 ++++-
 3 files changed, 218 insertions(+), 78 deletions(-)
Signature · 1 line
2.34.1