Skip to contentSkip to conversation

3.18.7-rt2: NOHZ_FULL warning at boot

DISCUSSION Luis Claudio R. Goncalves 3 participants 4 messages Feb 26, 2015 – Mar 6, 2015

lore
Share & tools
lore
#0 lore raw

Hello,

I have been seeing the warning below at boot time on 3.18.7-rt2. Anybody else seeing it?

[    1.537207] ------------[ cut here ]------------
[    1.537215] WARNING: CPU: 1 PID: 20 at /home/lclaudio/SANDBOX/kernel-rt/kernel/time/tick-sched.c:167 can_stop_full_tick+0x1ee/0x2b0()
[    1.537219] Modules linked in: efivarfs
[    1.537224] CPU: 1 PID: 20 Comm: ksoftirqd/1 Not tainted 3.18.7-rt2+ #2
[    1.537225] Hardware name: Hewlett-Packard p7-1512/2ADA, BIOS 8.15 02/05/2013
[    1.537227]  0000000000000000 000000005b4bc999 ffff880191503bb8 ffffffff8175a165
[    1.537228]  0000000000000000 0000000000000000 ffff880191503bf8 ffffffff8107add1
[    1.537229]  ffff880191503be8 ffff880198fce280 ffff880198e00000 0000000000000000
[    1.537229] Call Trace:
[    1.537233]  [<ffffffff8175a165>] dump_stack+0x4f/0x7c
[    1.537235]  [<ffffffff8107add1>] warn_slowpath_common+0x81/0xc0
[    1.537237]  [<ffffffff8107af0a>] warn_slowpath_null+0x1a/0x20
[    1.537238]  [<ffffffff811074fe>] can_stop_full_tick+0x1ee/0x2b0
[    1.537240]  [<ffffffff81107656>] __tick_nohz_full_check+0x76/0xc0
[    1.537241]  [<ffffffff811076ae>] nohz_full_kick_work_func+0xe/0x10
[    1.537243]  [<ffffffff8116ff7c>] irq_work_run_list+0x3c/0x60
[    1.537244]  [<ffffffff8117037e>] irq_work_tick+0x3e/0x90
[    1.537246]  [<ffffffff810f25c4>] run_timer_softirq+0x44/0x3b0
[    1.537248]  [<ffffffff81760855>] ? _raw_spin_unlock+0x35/0x60
[    1.537250]  [<ffffffff810b0b89>] ? vtime_account_system+0x99/0xb0
[    1.537252]  [<ffffffff810b0d9b>] ? vtime_common_account_irq_enter+0x4b/0x80
[    1.537254]  [<ffffffff8107fbd4>] do_current_softirqs.isra.11+0x274/0x550
[    1.537255]  [<ffffffff8107ff86>] run_ksoftirqd+0x26/0x50
[    1.537257]  [<ffffffff810a1a0d>] smpboot_thread_fn+0x18d/0x2d0
[    1.537259]  [<ffffffff810a1880>] ? SyS_setgroups+0x150/0x150
[    1.537260]  [<ffffffff8109d68d>] kthread+0xfd/0x120
[    1.537262]  [<ffffffff8175e177>] ? wait_for_completion+0xe7/0x120
[    1.537264]  [<ffffffff8109d590>] ? kthread_create_on_node+0x260/0x260
[    1.537266]  [<ffffffff817618ac>] ret_from_fork+0x7c/0xb0
[    1.537267]  [<ffffffff8109d590>] ? kthread_create_on_node+0x260/0x260
[    1.537268] ---[ end trace 0000000000000002 ]---
Quoted 1 line

From kernel/time/tick-sched.c line 167:

static bool can_stop_full_tick(void)

{
        WARN_ON_ONCE(!irqs_disabled());

The config in use, NO_HZ related is:

	CONFIG_NO_HZ_COMMON=y
	# CONFIG_NO_HZ_IDLE is not set
		CONFIG_NO_HZ_FULL=y
	CONFIG_NO_HZ_FULL_ALL=y
	CONFIG_NO_HZ_FULL_SYSIDLE=y
	CONFIG_NO_HZ_FULL_SYSIDLE_SMALL=8
	CONFIG_NO_HZ=y
	CONFIG_HZ_1000=y
	CONFIG_HZ=1000

Cheers,
Luis

Signature · 2 lines
[ Luis Claudio R. Goncalves                    Bass - Gospel - RT ]
[ Fingerprint: 4FDD B8C4 3C59 34BD 8BE9  2696 7203 D980 A448 C8F8 ]
#1 lore raw
Quoted 1 line · * Luis Claudio R. Goncalves | 2015-02-26 17:05:47 [-0300]:
[    1.537215] WARNING: CPU: 1 PID: 20 at /home/lclaudio/SANDBOX/kernel-rt/kernel/time/tick-sched.c:167 can_stop_full_tick+0x1ee/0x2b0()

I haven't run full-dynticks for some time. I might have broken that, I take a look at that.

Sebastian

#2 lore raw
In reply to Luis Claudio R. Goncalves
Quoted 2 lines · * Luis Claudio R. Goncalves | 2015-02-26 17:05:47 [-0300]:

I have been seeing the warning below at boot time on 3.18.7-rt2. Anybody else seeing it?

This should fix it.

kernel/irq_work.c +24 -13
index 19c363a..0c64912 100644
@@ -71,6 +71,8 @@ void __weak arch_irq_work_raise(void)
  */
 bool irq_work_queue_on(struct irq_work *work, int cpu)
 {
+	bool raise_irqwork;
+
 	/* All work should have been flushed before going offline */
 	WARN_ON_ONCE(cpu_is_offline(cpu));
 
@@ -81,7 +83,19 @@ bool irq_work_queue_on(struct irq_work *work, int cpu)
 	if (!irq_work_claim(work))
 		return false;
 
-	if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
+#ifdef CONFIG_PREEMPT_RT_FULL
+	if (work->flags & IRQ_WORK_HARD_IRQ)
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(hirq_work_list, cpu));
+	else
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(lazy_list, cpu));
+#else
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(raised_list, cpu));
+#endif
+
+	if (raise_irqwork)
 		arch_send_call_function_single_ipi(cpu);
 
 	return true;
@@ -101,19 +115,14 @@ bool irq_work_queue(struct irq_work *work)
 
 #ifdef CONFIG_PREEMPT_RT_FULL
 	if (work->flags & IRQ_WORK_HARD_IRQ) {
-		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list))) {
-			if (work->flags & IRQ_WORK_LAZY) {
-				if (tick_nohz_tick_stopped())
-					arch_irq_work_raise();
-			} else {
-				arch_irq_work_raise();
-			}
-		}
-	/* If the work is "lazy", handle it from next tick if any */
-	} else if (work->flags & IRQ_WORK_LAZY) {
+		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list)))
+			arch_irq_work_raise();
+	} else {
+		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)))
+			arch_irq_work_raise();
+	}
 #else
 	if (work->flags & IRQ_WORK_LAZY) {
-#endif
 		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
 		    tick_nohz_tick_stopped())
 			arch_irq_work_raise();
@@ -121,6 +130,7 @@ bool irq_work_queue(struct irq_work *work)
 		if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
 			arch_irq_work_raise();
 	}
+#endif
 
 	preempt_enable();
 
@@ -137,7 +147,8 @@ bool irq_work_needs_cpu(void)
 
 	if (llist_empty(raised))
 		if (llist_empty(lazy))
-			return false;
+			if (llist_empty(this_cpu_ptr(&hirq_work_list)))
+				return false;
 
 	/* All work should have been flushed before going offline */
 	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

Sebastian

#3 lore raw
Quoted 76 lines · Sebastian Andrzej Siewior
Quoted 2 lines · * Luis Claudio R. Goncalves | 2015-02-26 17:05:47 [-0300]:

I have been seeing the warning below at boot time on 3.18.7-rt2. Anybody else seeing it?

This should fix it.

kernel/irq_work.c +22 -12
index 19c363a..0c64912 100644
@@ -71,6 +71,8 @@ void __weak arch_irq_work_raise(void)
   */
  bool irq_work_queue_on(struct irq_work *work, int cpu)
  {
+	bool raise_irqwork;
+
  	/* All work should have been flushed before going offline */
  	WARN_ON_ONCE(cpu_is_offline(cpu));
 
@@ -81,7 +83,19 @@ bool irq_work_queue_on(struct irq_work *work, int cpu)
  	if (!irq_work_claim(work))
  		return false;
 
-	if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
+#ifdef CONFIG_PREEMPT_RT_FULL
+	if (work->flags & IRQ_WORK_HARD_IRQ)
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(hirq_work_list, cpu));
+	else
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(lazy_list, cpu));
+#else
+		raise_irqwork = llist_add(&work->llnode,
+					  &per_cpu(raised_list, cpu));
+#endif
+
+	if (raise_irqwork)
  		arch_send_call_function_single_ipi(cpu);
 
  	return true;
@@ -101,19 +115,14 @@ bool irq_work_queue(struct irq_work *work)
 
  #ifdef CONFIG_PREEMPT_RT_FULL
  	if (work->flags & IRQ_WORK_HARD_IRQ) {
-		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list))) {
-			if (work->flags & IRQ_WORK_LAZY) {
-				if (tick_nohz_tick_stopped())
-					arch_irq_work_raise();
-			} else {
-				arch_irq_work_raise();
-			}
-		}
-	/* If the work is "lazy", handle it from next tick if any */
-	} else if (work->flags & IRQ_WORK_LAZY) {
+		if (llist_add(&work->llnode, this_cpu_ptr(&hirq_work_list)))
+			arch_irq_work_raise();
+	} else {
+		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)))
+			arch_irq_work_raise();
+	}
  #else
  	if (work->flags & IRQ_WORK_LAZY) {
-#endif
  		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
  		    tick_nohz_tick_stopped())
  			arch_irq_work_raise();
@@ -121,6 +130,7 @@ bool irq_work_queue(struct irq_work *work)
  		if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
  			arch_irq_work_raise();
  	}
+#endif
 
  	preempt_enable();
 

kernel/irq_work.c bool irq_work_needs_cpu(void) · line 154

Quoted 9 lines
kernel/irq_work.c +2 -1
@@ -137,7 +147,8 @@ bool irq_work_needs_cpu(void)
 
  	if (llist_empty(raised))
  		if (llist_empty(lazy))
-			return false;
+			if (llist_empty(this_cpu_ptr(&hirq_work_list)))
+				return false;
 
  	/* All work should have been flushed before going offline */
  	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));

I can confirm that the warning is no longer issued when this patch is applied.

Thanks,
-Carsten.