Skip to contentSkip to conversation

kernel/timer: avoid spurious ksoftirqd wakeups

PATCH Marcelo Tosatti 3 participants 9 messages Apr 2, 2015 – Apr 7, 2015

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

It is only necessary to raise timer softirq
in case there are active timers or irq work
to do.

Limit the ksoftirqd wakeup to those cases.

Fixes a latency spike with isolated CPUs and
nohz full mode.

include/linux/timer.h +1 -1
index 8c5a197..0c065f9 100644
@@ -192,7 +192,7 @@ extern void set_timer_slack(struct timer_list *time, int slack_hz);
  * locks the timer base and does the comparison against the given
  * jiffie.
  */
-extern unsigned long get_next_timer_interrupt(unsigned long now);
+extern unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq);
 
 /*
  * Timer-statistics info:
kernel/time/tick-sched.c +6 -2
index a4c4eda..615e276 100644
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;
 
 	time_delta = timekeeping_max_deferment();
 
@@ -582,9 +583,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	    arch_needs_cpu() || irq_work_needs_cpu()) {
 		next_jiffies = last_jiffies + 1;
 		delta_jiffies = 1;
+		raise_softirq = true;
 	} else {
 		/* Get the next timer wheel timer */
-		next_jiffies = get_next_timer_interrupt(last_jiffies);
+		next_jiffies = get_next_timer_interrupt(last_jiffies,
+							&raise_softirq);
 		delta_jiffies = next_jiffies - last_jiffies;
 		if (rcu_delta_jiffies < delta_jiffies) {
 			next_jiffies = last_jiffies + rcu_delta_jiffies;
@@ -703,7 +706,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 		 */
 		tick_do_update_jiffies64(ktime_get());
 	}
-	raise_softirq_irqoff(TIMER_SOFTIRQ);
+	if (raise_softirq)
+		raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
 	ts->next_jiffies = next_jiffies;
 	ts->last_jiffies = last_jiffies;
kernel/time/timer.c +2 -1
index 2d3f5c5..771f811 100644
@@ -1343,7 +1343,7 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now,
  * get_next_timer_interrupt - return the jiffy of the next pending timer
  * @now: current time (in jiffies)
  */
-unsigned long get_next_timer_interrupt(unsigned long now)
+unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq)
 {
 	struct tvec_base *base = __this_cpu_read(tvec_bases);
 	unsigned long expires = now + NEXT_TIMER_MAX_DELTA;
@@ -1357,6 +1357,7 @@ unsigned long get_next_timer_interrupt(unsigned long now)
 
 	spin_lock(&base->lock);
 	if (base->active_timers) {
+		*raise_softirq = true;
 		if (time_before_eq(base->next_timer, base->timer_jiffies))
 			base->next_timer = __next_timer_interrupt(base);
 		expires = base->next_timer;
#1 lore raw

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 571

Quoted 6 lines · Marcelo Tosatti

+++ b/kernel/time/tick-sched.c

kernel/time/tick-sched.c +1 -0
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;

You may want to initialize this to false. Nothing else in the code ever seems to set it to false.

It may work in your test due to that address on the stack already being zeroed out due to a lucky coincidence, but that is not a guarantee.

Quoted 12 lines
kernel/time/timer.c +1 -1
@@ -1343,7 +1343,7 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now,
  * get_next_timer_interrupt - return the jiffy of the next pending timer
  * @now: current time (in jiffies)
  */
-unsigned long get_next_timer_interrupt(unsigned long now)
+unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq)
 {
 	struct tvec_base *base = __this_cpu_read(tvec_bases);
 	unsigned long expires = now + NEXT_TIMER_MAX_DELTA;

kernel/time/timer.c unsigned long get_next_timer_interrupt(unsigned long now) · line 1363

Quoted 7 lines
kernel/time/timer.c +1 -0
@@ -1357,6 +1357,7 @@ unsigned long get_next_timer_interrupt(unsigned long now)
 
 	spin_lock(&base->lock);
 	if (base->active_timers) {
+		*raise_softirq = true;
 		if (time_before_eq(base->next_timer, base->timer_jiffies))
 			base->next_timer = __next_timer_interrupt(base);
 		expires = base->next_timer;

Given that run_timer_softirq() only actually does something if the timer has expired, would it make sense to only raise the softirq after the timer has expired?

#3 lore raw
In reply to Marcelo Tosatti
Quoted 39 lines · Marcelo Tosatti

It is only necessary to raise timer softirq
in case there are active timers or irq work
to do.

Limit the ksoftirqd wakeup to those cases.

Fixes a latency spike with isolated CPUs and
nohz full mode.

include/linux/timer.h +1 -1
index 8c5a197..0c065f9 100644
@@ -192,7 +192,7 @@ extern void set_timer_slack(struct timer_list *time, int slack_hz);
  * locks the timer base and does the comparison against the given
  * jiffie.
  */
-extern unsigned long get_next_timer_interrupt(unsigned long now);
+extern unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq);
 
 /*
  * Timer-statistics info:
kernel/time/tick-sched.c +1 -0
index a4c4eda..615e276 100644
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;
 
 	time_delta = timekeeping_max_deferment();
 

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 586

Quoted 4 lines
kernel/time/tick-sched.c +1 -0
@@ -582,9 +583,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	    arch_needs_cpu() || irq_work_needs_cpu()) {
 		next_jiffies = last_jiffies + 1;
 		delta_jiffies = 1;
+		raise_softirq = true;

I believe that irq_work doesn't need the softirq. It needs a tick only in order to call irq_work_tick(). And I believe this is the same for RCU which needs a call to rcu_check_callbacks(), but it might need something else that the softirq does (but this is the timer softirq, not the rcu one).

Quoted 9 lines
kernel/time/tick-sched.c +2 -1
 	} else {
 		/* Get the next timer wheel timer */
-		next_jiffies = get_next_timer_interrupt(last_jiffies);
+		next_jiffies = get_next_timer_interrupt(last_jiffies,
+							&raise_softirq);
 		delta_jiffies = next_jiffies - last_jiffies;
 		if (rcu_delta_jiffies < delta_jiffies) {
 			next_jiffies = last_jiffies + rcu_delta_jiffies;

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 713

Quoted 9 lines
kernel/time/tick-sched.c +2 -1
@@ -703,7 +706,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 		 */
 		tick_do_update_jiffies64(ktime_get());
 	}
-	raise_softirq_irqoff(TIMER_SOFTIRQ);
+	if (raise_softirq)
+		raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
 	ts->next_jiffies = next_jiffies;
 	ts->last_jiffies = last_jiffies;

Lets look at the things outside the pending timer list that can end up failing to program the timer because it is in the past already:

_ timekeeping_max_deferment(): I doubt, the value is pretty high
_ scheduler_tick_max_deferment(); it's one second long, way enough to never be in
the past by the time we program the clock
_ RCU, irq_work, arch: may be, if the last jiffies update is far enough. But apparently
the problem is elsewhere since you keep the softirq for these and your patch solves your problem.
_ In case hrtimer runs in low-res mode and the next hrtimer is very close, or even in the past
already, you may run into such issue. And hrtimer doesn't need the timer softirq, at least not to run the callbacks. It needs it only if hrtimer is switching to high-res mode, I think it's a rare event.

Now it would be nice to identify the issue we are facing here. Are you running in hrtimer low-res mode?

#2 lore raw
In reply to Rik van Riel
Quoted 29 lines · Rik van Riel

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 571

Quoted 6 lines · Marcelo Tosatti

+++ b/kernel/time/tick-sched.c

kernel/time/tick-sched.c +1 -0
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;

You may want to initialize this to false. Nothing else in the code ever seems to set it to false.

It may work in your test due to that address on the stack already being zeroed out due to a lucky coincidence, but that is not a guarantee.

Quoted 13 lines
kernel/time/timer.c +1 -1
@@ -1343,7 +1343,7 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now,
  * get_next_timer_interrupt - return the jiffy of the next pending timer
  * @now: current time (in jiffies)
  */
-unsigned long get_next_timer_interrupt(unsigned long now)
+unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq)
 {
 	struct tvec_base *base = __this_cpu_read(tvec_bases);
 	unsigned long expires = now + NEXT_TIMER_MAX_DELTA;
@@ -1357,6 +1357,7 @@ unsigned long get_next_timer_interrupt(unsigned long now)
 
Quoted 10 lines

kernel/time/timer.c unsigned long get_next_timer_interrupt(unsigned long now) · line 1363

Quoted 6 lines
kernel/time/timer.c +1 -0
 	spin_lock(&base->lock);
 	if (base->active_timers) {
+		*raise_softirq = true;
 		if (time_before_eq(base->next_timer, base->timer_jiffies))
 			base->next_timer = __next_timer_interrupt(base);
 		expires = base->next_timer;

Given that run_timer_softirq() only actually does something if the timer has expired, would it make sense to only raise the softirq after the timer has expired?

jiffies might be increased by tick_nohz_stop_sched_tick.

So you'd have to test again after increasing jiffies.

#4 lore raw
In reply to Frederic Weisbecker
Quoted 39 lines · Frederic Weisbecker
Quoted 38 lines · Marcelo Tosatti

It is only necessary to raise timer softirq
in case there are active timers or irq work
to do.

Limit the ksoftirqd wakeup to those cases.

Fixes a latency spike with isolated CPUs and
nohz full mode.

include/linux/timer.h +1 -1
index 8c5a197..0c065f9 100644
@@ -192,7 +192,7 @@ extern void set_timer_slack(struct timer_list *time, int slack_hz);
  * locks the timer base and does the comparison against the given
  * jiffie.
  */
-extern unsigned long get_next_timer_interrupt(unsigned long now);
+extern unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq);
 
 /*
  * Timer-statistics info:
kernel/time/tick-sched.c +1 -0
index a4c4eda..615e276 100644
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;
 
 	time_delta = timekeeping_max_deferment();
 
Quoted 7 lines

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 586

Quoted 5 lines
kernel/time/tick-sched.c +1 -0
@@ -582,9 +583,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	    arch_needs_cpu() || irq_work_needs_cpu()) {
 		next_jiffies = last_jiffies + 1;
 		delta_jiffies = 1;
+		raise_softirq = true;

I believe that irq_work doesn't need the softirq.

Can drop that, right.

Quoted 18 lines

It needs a tick only in order to call
irq_work_tick(). And I believe this is the same for RCU which needs a call to rcu_check_callbacks(), but it might need something else that the softirq does (but this is the timer softirq, not the rcu one).

Quoted 13 lines
kernel/time/tick-sched.c +2 -2
 	} else {
 		/* Get the next timer wheel timer */
-		next_jiffies = get_next_timer_interrupt(last_jiffies);
+		next_jiffies = get_next_timer_interrupt(last_jiffies,
+							&raise_softirq);
 		delta_jiffies = next_jiffies - last_jiffies;
 		if (rcu_delta_jiffies < delta_jiffies) {
 			next_jiffies = last_jiffies + rcu_delta_jiffies;
@@ -703,7 +706,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 		 */
 		tick_do_update_jiffies64(ktime_get());
 	}
-	raise_softirq_irqoff(TIMER_SOFTIRQ);
Quoted 8 lines

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 713

Quoted 5 lines
kernel/time/tick-sched.c +2 -0
+	if (raise_softirq)
+		raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
 	ts->next_jiffies = next_jiffies;
 	ts->last_jiffies = last_jiffies;

Lets look at the things outside the pending timer list that can end up failing to program the timer because it is in the past already:

Is this an attempt to find possible regressions introduced by this change ?

Quoted 11 lines

_ timekeeping_max_deferment(): I doubt, the value is pretty high
_ scheduler_tick_max_deferment(); it's one second long, way enough to never be in
the past by the time we program the clock
_ RCU, irq_work, arch: may be, if the last jiffies update is far enough. But apparently
the problem is elsewhere since you keep the softirq for these and your patch solves your problem.
_ In case hrtimer runs in low-res mode and the next hrtimer is very close, or even in the past
already, you may run into such issue. And hrtimer doesn't need the timer softirq, at least not to run the callbacks. It needs it only if hrtimer is switching to high-res mode, I think it's a rare event.

Quoted 2 lines

Now it would be nice to identify the issue we are facing here. Are you running in hrtimer low-res mode?

The issue is a latency spike due to ksoftirqd waking up to process pending timers, processing two deferred timers, but no non-deferred timers.

hrtimer is not in low-res mode.

The issue is ksoftirqd waking up in the first place.

#5 lore raw
Quoted 77 lines · Marcelo Tosatti
Quoted 39 lines · Frederic Weisbecker
Quoted 38 lines · Marcelo Tosatti

It is only necessary to raise timer softirq
in case there are active timers or irq work
to do.

Limit the ksoftirqd wakeup to those cases.

Fixes a latency spike with isolated CPUs and
nohz full mode.

include/linux/timer.h +1 -1
index 8c5a197..0c065f9 100644
@@ -192,7 +192,7 @@ extern void set_timer_slack(struct timer_list *time, int slack_hz);
  * locks the timer base and does the comparison against the given
  * jiffie.
  */
-extern unsigned long get_next_timer_interrupt(unsigned long now);
+extern unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq);
 
 /*
  * Timer-statistics info:
kernel/time/tick-sched.c +1 -0
index a4c4eda..615e276 100644
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;
 
 	time_delta = timekeeping_max_deferment();
 
Quoted 7 lines

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 586

Quoted 5 lines
kernel/time/tick-sched.c +1 -0
@@ -582,9 +583,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	    arch_needs_cpu() || irq_work_needs_cpu()) {
 		next_jiffies = last_jiffies + 1;
 		delta_jiffies = 1;
+		raise_softirq = true;

I believe that irq_work doesn't need the softirq.

Can drop that, right.

Quoted 18 lines

It needs a tick only in order to call
irq_work_tick(). And I believe this is the same for RCU which needs a call to rcu_check_callbacks(), but it might need something else that the softirq does (but this is the timer softirq, not the rcu one).

Quoted 13 lines
kernel/time/tick-sched.c +2 -2
 	} else {
 		/* Get the next timer wheel timer */
-		next_jiffies = get_next_timer_interrupt(last_jiffies);
+		next_jiffies = get_next_timer_interrupt(last_jiffies,
+							&raise_softirq);
 		delta_jiffies = next_jiffies - last_jiffies;
 		if (rcu_delta_jiffies < delta_jiffies) {
 			next_jiffies = last_jiffies + rcu_delta_jiffies;
@@ -703,7 +706,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 		 */
 		tick_do_update_jiffies64(ktime_get());
 	}
-	raise_softirq_irqoff(TIMER_SOFTIRQ);
Quoted 8 lines

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 713

Quoted 5 lines
kernel/time/tick-sched.c +2 -0
+	if (raise_softirq)
+		raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
 	ts->next_jiffies = next_jiffies;
 	ts->last_jiffies = last_jiffies;

Lets look at the things outside the pending timer list that can end up failing to program the timer because it is in the past already:

Quoted 2 lines

Is this an attempt to find possible regressions introduced by this change ?

Yeah, it would be nice to make sure that the cause of these softirqs isn't mistakenly ignored. And also I want to be sure we really understand what we are doing, which is not the case right now as we don't know what is causing this expired timer.

Quoted 15 lines
Quoted 12 lines

_ timekeeping_max_deferment(): I doubt, the value is pretty high
_ scheduler_tick_max_deferment(); it's one second long, way enough to never be in
the past by the time we program the clock
_ RCU, irq_work, arch: may be, if the last jiffies update is far enough. But apparently
the problem is elsewhere since you keep the softirq for these and your patch solves your problem.
_ In case hrtimer runs in low-res mode and the next hrtimer is very close, or even in the past
already, you may run into such issue. And hrtimer doesn't need the timer softirq, at least not to run the callbacks. It needs it only if hrtimer is switching to high-res mode, I think it's a rare event.

Quoted 2 lines

Now it would be nice to identify the issue we are facing here. Are you running in hrtimer low-res mode?

Quoted 7 lines

The issue is a latency spike due to ksoftirqd waking up to process pending timers, processing two deferred timers, but no non-deferred timers.

hrtimer is not in low-res mode.

The issue is ksoftirqd waking up in the first place.

Sure, but why is it waking up exactly?

#6 lore raw
Quoted 78 lines · Frederic Weisbecker
Quoted 77 lines · Marcelo Tosatti
Quoted 39 lines · Frederic Weisbecker
Quoted 38 lines · Marcelo Tosatti

It is only necessary to raise timer softirq
in case there are active timers or irq work
to do.

Limit the ksoftirqd wakeup to those cases.

Fixes a latency spike with isolated CPUs and
nohz full mode.

include/linux/timer.h +1 -1
index 8c5a197..0c065f9 100644
@@ -192,7 +192,7 @@ extern void set_timer_slack(struct timer_list *time, int slack_hz);
  * locks the timer base and does the comparison against the given
  * jiffie.
  */
-extern unsigned long get_next_timer_interrupt(unsigned long now);
+extern unsigned long get_next_timer_interrupt(unsigned long now, bool *raise_softirq);
 
 /*
  * Timer-statistics info:
kernel/time/tick-sched.c +1 -0
index a4c4eda..615e276 100644
@@ -568,6 +568,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	unsigned long rcu_delta_jiffies;
 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
 	u64 time_delta;
+	bool raise_softirq;
 
 	time_delta = timekeeping_max_deferment();
 
Quoted 7 lines

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 586

Quoted 5 lines
kernel/time/tick-sched.c +1 -0
@@ -582,9 +583,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 	    arch_needs_cpu() || irq_work_needs_cpu()) {
 		next_jiffies = last_jiffies + 1;
 		delta_jiffies = 1;
+		raise_softirq = true;

I believe that irq_work doesn't need the softirq.

Can drop that, right.

Quoted 26 lines

It needs a tick only in order to call
irq_work_tick(). And I believe this is the same for RCU which needs a call to rcu_check_callbacks(), but it might need something else that the softirq does (but this is the timer softirq, not the rcu one).

Quoted 9 lines
kernel/time/tick-sched.c +2 -1
 	} else {
 		/* Get the next timer wheel timer */
-		next_jiffies = get_next_timer_interrupt(last_jiffies);
+		next_jiffies = get_next_timer_interrupt(last_jiffies,
+							&raise_softirq);
 		delta_jiffies = next_jiffies - last_jiffies;
 		if (rcu_delta_jiffies < delta_jiffies) {
 			next_jiffies = last_jiffies + rcu_delta_jiffies;

kernel/time/tick-sched.c static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · line 713

Quoted 9 lines
kernel/time/tick-sched.c +2 -1
@@ -703,7 +706,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 		 */
 		tick_do_update_jiffies64(ktime_get());
 	}
-	raise_softirq_irqoff(TIMER_SOFTIRQ);
+	if (raise_softirq)
+		raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
 	ts->next_jiffies = next_jiffies;
 	ts->last_jiffies = last_jiffies;

Lets look at the things outside the pending timer list that can end up failing to program the timer because it is in the past already:

Quoted 8 lines
Quoted 2 lines

Is this an attempt to find possible regressions introduced by this change ?

Yeah, it would be nice to make sure that the cause of these softirqs isn't mistakenly ignored.
And also I want to be sure we really understand what we
are doing, which is not the case right now as we don't know what is causing this expired timer.

What is the interrupt that is the cause for tick_nohz_stop_sched_tick, you mean?

       <...>-45815 [015] d...2.. 25722056692012 (+961446): kvm_exit: reason EXTERNAL_INTERRUPT rip 0x7f5e448479d0 info 0 800000ef
       <...>-45815 [015] d..h1.. 25722056692844 (+832): apic_timer_fn<-__run_hrtimer
       <...>-45815 [015] d...1.. 25722056695442 (+2598): raise_softirq_irqoff <-tick_nohz_stop_sched_tick

Emulation of guest APIC timer by hrtimer (apic_timer_fn).

Quoted 18 lines
Quoted 18 lines
Quoted 11 lines

_ timekeeping_max_deferment(): I doubt, the value is pretty high
_ scheduler_tick_max_deferment(); it's one second long, way enough to never be in
the past by the time we program the clock
_ RCU, irq_work, arch: may be, if the last jiffies update is far enough. But apparently
the problem is elsewhere since you keep the softirq for these and your patch solves your problem.
_ In case hrtimer runs in low-res mode and the next hrtimer is very close, or even in the past
already, you may run into such issue. And hrtimer doesn't need the timer softirq, at least not to run the callbacks. It needs it only if hrtimer is switching to high-res mode, I think it's a rare event.

Quoted 2 lines

Now it would be nice to identify the issue we are facing here. Are you running in hrtimer low-res mode?

The issue is a latency spike due to ksoftirqd waking up to process pending timers, processing two deferred timers, but no non-deferred timers.

Quoted 5 lines
Quoted 3 lines

hrtimer is not in low-res mode.

The issue is ksoftirqd waking up in the first place.

Sure, but why is it waking up exactly?

Because there is a bug (the patch is trying to fix the bug by raising timer softirq only when timer softirq handler has any work to do).

The only timers pending in the timer list are deferred ones from vmstat_update:

ksoftirqd/15-265   [015] ....111 25722056709372 (+7098): softirq_entry: vec=1 [action=TIMER]
ksoftirqd/15-265   [015] .....11 25722056709964 (+592): run_timer_softirq <-do_current_softirqs
ksoftirqd/15-265   [015] ....111 25722056714034 (+4070): timer_expire_entry: timer=ffff88082f6f14a0 function=delayed_work_timer_fn now=4480299175
ksoftirqd/15-265   [015] ....112 25722056715738 (+1704):
workqueue_queue_work: work struct=ffff88082f6f1480 function=vmstat_update workqueue=ffff88041f408000 req_cpu=5120 cpu=15
ksoftirqd/15-265   [015] ....112 25722056716304 (+566): workqueue_activate_work: work struct ffff88082f6f1480
ksoftirqd/15-265   [015] ....111 25722056719052 (+2748): timer_expire_exit: timer=ffff88082f6f14a0
ksoftirqd/15-265   [015] ....111 25722056719384 (+332): softirq_exit: vec=1 [action=TIMER]

Which should only be processed once there are actual add_timer timers being fired (there are no such add_timer timers on this CPU).

Does that make sense?

#7 lore raw
Quoted 7 lines · Marcelo Tosatti
Quoted 5 lines · Frederic Weisbecker

Yeah, it would be nice to make sure that the cause of these softirqs isn't mistakenly ignored.
And also I want to be sure we really understand what we
are doing, which is not the case right now as we don't know what is causing this expired timer.

Quoted 8 lines

What is the interrupt that is the cause for tick_nohz_stop_sched_tick, you mean?

       <...>-45815 [015] d...2.. 25722056692012 (+961446): kvm_exit: reason EXTERNAL_INTERRUPT rip 0x7f5e448479d0 info 0 800000ef
       <...>-45815 [015] d..h1.. 25722056692844 (+832): apic_timer_fn<-__run_hrtimer
       <...>-45815 [015] d...1.. 25722056695442 (+2598): raise_softirq_irqoff <-tick_nohz_stop_sched_tick

Emulation of guest APIC timer by hrtimer (apic_timer_fn).

Nope, I meant what is the root cause of the softirq. But lets continue on that below:

Quoted 18 lines
Quoted 1 line

Sure, but why is it waking up exactly?

Because there is a bug (the patch is trying to fix the bug by raising timer softirq only when timer softirq handler has any work to do).

The only timers pending in the timer list are deferred ones from vmstat_update:

ksoftirqd/15-265   [015] ....111 25722056709372 (+7098): softirq_entry: vec=1 [action=TIMER]
ksoftirqd/15-265   [015] .....11 25722056709964 (+592): run_timer_softirq <-do_current_softirqs
ksoftirqd/15-265   [015] ....111 25722056714034 (+4070): timer_expire_entry: timer=ffff88082f6f14a0 function=delayed_work_timer_fn now=4480299175
ksoftirqd/15-265   [015] ....112 25722056715738 (+1704):
workqueue_queue_work: work struct=ffff88082f6f1480 function=vmstat_update workqueue=ffff88041f408000 req_cpu=5120 cpu=15
ksoftirqd/15-265   [015] ....112 25722056716304 (+566): workqueue_activate_work: work struct ffff88082f6f1480
ksoftirqd/15-265   [015] ....111 25722056719052 (+2748): timer_expire_exit: timer=ffff88082f6f14a0
ksoftirqd/15-265   [015] ....111 25722056719384 (+332): softirq_exit: vec=1 [action=TIMER]
Quoted 4 lines

Which should only be processed once there are actual add_timer timers being fired (there are no such add_timer timers on this CPU).

Does that make sense?

So the source of these softirqs is those deffered timers? But defferable timers are only defferable in idle-nohz mode, not full-nohz. They are actually deffered in practice in full-nohz but it's a bug :o) (which I need to fix).

Still, I don't think this is the source of the softirqs since your patch fixes the issue of non-timers triggering softirqs.

So here is the issue: something that is not a "struct timer_list" is causing the expiry time of the next tick to be in the past or now. See tick_nohz_stop_sched_tick(), the softirq is triggered when delta_jiffies < 1 or when the timer fails to be reprogrammed because it has already expired.

What can cause this expiry time to be now or in the past? Well for that we need to check everything that is used to evaluate the next tick:

1) struct timer_list Timers
2) low-res hrtimers
3) scheduler_tick_max_deferment
4) timekeeping_max_deferment
5) (rcu|arch|irq_work)_needs_tick()
6) maybe something else I'm missing

Your patch has reduced the softirq to only be triggered in case 1) and it works for you. This means the spurious softirqs that you saw were caused by 2,3,4,5 or 6. I want to know which one and why because I need to understand exactly which event is going to not trigger a softirq anymore after this patch. We want know that to ensure there is no side effect after your patch.

Thanks.

#8 lore raw
Quoted 47 lines · Frederic Weisbecker
Quoted 7 lines · Marcelo Tosatti
Quoted 5 lines · Frederic Weisbecker

Yeah, it would be nice to make sure that the cause of these softirqs isn't mistakenly ignored.
And also I want to be sure we really understand what we
are doing, which is not the case right now as we don't know what is causing this expired timer.

Quoted 8 lines

What is the interrupt that is the cause for tick_nohz_stop_sched_tick, you mean?

       <...>-45815 [015] d...2.. 25722056692012 (+961446): kvm_exit: reason EXTERNAL_INTERRUPT rip 0x7f5e448479d0 info 0 800000ef
       <...>-45815 [015] d..h1.. 25722056692844 (+832): apic_timer_fn<-__run_hrtimer
       <...>-45815 [015] d...1.. 25722056695442 (+2598): raise_softirq_irqoff <-tick_nohz_stop_sched_tick

Emulation of guest APIC timer by hrtimer (apic_timer_fn).

Nope, I meant what is the root cause of the softirq. But lets continue on that below:

Quoted 18 lines
Quoted 1 line

Sure, but why is it waking up exactly?

Because there is a bug (the patch is trying to fix the bug by raising timer softirq only when timer softirq handler has any work to do).

The only timers pending in the timer list are deferred ones from vmstat_update:

ksoftirqd/15-265   [015] ....111 25722056709372 (+7098): softirq_entry: vec=1 [action=TIMER]
ksoftirqd/15-265   [015] .....11 25722056709964 (+592): run_timer_softirq <-do_current_softirqs
ksoftirqd/15-265   [015] ....111 25722056714034 (+4070): timer_expire_entry: timer=ffff88082f6f14a0 function=delayed_work_timer_fn now=4480299175
ksoftirqd/15-265   [015] ....112 25722056715738 (+1704):
workqueue_queue_work: work struct=ffff88082f6f1480 function=vmstat_update workqueue=ffff88041f408000 req_cpu=5120 cpu=15
ksoftirqd/15-265   [015] ....112 25722056716304 (+566): workqueue_activate_work: work struct ffff88082f6f1480
ksoftirqd/15-265   [015] ....111 25722056719052 (+2748): timer_expire_exit: timer=ffff88082f6f14a0
ksoftirqd/15-265   [015] ....111 25722056719384 (+332): softirq_exit: vec=1 [action=TIMER]
Quoted 4 lines

Which should only be processed once there are actual add_timer timers being fired (there are no such add_timer timers on this CPU).

Does that make sense?

So the source of these softirqs is those deffered timers? But defferable timers are only defferable in idle-nohz mode, not full-nohz. They are actually deffered in practice in full-nohz but it's a bug :o) (which I need to fix).

Quoted 6 lines

Still, I don't think this is the source of the softirqs since your patch fixes the issue of non-timers triggering softirqs.

So here is the issue: something that is not a "struct timer_list" is causing the expiry time of the next tick to be in the past or now. See tick_nohz_stop_sched_tick(), the softirq is triggered when delta_jiffies < 1

delta_jiffies = NEXT_TIMER_MAX_DELTA.

tick_nohz_stop_sched_tick: delta_jiffies: 1073741823 rcu_delta_jiffies: 18446744073709551615 tick_stopped: 1

Quoted 2 lines

or when the timer fails to be reprogrammed
because it has already expired.

Right, missed that. I'll ask Luiz to gather info on why its failing.

Quoted 18 lines

What can cause this expiry time to be now or in the past? Well for that we need to check everything that is used to evaluate the next tick:

1) struct timer_list Timers
2) low-res hrtimers
3) scheduler_tick_max_deferment
4) timekeeping_max_deferment
5) (rcu|arch|irq_work)_needs_tick()
6) maybe something else I'm missing

Your patch has reduced the softirq to only be triggered in case 1) and it works for you. This means the spurious softirqs that you saw were caused by 2,3,4,5 or 6. I want to know which one and why because I need to understand exactly which event is going to not trigger a softirq anymore after this patch. We want know that to ensure there is no side effect after your patch.

Thanks.