Skip to contentSkip to conversation

time: validate watchdog clocksource using second best candidate

RFC PATCH In review Konstantin Khlebnikov 2 participants 7 messages May 15, 2019 – Oct 23, 2019 Timekeeping, clocksource core, NTP, alarmtimer

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 time: validate watchdog clocksource using second best candidate

Timekeeping watchdog verifies doubtful clocksources using more reliable candidates. For x86 it likely verifies 'tsc' using 'hpet'. But 'hpet' is far from perfect too. It's better to have second opinion if possible.

We're seeing sudden jumps of hpet counter to 0xffffffff:

timekeeping watchdog on CPU56: Marking clocksource 'tsc' as unstable because the skew is too large:
'hpet' wd_now: ffffffff wd_last: 19ec5720 mask: ffffffff
'tsc' cs_now: 69b8a15f0aed cs_last: 69b862c9947d mask: ffffffffffffffff

Shaohua Li reported the same case three years ago. His patch backlisted this exact value and re-read hpet counter.

This patch uses second reliable clocksource as backup for validation. For x86 this is usually 'acpi_pm'. If watchdog and backup are not consent then other clocksources will not be marked as unstable at this iteration.

In this case watchdog will print something like:
timekeeping watchdog on CPUxx: Ignoring watchdog 'hpet' hiccup because backup watchdog 'acpi_pm' is not consent:
and dump states of both clocksources.

Also this patch prints 'wd_nsec' and 'cs_nsec' for easier debug.

---
 kernel/time/clocksource.c |   67 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 16 deletions(-)
kernel/time/clocksource.c +51 -16
index 3bcc19ceb073..c7209c833e97 100644
@@ -99,6 +99,7 @@ static void clocksource_select(void);
 
 static LIST_HEAD(watchdog_list);
 static struct clocksource *watchdog;
+static struct clocksource *watchdog_backup;
 static struct timer_list watchdog_timer;
 static DECLARE_WORK(watchdog_work, clocksource_watchdog_work);
 static DEFINE_SPINLOCK(watchdog_lock);
@@ -190,6 +191,7 @@ static void clocksource_watchdog(struct timer_list *unused)
 	u64 csnow, wdnow, cslast, wdlast, delta;
 	int64_t wd_nsec, cs_nsec;
 	int next_cpu, reset_pending;
+	bool backup_consent = true;
 
 	spin_lock(&watchdog_lock);
 	if (!watchdog_running)
@@ -236,16 +238,33 @@ static void clocksource_watchdog(struct timer_list *unused)
 
 		/* Check the deviation from the watchdog clocksource. */
 		if (abs(cs_nsec - wd_nsec) > WATCHDOG_THRESHOLD) {
-			pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n",
-				smp_processor_id(), cs->name);
-			pr_warn("                      '%s' wd_now: %llx wd_last: %llx mask: %llx\n",
-				watchdog->name, wdnow, wdlast, watchdog->mask);
-			pr_warn("                      '%s' cs_now: %llx cs_last: %llx mask: %llx\n",
-				cs->name, csnow, cslast, cs->mask);
-			__clocksource_unstable(cs);
+
+			/* Backup watchdog clocksource is first in the list */
+			if (cs == watchdog_backup)
+				backup_consent = false;
+
+			if (backup_consent)
+				pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n",
+					smp_processor_id(), cs->name);
+			else
+				pr_warn("timekeeping watchdog on CPU%d: Ignoring watchdog '%s' hiccup because backup watchdog '%s' is not consent:\n",
+					smp_processor_id(), watchdog->name,
+					watchdog_backup->name);
+
+			pr_warn("                      '%s' wd_now: %llx wd_last: %llx mask: %llx wd_nsec: %lld\n",
+				watchdog->name, wdnow, wdlast, watchdog->mask,
+				wd_nsec);
+			pr_warn("                      '%s' cs_now: %llx cs_last: %llx mask: %llx cs_nsec: %lld\n",
+				cs->name, csnow, cslast, cs->mask, cs_nsec);
+
+			if (backup_consent)
+				__clocksource_unstable(cs);
 			continue;
 		}
 
+		if (cs == watchdog_backup)
+			continue;
+
 		if (cs == curr_clocksource && cs->tick_stable)
 			cs->tick_stable(cs);
 
@@ -345,7 +364,7 @@ static void clocksource_enqueue_watchdog(struct clocksource *cs)
 	}
 }
 
-static void clocksource_select_watchdog(bool fallback)
+static void clocksource_select_watchdog(struct clocksource *except)
 {
 	struct clocksource *cs, *old_wd;
 	unsigned long flags;
@@ -353,26 +372,42 @@ static void clocksource_select_watchdog(bool fallback)
 	spin_lock_irqsave(&watchdog_lock, flags);
 	/* save current watchdog */
 	old_wd = watchdog;
-	if (fallback)
+	if (watchdog == except)
 		watchdog = NULL;
 
+	if (watchdog_backup) {
+		list_del_init(&watchdog_backup->wd_list);
+		watchdog_backup = NULL;
+	}
+
 	list_for_each_entry(cs, &clocksource_list, list) {
 		/* cs is a clocksource to be watched. */
 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)
 			continue;
 
 		/* Skip current if we were requested for a fallback. */
-		if (fallback && cs == old_wd)
+		if (cs == except)
 			continue;
 
 		/* Pick the best watchdog. */
-		if (!watchdog || cs->rating > watchdog->rating)
+		if (!watchdog || cs->rating > watchdog->rating) {
+			watchdog_backup = watchdog;
 			watchdog = cs;
+		}
+
+		/* Pick the second best for cross-validation. */
+		if (cs != watchdog &&
+		    (!watchdog_backup || cs->rating > watchdog_backup->rating))
+			watchdog_backup = cs;
 	}
 	/* If we failed to find a fallback restore the old one. */
 	if (!watchdog)
 		watchdog = old_wd;
 
+	/* Backup watchdog must be first in the list. */
+	if (watchdog_backup)
+		list_add(&watchdog_backup->wd_list, &watchdog_list);
+
 	/* If we changed the watchdog we need to reset cycles. */
 	if (watchdog != old_wd)
 		clocksource_reset_watchdog();
@@ -430,7 +465,7 @@ static int clocksource_watchdog_kthread(void *data)
 
 static bool clocksource_is_watchdog(struct clocksource *cs)
 {
-	return cs == watchdog;
+	return cs == watchdog || cs == watchdog_backup;
 }
 
 #else /* CONFIG_CLOCKSOURCE_WATCHDOG */
@@ -441,7 +476,7 @@ static void clocksource_enqueue_watchdog(struct clocksource *cs)
 		cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
 }
 
-static void clocksource_select_watchdog(bool fallback) { }
+static void clocksource_select_watchdog(struct clocksource *except) { }
 static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { }
 static inline void clocksource_resume_watchdog(void) { }
 static inline int __clocksource_watchdog_kthread(void) { return 0; }
@@ -933,7 +968,7 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
 	clocksource_watchdog_unlock(&flags);
 
 	clocksource_select();
-	clocksource_select_watchdog(false);
+	clocksource_select_watchdog(NULL);
 	__clocksource_suspend_select(cs);
 	mutex_unlock(&clocksource_mutex);
 	return 0;
@@ -962,7 +997,7 @@ void clocksource_change_rating(struct clocksource *cs, int rating)
 	clocksource_watchdog_unlock(&flags);
 
 	clocksource_select();
-	clocksource_select_watchdog(false);
+	clocksource_select_watchdog(NULL);
 	clocksource_suspend_select(false);
 	mutex_unlock(&clocksource_mutex);
 }
@@ -977,7 +1012,7 @@ static int clocksource_unbind(struct clocksource *cs)
 
 	if (clocksource_is_watchdog(cs)) {
 		/* Select and try to install a replacement watchdog. */
-		clocksource_select_watchdog(true);
+		clocksource_select_watchdog(cs);
 		if (clocksource_is_watchdog(cs))
 			return -EBUSY;
 	}
#1 lore raw
Quoted 5 lines · Konstantin Khlebnikov

Timekeeping watchdog verifies doubtful clocksources using more reliable candidates. For x86 it likely verifies 'tsc' using 'hpet'. But 'hpet' is far from perfect too. It's better to have second opinion if possible.

We're seeing sudden jumps of hpet counter to 0xffffffff:

On which kind of hardware? A particular type of CPU or random ones?

Quoted 6 lines

timekeeping watchdog on CPU56: Marking clocksource 'tsc' as unstable because the skew is too large:
'hpet' wd_now: ffffffff wd_last: 19ec5720 mask: ffffffff
'tsc' cs_now: 69b8a15f0aed cs_last: 69b862c9947d mask: ffffffffffffffff

Shaohua Li reported the same case three years ago. His patch backlisted this exact value and re-read hpet counter.

Can you provide a reference please? Preferrably a lore.kernel.org/... URL

Quoted 3 lines

This patch uses second reliable clocksource as backup for validation. For x86 this is usually 'acpi_pm'. If watchdog and backup are not consent then other clocksources will not be marked as unstable at this iteration.

The mess you add to the watchdog code is unholy and that's broken as there is no guarantee for acpi_pm (or any other secondary watchdog) being available.

If the only wreckaged value is always ffffffff then I rather reread the hpet in that case. But not in the watchdog code, we need to do that in the HPET code as this affects any other HPET user as well.

Thanks,

tglx

#2 lore raw
Quoted 9 lines · Thomas Gleixner
Quoted 5 lines · Konstantin Khlebnikov

Timekeeping watchdog verifies doubtful clocksources using more reliable candidates. For x86 it likely verifies 'tsc' using 'hpet'. But 'hpet' is far from perfect too. It's better to have second opinion if possible.

We're seeing sudden jumps of hpet counter to 0xffffffff:

On which kind of hardware? A particular type of CPU or random ones?

In general this is very rare event.

This exact pattern have been seen ten times or so on several servers with Intel(R) Xeon(R) CPU E5-2660 v4 @ 2.00GHz
(this custom built platform with chipset Intel C610)

and haven't seen for previous generation
Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz
(this is another custom built platform)

So, this might be not related to cpu model.

Quoted 9 lines
Quoted 7 lines

timekeeping watchdog on CPU56: Marking clocksource 'tsc' as unstable because the skew is too large:
'hpet' wd_now: ffffffff wd_last: 19ec5720 mask: ffffffff
'tsc' cs_now: 69b8a15f0aed cs_last: 69b862c9947d mask: ffffffffffffffff

Shaohua Li reported the same case three years ago. His patch backlisted this exact value and re-read hpet counter.

Can you provide a reference please? Preferrably a lore.kernel.org/... URL

Link was in patch: https://lore.kernel.org/patchwork/patch/667413/

Quoted 8 lines
Quoted 4 lines

This patch uses second reliable clocksource as backup for validation. For x86 this is usually 'acpi_pm'. If watchdog and backup are not consent then other clocksources will not be marked as unstable at this iteration.

The mess you add to the watchdog code is unholy and that's broken as there is no guarantee for acpi_pm (or any other secondary watchdog) being available.

ACPI power management timer is a pretty standard x86 hardware.
But my patch should work for any platform with any second reliable clocksource.

If there is no second clocksource my patch does noting:
watchdog_backup stays NULL and backup_consent always true.

Quoted 9 lines

If the only wreckaged value is always ffffffff then I rather reread the hpet in that case. But not in the watchdog code, we need to do that in the HPET code as this affects any other HPET user as well.

Thanks,

tglx

#3 lore raw
Quoted 13 lines · Konstantin Khlebnikov
Quoted 9 lines · Thomas Gleixner
Quoted 5 lines · Konstantin Khlebnikov

Timekeeping watchdog verifies doubtful clocksources using more reliable candidates. For x86 it likely verifies 'tsc' using 'hpet'. But 'hpet' is far from perfect too. It's better to have second opinion if possible.

We're seeing sudden jumps of hpet counter to 0xffffffff:

On which kind of hardware? A particular type of CPU or random ones?

In general this is very rare event.

Quoted 7 lines

This exact pattern have been seen ten times or so on several servers with Intel(R) Xeon(R) CPU E5-2660 v4 @ 2.00GHz
(this custom built platform with chipset Intel C610)

and haven't seen for previous generation
Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz
(this is another custom built platform)

Same chipset? Note the HPET is part of the chipset not of the CPU.

Quoted 1 line

Link was in patch: https://lore.kernel.org/patchwork/patch/667413/

Hmm. Not really helpful either.

Quoted 9 lines
Quoted 7 lines
Quoted 3 lines

This patch uses second reliable clocksource as backup for validation. For x86 this is usually 'acpi_pm'. If watchdog and backup are not consent then other clocksources will not be marked as unstable at this iteration.

The mess you add to the watchdog code is unholy and that's broken as there is no guarantee for acpi_pm (or any other secondary watchdog) being available.

ACPI power management timer is a pretty standard x86 hardware.

Used to be.

Quoted 2 lines

But my patch should work for any platform with any second reliable clocksource.

Which is close to zero if PM timer is not exposed.

Quoted 2 lines

If there is no second clocksource my patch does noting:
watchdog_backup stays NULL and backup_consent always true.

That still does not justify the extra complexity for a few custom built systems.

Thanks,

tglx

#4 lore raw
Quoted 6 lines · Thomas Gleixner
Quoted 2 lines · Konstantin Khlebnikov

If there is no second clocksource my patch does noting:
watchdog_backup stays NULL and backup_consent always true.

That still does not justify the extra complexity for a few custom built systems.

Aside of that this leaves the HPET in a half broken state. HPET is not only used as a clock event device it's also exposed by HPET device. So no, if we figure out that HPET is broken on some platforms we have to blacklist and disable it completely and not just duct tape the place which exposes the wreckage.

Thanks,

tglx

#5 lore raw
In reply to Thomas Gleixner
Quoted 19 lines · Thomas Gleixner
Quoted 17 lines · Konstantin Khlebnikov
Quoted 9 lines · Thomas Gleixner
Quoted 5 lines · Konstantin Khlebnikov

Timekeeping watchdog verifies doubtful clocksources using more reliable candidates. For x86 it likely verifies 'tsc' using 'hpet'. But 'hpet' is far from perfect too. It's better to have second opinion if possible.

We're seeing sudden jumps of hpet counter to 0xffffffff:

On which kind of hardware? A particular type of CPU or random ones?

In general this is very rare event.

This exact pattern have been seen ten times or so on several servers with Intel(R) Xeon(R) CPU E5-2660 v4 @ 2.00GHz
(this custom built platform with chipset Intel C610)

Quoted 5 lines
Quoted 3 lines

and haven't seen for previous generation
Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz
(this is another custom built platform)

Same chipset? Note the HPET is part of the chipset not of the CPU.

Almost the same. Intel C600.

Quoted 29 lines
Quoted 2 lines

Link was in patch: https://lore.kernel.org/patchwork/patch/667413/

Hmm. Not really helpful either.

Quoted 9 lines
Quoted 7 lines
Quoted 3 lines

This patch uses second reliable clocksource as backup for validation. For x86 this is usually 'acpi_pm'. If watchdog and backup are not consent then other clocksources will not be marked as unstable at this iteration.

The mess you add to the watchdog code is unholy and that's broken as there is no guarantee for acpi_pm (or any other secondary watchdog) being available.

ACPI power management timer is a pretty standard x86 hardware.

Used to be.

Quoted 2 lines

But my patch should work for any platform with any second reliable clocksource.

Which is close to zero if PM timer is not exposed.

Quoted 2 lines

If there is no second clocksource my patch does noting:
watchdog_backup stays NULL and backup_consent always true.

That still does not justify the extra complexity for a few custom built systems.

Quoted 6 lines

Aside of that this leaves the HPET in a half broken state. HPET is not only used as a clock event device it's also exposed by HPET device. So no, if we figure out that HPET is broken on some platforms we have to blacklist and disable it completely and not just duct tape the place which exposes the wreckage.

If re-reading helps then HPET is fine.
This is temporary failure, probably bus issue.

I'll add re-reading with debug logging and try to collect more information this year.

#6 lore raw
Quoted 64 lines · Konstantin Khlebnikov
Quoted 19 lines · Thomas Gleixner
Quoted 17 lines · Konstantin Khlebnikov
Quoted 9 lines · Thomas Gleixner
Quoted 5 lines · Konstantin Khlebnikov

Timekeeping watchdog verifies doubtful clocksources using more reliable candidates. For x86 it likely verifies 'tsc' using 'hpet'. But 'hpet' is far from perfect too. It's better to have second opinion if possible.

We're seeing sudden jumps of hpet counter to 0xffffffff:

On which kind of hardware? A particular type of CPU or random ones?

In general this is very rare event.

This exact pattern have been seen ten times or so on several servers with Intel(R) Xeon(R) CPU E5-2660 v4 @ 2.00GHz
(this custom built platform with chipset Intel C610)

Quoted 5 lines
Quoted 3 lines

and haven't seen for previous generation
Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz
(this is another custom built platform)

Same chipset? Note the HPET is part of the chipset not of the CPU.

Almost the same. Intel C600.

Quoted 29 lines
Quoted 2 lines

Link was in patch: https://lore.kernel.org/patchwork/patch/667413/

Hmm. Not really helpful either.

Quoted 9 lines
Quoted 7 lines
Quoted 3 lines

This patch uses second reliable clocksource as backup for validation. For x86 this is usually 'acpi_pm'. If watchdog and backup are not consent then other clocksources will not be marked as unstable at this iteration.

The mess you add to the watchdog code is unholy and that's broken as there is no guarantee for acpi_pm (or any other secondary watchdog) being available.

ACPI power management timer is a pretty standard x86 hardware.

Used to be.

Quoted 2 lines

But my patch should work for any platform with any second reliable clocksource.

Which is close to zero if PM timer is not exposed.

Quoted 2 lines

If there is no second clocksource my patch does noting:
watchdog_backup stays NULL and backup_consent always true.

That still does not justify the extra complexity for a few custom built systems.

Quoted 6 lines

Aside of that this leaves the HPET in a half broken state. HPET is not only used as a clock event device it's also exposed by HPET device. So no, if we figure out that HPET is broken on some platforms we have to blacklist and disable it completely and not just duct tape the place which exposes the wreckage.

Quoted 5 lines

If re-reading helps then HPET is fine.
This is temporary failure, probably bus issue.

I'll add re-reading with debug logging and try to collect more information this year.

Good news, everyone! We've found conditions when HPET counter returns '-1'.

clocksource: timekeeping watchdog on CPU21: Marking clocksource 'tsc' as unstable because the skew is too large:
clocksource:                       'hpet' wd_now: ffffffff wd_last: 40bc1ee3 mask: ffffffff
clocksource:                       'tsc' cs_now: 919b39935acdaa cs_last: 919b3957c0ec24 mask: ffffffffffffffff
clocksource: Switched to clocksource hpet

This happens when user-space does inappropriate access to directly mapped HPET. For example dumping whole "vvar" area: memcpy(buf, addr("vvar"), 0x3000).

In our case sequence was triggered by bug in "atop" crashed at "\n" in task comm. =)

In upstream everything is fine. Direct access to HPET was sealed in 4.7 (we seen bug in 4.4)
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=1ed95e52d902035e39a715ff3a314a893a96e5b7

Kudos to Arseny Smalyuk for (accidental) reproducing and investigation.

---

#include <stdio.h>
#include <string.h>

char tmp[0x3000];

int main() {
    void* vvar = NULL;
    FILE* fp = fopen("/proc/self/maps", "r");

    char buf[4096];
    while (fgets(buf, 4096, fp)) {
        size_t slen = strlen(buf);
        if (slen > 0 && buf[slen - 1] == '\n') {
            --slen;
        }
        if (slen > 6 && !memcmp(buf + slen - 6, "[vvar]", 6)) {
            sscanf(buf, "%p", &vvar);
            break;
        }
    }

    fclose(fp);

    memcpy(tmp, vvar, 0x3000);
    return 0;
}