Skip to contentSkip to conversation

therm_throt: Refactor thermal throttle processing, and keep a total count of events.

PATCH In review Dmitriy Zavin 2 participants 8 messages Sep 22, 2006

lore
Share & tools
lore
#0 lore raw

v2 cover therm_throt: Refactor thermal throttle processing, and keep a total count of events.

This patch-set factors out the thermal throttle processing code from i386 and x86_64 into a separate file (therm_throt.c). This allows consistent reporting of CPU thermal throttle events. Furthermore, a counter is added to /sys that keeps track of the number of thermal events, such that the user knows how bad the thermal problem might be (since the logging to syslog and mcelog is rate limited).

Tested on 32bit and 64bit Intel P4 Xeons.

Differences from last cut:
- No more #ifdef CONFIG_X86_64.. sorry Andi :). x86_64 specific code is in arch/x86_64/kernel/mce.c
- Commented the need/use for time_before64/time_after64. (I addressed Andi Kleen's concern about these macros in a followup from last attempt, but never got a response.)
- More comments in therm_throt.c and individual patch descriptions

 arch/i386/kernel/cpu/mcheck/Makefile      |    2
 arch/i386/kernel/cpu/mcheck/p4.c          |   26 +---
 arch/i386/kernel/cpu/mcheck/therm_throt.c |  171 ++++++++++++++++++++++++++++++
 arch/x86_64/kernel/Makefile               |    4
 arch/x86_64/kernel/mce.c                  |   27 ++++
 arch/x86_64/kernel/mce_intel.c            |   30 +----
 include/asm-i386/therm_throt.h            |    9 +
 include/asm-x86_64/mce.h                  |    4
 include/asm-x86_64/therm_throt.h          |    1
 include/linux/jiffies.h                   |   15 ++
 10 files changed, 248 insertions(+), 41 deletions(-)
#1 lore raw

PATCH v2 1/4 x86_64/i386 therm mce: Refactor thermal throttle processing

Refactor the event processing (syslog messaging and rate limiting) into separate file therm_throt.c. This allows consistent reporting of CPU thermal throttle events.

After ACK'ing the interrupt, if the event is current, the user (p4.c/mce_intel.c) calls therm_throt_process to log (and rate limit) the event. If that function returns 1, the user has the option to log things further (such as to mce_log in x86_64).

---
 arch/i386/kernel/cpu/mcheck/Makefile      |    2 +
 arch/i386/kernel/cpu/mcheck/p4.c          |   23 +++---------
 arch/i386/kernel/cpu/mcheck/therm_throt.c |   58 +++++++++++++++++++++++++++++
 arch/x86_64/kernel/Makefile               |    4 +-
 arch/x86_64/kernel/mce.c                  |   27 ++++++++++++++
 arch/x86_64/kernel/mce_intel.c            |   27 +++-----------
 include/asm-i386/therm_throt.h            |    6 +++
 include/asm-x86_64/mce.h                  |    4 ++
 include/asm-x86_64/therm_throt.h          |    1 +
 9 files changed, 111 insertions(+), 41 deletions(-)
arch/i386/kernel/cpu/mcheck/Makefile +1 -1
index 30808f3..f1ebe1c 100644
@@ -1,2 +1,2 @@
-obj-y	=	mce.o k7.o p4.o p5.o p6.o winchip.o
+obj-y	=	mce.o k7.o p4.o p5.o p6.o winchip.o therm_throt.o
 obj-$(CONFIG_X86_MCE_NONFATAL)	+=	non-fatal.o
arch/i386/kernel/cpu/mcheck/p4.c +6 -17
index b95f1b3..d83a669 100644
@@ -13,6 +13,8 @@ #include <asm/system.h>
 #include <asm/msr.h>
 #include <asm/apic.h>
 
+#include <asm/therm_throt.h>
+
 #include "mce.h"
 
 /* as supported by the P4/Xeon family */
@@ -44,25 +46,12 @@ static void unexpected_thermal_interrupt
 /* P4/Xeon Thermal transition interrupt handler */
 static void intel_thermal_interrupt(struct pt_regs *regs)
 {
-	u32 l, h;
-	unsigned int cpu = smp_processor_id();
-	static unsigned long next[NR_CPUS];
+	__u64 msr_val;
 
 	ack_APIC_irq();
 
-	if (time_after(next[cpu], jiffies))
-		return;
-
-	next[cpu] = jiffies + HZ*5;
-	rdmsr(MSR_IA32_THERM_STATUS, l, h);
-	if (l & 0x1) {
-		printk(KERN_EMERG "CPU%d: Temperature above threshold\n", cpu);
-		printk(KERN_EMERG "CPU%d: Running in modulated clock mode\n",
-				cpu);
-		add_taint(TAINT_MACHINE_CHECK);
-	} else {
-		printk(KERN_INFO "CPU%d: Temperature/speed normal\n", cpu);
-	}
+	rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
+	therm_throt_process(msr_val & 0x1);
 }
 
 /* Thermal interrupt handler for this CPU setup */
@@ -122,7 +111,7 @@ static void intel_init_thermal(struct cp
 	
 	rdmsr (MSR_IA32_MISC_ENABLE, l, h);
 	wrmsr (MSR_IA32_MISC_ENABLE, l | (1<<3), h);
-	
+
 	l = apic_read (APIC_LVTTHMR);
 	apic_write_around (APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
 	printk (KERN_INFO "CPU%d: Thermal monitoring enabled\n", cpu);
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +58 -0

58 lines not on this page · raw message

arch/x86_64/kernel/Makefile +2 -2
index b5aaeaf..d97cec0 100644
@@ -11,7 +11,7 @@ obj-y	:= process.o signal.o entry.o trap
 		pci-dma.o pci-nommu.o alternative.o
 
 obj-$(CONFIG_STACKTRACE)	+= stacktrace.o
-obj-$(CONFIG_X86_MCE)         += mce.o
+obj-$(CONFIG_X86_MCE)		+= mce.o therm_throt.o
 obj-$(CONFIG_X86_MCE_INTEL)	+= mce_intel.o
 obj-$(CONFIG_X86_MCE_AMD)	+= mce_amd.o
 obj-$(CONFIG_MTRR)		+= ../../i386/kernel/cpu/mtrr/
@@ -45,6 +45,7 @@ obj-y				+= intel_cacheinfo.o
 
 CFLAGS_vsyscall.o		:= $(PROFILING) -g0
 
+therm_throt-y                   += ../../i386/kernel/cpu/mcheck/therm_throt.o
 bootflag-y			+= ../../i386/kernel/bootflag.o
 cpuid-$(subst m,y,$(CONFIG_X86_CPUID))  += ../../i386/kernel/cpuid.o
 topology-y                     += ../../i386/kernel/topology.o
@@ -54,4 +55,3 @@ quirks-y			+= ../../i386/kernel/quirks.o
 i8237-y				+= ../../i386/kernel/i8237.o
 msr-$(subst m,y,$(CONFIG_X86_MSR))  += ../../i386/kernel/msr.o
 alternative-y			+= ../../i386/kernel/alternative.o
-
arch/x86_64/kernel/mce.c +27 -0
index 4e017fb..6314389 100644
@@ -274,6 +274,33 @@ void do_machine_check(struct pt_regs * r
 	atomic_dec(&mce_entry);
 }
 
+#ifdef CONFIG_X86_MCE_INTEL
+/***
+ * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
+ * @cpu: The CPU on which the event occured.
+ * @status: Event status information
+ *
+ * This function should be called by the thermal interrupt after the
+ * event has been processed and the decision was made to log the event
+ * further.
+ *
+ * The status parameter will be saved to the 'status' field of 'struct mce'
+ * and historically has been the register value of the
+ * MSR_IA32_THERMAL_STATUS (Intel) msr.
+ */
+void mce_log_therm_throt_event(unsigned int cpu, __u64 status)
+{
+	struct mce m;
+
+	memset(&m, 0, sizeof(m));
+	m.cpu = cpu;
+	m.bank = MCE_THERMAL_BANK;
+	m.status = status;
+	rdtscll(m.tsc);
+	mce_log(&m);
+}
+#endif /* CONFIG_X86_MCE_INTEL */
+
 /*
  * Periodic polling timer for "silent" machine check errors.
  */
arch/x86_64/kernel/mce_intel.c +6 -21
index 8f533d2..dec1121 100644
@@ -11,36 +11,21 @@ #include <asm/msr.h>
 #include <asm/mce.h>
 #include <asm/hw_irq.h>
 #include <asm/idle.h>
-
-static DEFINE_PER_CPU(unsigned long, next_check);
+#include <asm/therm_throt.h>
 
 asmlinkage void smp_thermal_interrupt(void)
 {
-	struct mce m;
+	__u64 msr_val;
 
 	ack_APIC_irq();
 
 	exit_idle();
 	irq_enter();
-	if (time_before(jiffies, __get_cpu_var(next_check)))
-		goto done;
-
-	__get_cpu_var(next_check) = jiffies + HZ*300;
-	memset(&m, 0, sizeof(m));
-	m.cpu = smp_processor_id();
-	m.bank = MCE_THERMAL_BANK;
-	rdtscll(m.tsc);
-	rdmsrl(MSR_IA32_THERM_STATUS, m.status);
-	if (m.status & 0x1) {
-		printk(KERN_EMERG
-			"CPU%d: Temperature above threshold, cpu clock throttled\n", m.cpu);
-		add_taint(TAINT_MACHINE_CHECK);
-	} else {
-		printk(KERN_EMERG "CPU%d: Temperature/speed normal\n", m.cpu);
-	}
 
-	mce_log(&m);
-done:
+	rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
+	if (therm_throt_process(msr_val & 1))
+		mce_log_therm_throt_event(smp_processor_id(), msr_val);
+
 	irq_exit();
 }
 
include/asm-i386/therm_throt.h new file +6 -0
new file mode 100644
index 0000000..3c9c22c
@@ -0,0 +1,6 @@
+#ifndef __ASM_I386_THERM_THROT_H__
+#define __ASM_I386_THERM_THROT_H__ 1
+
+int therm_throt_process(int curr);
+
+#endif /* __ASM_I386_THERM_THROT_H__ */
include/asm-x86_64/mce.h +4 -0
index d13687d..5a0def5 100644
@@ -99,6 +99,10 @@ static inline void mce_amd_feature_init(
 }
 #endif
 
+#ifdef CONFIG_X86_MCE_INTEL
+void mce_log_therm_throt_event(unsigned int cpu, __u64 status);
+#endif
+
 extern atomic_t mce_entry;
 
 #endif
include/asm-x86_64/therm_throt.h new file +1 -0
new file mode 100644
index 0000000..5aac059
@@ -0,0 +1 @@
+#include <asm-i386/therm_throt.h>
Signature · 1 line
1.4.2
#2 lore raw

PATCH v2 2/4 jiffies: Add 64bit jiffies compares (for use with get_jiffies_64)

The current time_before/time_after macros will fail typechecks when passed u64 values (as returned by get_jiffies_64()). On 64bit systems, this will just result in a warning about mismatching types without explicit casts, but since unsigned long and u64 (unsigned long long) are of same size, it will still work. On 32bit systems, a long is 32bits, so the value from get_jiffies_64() will be truncated by the cast and thus lose all the precision gained by 64bit jiffies.

---
 include/linux/jiffies.h |   15 +++++++++++++++
 1 files changed, 15 insertions(+), 0 deletions(-)
include/linux/jiffies.h +15 -0
index 329ebcf..c8d5f20 100644
@@ -115,6 +115,21 @@ #define time_after_eq(a,b)	\
 	 ((long)(a) - (long)(b) >= 0))
 #define time_before_eq(a,b)	time_after_eq(b,a)
 
+/* Same as above, but does so with platform independent 64bit types.
+ * These must be used when utilizing jiffies_64 (i.e. return value of
+ * get_jiffies_64() */
+#define time_after64(a,b)	\
+	(typecheck(__u64, a) &&	\
+	 typecheck(__u64, b) && \
+	 ((__s64)(b) - (__s64)(a) < 0))
+#define time_before64(a,b)	time_after64(b,a)
+
+#define time_after_eq64(a,b)	\
+	(typecheck(__u64, a) && \
+	 typecheck(__u64, b) && \
+	 ((__s64)(a) - (__s64)(b) >= 0))
+#define time_before_eq64(a,b)	time_after_eq64(b,a)
+
 /*
  * Have the 32 bit jiffies value wrap 5 minutes after boot
  * so jiffies wrap bugs show up earlier.
Signature · 1 line
1.4.2
#3 lore raw

PATCH v2 3/4 therm_throt: Make the jiffies compares use the 64bit safe macros.

---
 arch/i386/kernel/cpu/mcheck/therm_throt.c |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)
arch/i386/kernel/cpu/mcheck/therm_throt.c +4 -3
index 85eba00..101f7ac 100644
@@ -18,7 +18,7 @@ #include <asm/therm_throt.h>
 /* How long to wait between reporting thermal events */
 #define CHECK_INTERVAL              (300 * HZ)
 
-static DEFINE_PER_CPU(unsigned long, next_check);
+static DEFINE_PER_CPU(__u64, next_check);
 
 /***
  * therm_throt_process - Process thermal throttling event
@@ -39,11 +39,12 @@ static DEFINE_PER_CPU(unsigned long, nex
 int therm_throt_process(int curr)
 {
 	unsigned int cpu = smp_processor_id();
+	__u64 tmp_jiffs = get_jiffies_64();
 
-	if (time_before(jiffies, __get_cpu_var(next_check)))
+	if (time_before64(tmp_jiffs, __get_cpu_var(next_check)))
 		return 0;
 
-	__get_cpu_var(next_check) = jiffies + CHECK_INTERVAL;
+	__get_cpu_var(next_check) = tmp_jiffs + CHECK_INTERVAL;
 
 	/* if we just entered the thermal event */
 	if (curr) {
Signature · 1 line
1.4.2
#4 lore raw

PATCH v2 4/4 therm_throt: Add a cumulative thermal throttle event counter.

The counter is exported to /sys that keeps track of the number of thermal events, such that the user knows how bad the thermal problem might be (since the logging to syslog and mcelog is rate limited).

---
 arch/i386/kernel/cpu/mcheck/p4.c          |    3 +
 arch/i386/kernel/cpu/mcheck/therm_throt.c |  124 ++++++++++++++++++++++++++++-
 arch/x86_64/kernel/mce_intel.c            |    3 +
 include/asm-i386/therm_throt.h            |    3 +
 4 files changed, 127 insertions(+), 6 deletions(-)
arch/i386/kernel/cpu/mcheck/p4.c +3 -0
index d83a669..504434a 100644
@@ -115,6 +115,9 @@ static void intel_init_thermal(struct cp
 	l = apic_read (APIC_LVTTHMR);
 	apic_write_around (APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
 	printk (KERN_INFO "CPU%d: Thermal monitoring enabled\n", cpu);
+
+	/* enable thermal throttle processing */
+	atomic_set(&therm_throt_en, 1);
 	return;
 }
 #endif /* CONFIG_X86_MCE_P4THERMAL */
arch/i386/kernel/cpu/mcheck/therm_throt.c +118 -6

164 lines not on this page · raw message

arch/x86_64/kernel/mce_intel.c +3 -0
index dec1121..6551505 100644
@@ -77,6 +77,9 @@ static void __cpuinit intel_init_thermal
 	apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
 	printk(KERN_INFO "CPU%d: Thermal monitoring enabled (%s)\n",
 		cpu, tm2 ? "TM2" : "TM1");
+
+	/* enable thermal throttle processing */
+	atomic_set(&therm_throt_en, 1);
 	return;
 }
 
include/asm-i386/therm_throt.h +3 -0
index 3c9c22c..399bf60 100644
@@ -1,6 +1,9 @@
 #ifndef __ASM_I386_THERM_THROT_H__
 #define __ASM_I386_THERM_THROT_H__ 1
 
+#include <asm/atomic.h>
+
+extern atomic_t therm_throt_en;
 int therm_throt_process(int curr);
 
 #endif /* __ASM_I386_THERM_THROT_H__ */
Signature · 1 line
1.4.2
#5 lore raw

Re: [PATCH 4/4] therm_throt: Add a cumulative thermal throttle event counter.

arch/i386/kernel/cpu/mcheck/therm_throt.c int therm_throt_process(int curr) · line 164

Quoted 7 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c +6 -0
+	preempt_disable();         /* CPU hotplug */
+	for_each_online_cpu(cpu) {
+		/* connect live CPUs to sysfs */
+		thermal_throttle_add_dev(get_cpu_sysdev(cpu));
+	}
+	preempt_enable();

There was still one problem in there: sysfs_create_group can sleep and that's not ok with preemption disabled.

I changed it to register the notifier first and do the necessary locking in there using a private lock (that the new akpm-recommended(tm) way to do this)

-Andi

#6 lore raw

Re: [PATCH 1/4] x86_64/i386 therm mce: Refactor thermal throttle processing

In reply to Dmitriy Zavin

include/asm-x86_64/mce.h static inline void mce_amd_feature_init( · line 104

Quoted 4 lines
include/asm-x86_64/mce.h +3 -0
+#ifdef CONFIG_X86_MCE_INTEL
+void mce_log_therm_throt_event(unsigned int cpu, __u64 status);
+#endif

I removed that ifdef.

-Andi

#7 lore raw
In reply to Dmitriy Zavin
Quoted 7 lines · Dmitriy Zavin

Differences from last cut:
- No more #ifdef CONFIG_X86_64.. sorry Andi :). x86_64 specific code is in arch/x86_64/kernel/mce.c
- Commented the need/use for time_before64/time_after64. (I addressed Andi Kleen's concern about these macros in a followup from last attempt, but never got a response.)
- More comments in therm_throt.c and individual patch descriptions

Added thanks with minor changes.

-Andi