Skip to contentSkip to conversation

x86_64/i386: Rework thermal throttling detection/handling code for Intel P4/Xeons.

PATCH In review Dima Zavin 2 participants 7 messages Sep 15, 2006 – Sep 19, 2006

lore
Share & tools
lore

Dima Zavin

author
#0 lore raw

PATCH 1/1 x86_64/i386: Rework thermal throttling detection/handling code for Intel P4/Xeons.

- Maintain a count of thermal throttle events and export them to /sys/devices/system/cpu/cpuX/thermal_throttle/count
- Factor out the thermal throttle event handling (i.e. logging) to a separate file, therm_throt.c under i386/kernel/cpu/mcheck.
- The interrupt handlers call into these functions, so that i386 and x86_64 behaviour is the same, and there is no code duplication (except for registering the irq, and actually handling it. It's probably better to keep that separate even though it is, in fact, almost identical)
- I will see if it's plausible to refactor that as well, but for a later patch.
- These changes make it easy to add thermal throttling detection for other x86 vendors (coming in a later patch)

---
 arch/i386/kernel/cpu/mcheck/Makefile      |    2 
 arch/i386/kernel/cpu/mcheck/p4.c          |   25 +--
 arch/i386/kernel/cpu/mcheck/therm_throt.c |  239 +++++++++++++++++++++++++++++
 arch/x86_64/kernel/Makefile               |    4 
 arch/x86_64/kernel/mce.c                  |   50 ++++++
 arch/x86_64/kernel/mce_intel.c            |   34 ++--
 include/asm-i386/therm_throt.h            |   17 ++
 include/asm-x86_64/mce.h                  |    4 
 include/asm-x86_64/therm_throt.h          |    1 
 9 files changed, 338 insertions(+), 38 deletions(-)
arch/i386/kernel/cpu/mcheck/Makefile +1 -1
index 30808f3..f1ebe1c 100644
@@ -1,2 +1,2 @@
-obj-y	=	mce.o k7.o p4.o p5.o p6.o winchip.o
+obj-y	=	mce.o k7.o p4.o p5.o p6.o winchip.o therm_throt.o
 obj-$(CONFIG_X86_MCE_NONFATAL)	+=	non-fatal.o
arch/i386/kernel/cpu/mcheck/p4.c +10 -15
index b95f1b3..4a64bb9 100644
@@ -13,6 +13,8 @@ #include <asm/system.h>
 #include <asm/msr.h>
 #include <asm/apic.h>
 
+#include <asm/therm_throt.h>
+
 #include "mce.h"
 
 /* as supported by the P4/Xeon family */
@@ -45,24 +47,14 @@ static void unexpected_thermal_interrupt
 static void intel_thermal_interrupt(struct pt_regs *regs)
 {
 	u32 l, h;
-	unsigned int cpu = smp_processor_id();
-	static unsigned long next[NR_CPUS];
 
 	ack_APIC_irq();
 
-	if (time_after(next[cpu], jiffies))
-		return;
-
-	next[cpu] = jiffies + HZ*5;
 	rdmsr(MSR_IA32_THERM_STATUS, l, h);
-	if (l & 0x1) {
-		printk(KERN_EMERG "CPU%d: Temperature above threshold\n", cpu);
-		printk(KERN_EMERG "CPU%d: Running in modulated clock mode\n",
-				cpu);
-		add_taint(TAINT_MACHINE_CHECK);
-	} else {
-		printk(KERN_INFO "CPU%d: Temperature/speed normal\n", cpu);
-	}
+	if (l & 0x1)
+		thermal_throttle_inc_count();
+
+	thermal_throttle_process_event(NULL, (l & 0x1));
 }
 
 /* Thermal interrupt handler for this CPU setup */
@@ -122,7 +114,10 @@ static void intel_init_thermal(struct cp
 	
 	rdmsr (MSR_IA32_MISC_ENABLE, l, h);
 	wrmsr (MSR_IA32_MISC_ENABLE, l | (1<<3), h);
-	
+
+	/* init thermal throttle counting before re-enabling the interrupt */
+	thermal_throttle_count_init(cpu);
+
 	l = apic_read (APIC_LVTTHMR);
 	apic_write_around (APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
 	printk (KERN_INFO "CPU%d: Thermal monitoring enabled\n", cpu);
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +239 -0

239 lines not on this page · raw message

arch/x86_64/kernel/Makefile +2 -2
index b5aaeaf..d97cec0 100644
@@ -11,7 +11,7 @@ obj-y	:= process.o signal.o entry.o trap
 		pci-dma.o pci-nommu.o alternative.o
 
 obj-$(CONFIG_STACKTRACE)	+= stacktrace.o
-obj-$(CONFIG_X86_MCE)         += mce.o
+obj-$(CONFIG_X86_MCE)		+= mce.o therm_throt.o
 obj-$(CONFIG_X86_MCE_INTEL)	+= mce_intel.o
 obj-$(CONFIG_X86_MCE_AMD)	+= mce_amd.o
 obj-$(CONFIG_MTRR)		+= ../../i386/kernel/cpu/mtrr/
@@ -45,6 +45,7 @@ obj-y				+= intel_cacheinfo.o
 
 CFLAGS_vsyscall.o		:= $(PROFILING) -g0
 
+therm_throt-y                   += ../../i386/kernel/cpu/mcheck/therm_throt.o
 bootflag-y			+= ../../i386/kernel/bootflag.o
 cpuid-$(subst m,y,$(CONFIG_X86_CPUID))  += ../../i386/kernel/cpuid.o
 topology-y                     += ../../i386/kernel/topology.o
@@ -54,4 +55,3 @@ quirks-y			+= ../../i386/kernel/quirks.o
 i8237-y				+= ../../i386/kernel/i8237.o
 msr-$(subst m,y,$(CONFIG_X86_MSR))  += ../../i386/kernel/msr.o
 alternative-y			+= ../../i386/kernel/alternative.o
-
arch/x86_64/kernel/mce.c +50 -0

62 lines not on this page · raw message

arch/x86_64/kernel/mce_intel.c +14 -20
index 8f533d2..d658f42 100644
@@ -11,36 +11,27 @@ #include <asm/msr.h>
 #include <asm/mce.h>
 #include <asm/hw_irq.h>
 #include <asm/idle.h>
-
-static DEFINE_PER_CPU(unsigned long, next_check);
+#include <asm/therm_throt.h>
 
 asmlinkage void smp_thermal_interrupt(void)
 {
-	struct mce m;
+	__u64 msr_val;
+	struct therm_throt_event_info event;
 
 	ack_APIC_irq();
 
 	exit_idle();
 	irq_enter();
-	if (time_before(jiffies, __get_cpu_var(next_check)))
-		goto done;
-
-	__get_cpu_var(next_check) = jiffies + HZ*300;
-	memset(&m, 0, sizeof(m));
-	m.cpu = smp_processor_id();
-	m.bank = MCE_THERMAL_BANK;
-	rdtscll(m.tsc);
-	rdmsrl(MSR_IA32_THERM_STATUS, m.status);
-	if (m.status & 0x1) {
-		printk(KERN_EMERG
-			"CPU%d: Temperature above threshold, cpu clock throttled\n", m.cpu);
-		add_taint(TAINT_MACHINE_CHECK);
-	} else {
-		printk(KERN_EMERG "CPU%d: Temperature/speed normal\n", m.cpu);
+
+	rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
+	if (msr_val & 0x1)
+		thermal_throttle_inc_count();
+
+	/* process the event, and log to mce_log if necessary */
+	if (thermal_throttle_process_event(&event, (msr_val & 1))) {
+		mce_log_thermal_throttle_event(msr_val, &event);
 	}
 
-	mce_log(&m);
-done:
 	irq_exit();
 }
 
@@ -88,6 +79,9 @@ static void __cpuinit intel_init_thermal
 	rdmsr(MSR_IA32_MISC_ENABLE, l, h);
 	wrmsr(MSR_IA32_MISC_ENABLE, l | (1 << 3), h);
 
+	/* init thermal throttle counting before re-enabling the interrupt */
+	thermal_throttle_count_init(cpu);
+
 	l = apic_read(APIC_LVTTHMR);
 	apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
 	printk(KERN_INFO "CPU%d: Thermal monitoring enabled (%s)\n",
include/asm-i386/therm_throt.h new file +17 -0
new file mode 100644
index 0000000..228122c
@@ -0,0 +1,17 @@
+#ifndef __ASM_I386_THERM_THROT_H__
+#define __ASM_I386_THERM_THROT_H__ 1
+
+#include <asm/types.h>
+
+struct therm_throt_event_info {
+	__u64 interval;
+	__u64 interval_count;
+	unsigned int cpu;
+};
+
+void __cpuinit thermal_throttle_count_init(unsigned int cpu);
+void thermal_throttle_inc_count(void);
+int thermal_throttle_process_event(struct therm_throt_event_info *evinfo,
+				   int curr);
+
+#endif /* __ASM_I386_THERM_THROT_H__ */
include/asm-x86_64/mce.h +4 -0
index d13687d..0ba1b4d 100644
@@ -3,6 +3,7 @@ #define _ASM_MCE_H 1
 
 #include <asm/ioctls.h>
 #include <asm/types.h>
+#include <asm/therm_throt.h>
 
 /* 
  * Machine Check support for x86
@@ -99,6 +100,9 @@ static inline void mce_amd_feature_init(
 }
 #endif
 
+void mce_log_thermal_throttle_event(__u64 status,
+				    struct therm_throt_event_info *evinfo);
+
 extern atomic_t mce_entry;
 
 #endif
include/asm-x86_64/therm_throt.h new file +1 -0
new file mode 100644
index 0000000..5aac059
@@ -0,0 +1 @@
+#include <asm-i386/therm_throt.h>
Signature · 1 line
1.4.2
#1 lore raw
Quoted 4 lines · Dima Zavin

- Maintain a count of thermal throttle events and export them to /sys/devices/system/cpu/cpuX/thermal_throttle/count - Factor out the thermal throttle event handling (i.e. logging) to a separate file, therm_throt.c under i386/kernel/cpu/mcheck.

Please make a separate patch to do any refactoring. Otherwise nobody can evaluate your changes. Then any other changes on top of it.

arch/i386/kernel/cpu/mcheck/therm_throt.c line 30

Quoted 7 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +7 -0
+static DEFINE_PER_CPU(atomic_t, tt_report_flag);
+static DEFINE_PER_CPU(struct timer_list, tt_report_timer);
+
+static DEFINE_PER_CPU(unsigned long, last_time);
+static DEFINE_PER_CPU(unsigned long, thermal_throttle_count);
+static DEFINE_PER_CPU(unsigned long, last_count);
+static DEFINE_PER_CPU(atomic_t, tt_enabled);

Do we really need all of this per CPU? I think global variables would be just fine.

Review

Quoted 4 lines
(quoted diff) +3 -0
+#define define_one_ro(_name)                                              
  \ +        static SYSDEV_ATTR(_name, 0444, show_##_name, NULL)
+
+#define define_show_func(name)                                            

The names are too generic

Quoted 6 lines

+/* thermal_throttle_count_init -
+ * Initialize all the internal state variables. The inits to 0 are not
+ * that important since the BSS is always 0 in kernel.
+ *
+ * This function should be called ONLY if thermal throttle detection
was + * enabled on this 'cpu'.

I don't think your comments are actually correct kerneldoc, so the scripts parsing it would break.

arch/i386/kernel/cpu/mcheck/therm_throt.c line 85

Quoted 6 lines

+ */

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +5 -0
+void __cpuinit thermal_throttle_count_init(unsigned int cpu)
+{
+	per_cpu(last_time, cpu)  = INITIAL_JIFFIES;
+	per_cpu(thermal_throttle_count, cpu) = 0;
+	per_cpu(last_count, cpu) = 0;

Most of this can be just a normal initializer for the variable

arch/i386/kernel/cpu/mcheck/therm_throt.c line 94

Quoted 8 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +8 -0
+	/* initialize the check-flag-timer */
+	init_timer(&per_cpu(tt_report_timer, cpu));
+	per_cpu(tt_report_timer, cpu).function = tt_report_timer_tick;
+	per_cpu(tt_report_timer, cpu).data = cpu;
+
+	/* mark us as enabled */
+	atomic_set(&per_cpu(tt_enabled, cpu), 1);
+	return;

Useless return;

arch/i386/kernel/cpu/mcheck/therm_throt.c line 104

Quoted 10 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +10 -0
+}
+
+/* thermal_throttle_inc_count -
+ *    This function is normally called by the thermal interrupt if
+ *    the code determines that we just entered into a thermal event.
+ */
+void thermal_throttle_inc_count(void)
+{
+	__get_cpu_var(thermal_throttle_count)++;
+}

Can you please just expand that in the caller. It looks like overabstraction.

Quoted 20 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +7 -0
+
+/* thermal_throttle_process_event -
+ *    This function is normally called by the thermal interrupt,
+ *    after the thermal event has been counted by calling
+ *    thermal_throttle_inc_count().
+ *
+ *    evinfo is a pointer to astructure where event info will be filled

in. + * Currently that's just the cpu#, time since last logged event,
and + * number of skipped (silently counted) events since last logged +
* event.
+ *
+ * Since the thermal interrupt normally gets called both when the
thermal + * event begins and once the event has ended, the caller passes
in a + * boolean 'curr' to specify whether or not it is currently IN the
+ * thermal event.
+ *
+ * Returns: 0 : Event should NOT be further logged, i.e. still in
+ * "timeout" from previous log message.
+ * 1 : Event should be logged, and a message has been printed.
+ */

arch/i386/kernel/cpu/mcheck/therm_throt.c line 134

Quoted 10 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +10 -0
+int thermal_throttle_process_event(struct therm_throt_event_info *evinfo,
+				   int curr)
+{
+	unsigned int cpu = smp_processor_id();
+	__u64 intvl;
+	__u64 intvl_cnt;
+
+	/* Prevent flooding the logs with thermal entries */
+	if (atomic_read(&__get_cpu_var(tt_report_flag)))
+		return 0;

Isn't the test the wrong way around?

arch/i386/kernel/cpu/mcheck/therm_throt.c line 139

Quoted 5 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +5 -0
+
+	/* set the flag, and add the timer that will clear it */
+	atomic_set(&__get_cpu_var(tt_report_flag), 1);
+	__get_cpu_var(tt_report_timer).expires = jiffies + CHECK_INTERVAL;
+	add_timer_on(&__get_cpu_var(tt_report_timer), cpu);

I'm not sure what you need the timer for. Can't just just drive that from the interrupt handlers and check time stamps to do rate limiting? Then another timer wouldn't be needed.

arch/i386/kernel/cpu/mcheck/therm_throt.c line 142

Quoted 2 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +2 -0
+	/* get the interval */
+	intvl = (jiffies - __get_cpu_var(last_time)) / HZ;

Are you sure this handles jiffies wrapping correctly?

arch/i386/kernel/cpu/mcheck/therm_throt.c line 191

Quoted 6 lines
(quoted diff) +2 -0
+#ifdef CONFIG_HOTPLUG_CPU
+static __cpuinit int thermal_throttle_remove_dev(struct sys_device *

sys_dev) +{

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +3 -0
+	sysfs_remove_group(&sys_dev->kobj, &thermal_throttle_attr_group);
+	return 0;
+}

You don't remove the timer?

Quoted 1 line

+/* Get notified when a cpu comes on/off. Be hotplug friendly. */

And don't initialize it? Better get rid of it anyways.

Quoted 23 lines

+static __cpuinit int thermal_throttle_cpu_callback(struct notifier_block
*nfb, + unsigned long action,

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +21 -0
+						   void *hcpu)
+{
+	unsigned int cpu = (unsigned long)hcpu;
+	struct sys_device *sys_dev;
+
+	sys_dev = get_cpu_sysdev(cpu);
+	switch (action) {
+		case CPU_ONLINE:
+			thermal_throttle_add_dev(sys_dev);
+			break;
+		case CPU_DEAD:
+			thermal_throttle_remove_dev(sys_dev);
+			break;
+	}ss
+	return NOTIFY_OK;
+}
+
+static struct notifier_block thermal_throttle_cpu_notifier =
+{
+	.notifier_call = thermal_throttle_cpu_callback,
+};

arch/i386/kernel/cpu/mcheck/therm_throt.c line 226

Quoted 10 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +10 -0
+#endif
+
+static __init int thermal_throttle_init_device(void)
+{
+	unsigned int cpu = 0;
+
+	/* connect to sysfs */
+	for_each_online_cpu(cpu) {
+		preempt_disable();
+		if (cpu_online(cpu)) {

If you put the preempt_disable around the loop you don't need a second check and you even fix a race with new CPUs.

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 515

Quoted 13 lines

+/* mce_log_thermal_throttle_event -
+ * Add an mce_log entry for the thermal throttle event 'evinfo'.
'status' + * should be whatever the user wants to put into the
mce.status field, + * and historically has been the register value of
the
+ * MSR_IA32_THERMAL_STATUS.
+ *
+ * This will typically be called from a thermal throttle interrupt
+ * if the event should be logged.
+ */

arch/x86_64/kernel/mce.c +3 -0
+void mce_log_thermal_throttle_event(__u64 status,
+				    struct therm_throt_event_info *evinfo)
+{

Instead of having this evinfo structure you could just directly fill in struct mces in the caller.

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 520

Quoted 5 lines
arch/x86_64/kernel/mce.c +5 -0
+	struct mce m;
+
+	memset(&m, 0, sizeof(m));
+	m.cpu = evinfo->cpu;
+	m.bank = MCE_THERMAL_BANK;

So how should user space distingush that event from other thermal events?

Quoted 4 lines
arch/x86_64/kernel/mce.c +4 -0
+	m.status = status;
+	rdtscll(m.tsc);
+
+	/* save the delta of the throttle count to let the consumer of this

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 534

Quoted 10 lines
arch/x86_64/kernel/mce.c +10 -0
+	 * information know how many throttle instances have been seen.
+	 *
+	 * 'misc' is organized below as follows:
+	 * bits 15:0  - Contains a capped value of thermal events for last
+	 *              interval.
+	 * bits 31:16 - Time in seconds to qualify the interval
+	 *              (capped at 0xffff)
+	 *
+	 * We cap events count at 16 bits to leave room in 'misc' for future
+	 * extensions to this report */

I trust you will be also submitting a mcelog patch to decode all this?

-Andi

#2 lore raw

Andi,

Quoted 3 lines

Please make a separate patch to do any refactoring. Otherwise nobody can evaluate your changes. Then any other changes on top of it.

Will do.

Quoted 10 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 30

Quoted 7 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +7 -0
+static DEFINE_PER_CPU(atomic_t, tt_report_flag);
+static DEFINE_PER_CPU(struct timer_list, tt_report_timer);
+
+static DEFINE_PER_CPU(unsigned long, last_time);
+static DEFINE_PER_CPU(unsigned long, thermal_throttle_count);
+static DEFINE_PER_CPU(unsigned long, last_count);
+static DEFINE_PER_CPU(atomic_t, tt_enabled);

Do we really need all of this per CPU? I think global variables would be just fine.

Each cpu can throttle at different times and have different counts, so that is necessary on a per cpu basis. Maybe the enable can be shared, since it is highly unlikely that one cpu will have thermal throttle enabled and others not. But there can be BIOS bugs where it might register it's own SMIs for some cpu's and not others.

Quoted 9 lines
Quoted 6 lines

+/* thermal_throttle_count_init -
+ * Initialize all the internal state variables. The inits to 0 are not
+ * that important since the BSS is always 0 in kernel.
+ *
+ * This function should be called ONLY if thermal throttle detection
was + * enabled on this 'cpu'.

I don't think your comments are actually correct kerneldoc, so the scripts parsing it would break.

My fault, I haven't pushed anything upstream before. Will fix.

Quoted 9 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 85

Quoted 7 lines

+ */

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +5 -0
+void __cpuinit thermal_throttle_count_init(unsigned int cpu)
+{
+     per_cpu(last_time, cpu)  = INITIAL_JIFFIES;
+     per_cpu(thermal_throttle_count, cpu) = 0;
+     per_cpu(last_count, cpu) = 0;

Most of this can be just a normal initializer for the variable

The zeros yes, I will remove that. What about the last_time to INITIAL_JIFFIES?

Quoted 4 lines
Quoted 4 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +4 -0
+}
+
+/* thermal_throttle_inc_count -
+ *    This function is normally called by the thermal interrupt if
Quoted 8 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 104

Quoted 6 lines

+ * the code determines that we just entered into a thermal event.
+ */

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +4 -0
+void thermal_throttle_inc_count(void)
+{
+     __get_cpu_var(thermal_throttle_count)++;
+}

Can you please just expand that in the caller. It looks like overabstraction.

Then the caller has to maintain the thermal counter, which is what I don't want. I wanted the managmente code to be separate. I can make this an inline in the header if you want to get rid of the function call.

Quoted 6 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 134

Quoted 4 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +4 -0
+
+     /* Prevent flooding the logs with thermal entries */
+     if (atomic_read(&__get_cpu_var(tt_report_flag)))
+             return 0;

Isn't the test the wrong way around?

No. When the flag is cleared, the event should be reported (this way the flag has the right initial value when its declared). If the flag is set, i skip the report. The timer will clear the flag.

Quoted 3 lines

I'm not sure what you need the timer for. Can't just just drive that from the interrupt handlers and check time stamps to do rate limiting? Then another timer wouldn't be needed.

Problem is that the timestamps are in jiffies. You could potentially have very long periods of time in between thermal events, assuming the box has a long uptime. On 32bit systems, if the time delta in jiffies between 2 events is >= 0x80000000, the time elapsed will be negative, and will prevent you from logging events until jiffies wrap all the way around (the counter will still increment though). The timer takes care of clearing the flag after 5 minutes.

Quoted 5 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 142

Quoted 3 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +2 -0
+     /* get the interval */
+     intvl = (jiffies - __get_cpu_var(last_time)) / HZ;

Are you sure this handles jiffies wrapping correctly?

Not sure. Will fix.

Quoted 8 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 191

Quoted 6 lines
(quoted diff) +2 -0
+#ifdef CONFIG_HOTPLUG_CPU
+static __cpuinit int thermal_throttle_remove_dev(struct sys_device *

sys_dev) +{

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +3 -0
+     sysfs_remove_group(&sys_dev->kobj, &thermal_throttle_attr_group);
+     return 0;
+}

You don't remove the timer?

Oops. Will fix. At this point, does the PER_CPU variable still exist?

Quoted 3 lines
Quoted 1 line

+/* Get notified when a cpu comes on/off. Be hotplug friendly. */

And don't initialize it? Better get rid of it anyways.

The timer is initialized in thermal_throttle_count_init.

Quoted 7 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 226

Quoted 4 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +4 -0
+     /* connect to sysfs */
+     for_each_online_cpu(cpu) {
+             preempt_disable();
+             if (cpu_online(cpu)) {

If you put the preempt_disable around the loop you don't need a second check and you even fix a race with new CPUs.

Thanks, will fix.

Quoted 8 lines
Quoted 8 lines

+/* mce_log_thermal_throttle_event -
+ * Add an mce_log entry for the thermal throttle event 'evinfo'.
'status' + * should be whatever the user wants to put into the
mce.status field, + * and historically has been the register value of
the
+ * MSR_IA32_THERMAL_STATUS.
+ *
+ * This will typically be called from a thermal throttle interrupt

Quoted 8 lines

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 515

Quoted 5 lines

+ * if the event should be logged.
+ */

arch/x86_64/kernel/mce.c +3 -0
+void mce_log_thermal_throttle_event(__u64 status,
+                                 struct therm_throt_event_info *evinfo)
+{

Instead of having this evinfo structure you could just directly fill in struct mces in the caller.

But the caller won't know what to fill the struct mce with since this function does the logic of figuring out the thermal event info. I can't have this function take "struct mce *" since that doesn't exist on i386. I could have it accept pointers to values as arguments, but that's messy.

Quoted 7 lines

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 520

Quoted 5 lines
arch/x86_64/kernel/mce.c +5 -0
+     struct mce m;
+
+     memset(&m, 0, sizeof(m));
+     m.cpu = evinfo->cpu;
+     m.bank = MCE_THERMAL_BANK;

So how should user space distingush that event from other thermal events?

What do you mean? This is already what x86_64 code does today for thermal throttle events.

Quoted 15 lines
Quoted 4 lines
arch/x86_64/kernel/mce.c +4 -0
+     m.status = status;
+     rdtscll(m.tsc);
+
+     /* save the delta of the throttle count to let the consumer of this

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 534

Quoted 10 lines
arch/x86_64/kernel/mce.c +10 -0
+      * information know how many throttle instances have been seen.
+      *
+      * 'misc' is organized below as follows:
+      * bits 15:0  - Contains a capped value of thermal events for last
+      *              interval.
+      * bits 31:16 - Time in seconds to qualify the interval
+      *              (capped at 0xffff)
+      *
+      * We cap events count at 16 bits to leave room in 'misc' for future
+      * extensions to this report */
Quoted 1 line

I trust you will be also submitting a mcelog patch to decode all this?

Oops. Yes, I will add a patch for that as well.

Thank you very much for your feedback.

--Dima

#3 lore raw
Quoted 11 lines · Dmitriy Zavin
Quoted 10 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 30

Quoted 7 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +7 -0
+static DEFINE_PER_CPU(atomic_t, tt_report_flag);
+static DEFINE_PER_CPU(struct timer_list, tt_report_timer);
+
+static DEFINE_PER_CPU(unsigned long, last_time);
+static DEFINE_PER_CPU(unsigned long, thermal_throttle_count);
+static DEFINE_PER_CPU(unsigned long, last_count);
+static DEFINE_PER_CPU(atomic_t, tt_enabled);

Do we really need all of this per CPU? I think global variables would be just fine.

Quoted 1 line

Each cpu can throttle at different times and have different counts, so

The counts should be per CPU, but everything else (timestamp, enabled) etc. can be just global. You just don't want the logs flooded with events, but if the rate limiting is global or local doesn't make much difference That would give much simpler code and I believe
is sufficient for basically everybody

Currently your patch is a bit too large for the relatively simple things it tries to attempt, so such ways to simplify it and slim it down are needed. If you have other ideas to make it simpler that would be appreciated too.

Quoted 11 lines
Quoted 8 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 85

Quoted 6 lines

+ */

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +5 -0
+void __cpuinit thermal_throttle_count_init(unsigned int cpu)
+{
+     per_cpu(last_time, cpu)  = INITIAL_JIFFIES;
+     per_cpu(thermal_throttle_count, cpu) = 0;
+     per_cpu(last_count, cpu) = 0;

Most of this can be just a normal initializer for the variable

The zeros yes, I will remove that. What about the last_time to INITIAL_JIFFIES?

Should work as a initializer too.

Quoted 4 lines

Then the caller has to maintain the thermal counter, which is what I don't want. I wanted the managmente code to be separate. I can make this an inline in the header if you want to get rid of the function call.

The function call doesn't matter, it is just that we don't want overabstracted kernel in the Linux kernel and your patch currently definitely has too many functions doing very simple things. That makes it harder to read etc.

Quoted 10 lines
Quoted 6 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 134

Quoted 4 lines
arch/i386/kernel/cpu/mcheck/therm_throt.c new file +4 -0
+
+     /* Prevent flooding the logs with thermal entries */
+     if (atomic_read(&__get_cpu_var(tt_report_flag)))
+             return 0;

Isn't the test the wrong way around?

No. When the flag is cleared, the event should be reported (this way the flag has the right initial value when its declared). If the flag is set, i skip the report. The timer will clear the flag.

Looks like a quite strange way to do things, but ok.

Quoted 11 lines
Quoted 3 lines

I'm not sure what you need the timer for. Can't just just drive that from the interrupt handlers and check time stamps to do rate limiting? Then another timer wouldn't be needed.

Problem is that the timestamps are in jiffies. You could potentially have very long periods of time in between thermal events, assuming the box has a long uptime. On 32bit systems, if the time delta in jiffies between 2 events is >= 0x80000000, the time elapsed will be negative, and will prevent you from logging events until jiffies wrap all the way around (the counter will still increment though). The timer takes care of clearing the flag after 5 minutes.

We have a 64bit jiffies for that now on 32bit too.

But Jiffies wrap is several years. Even if we get a bogus overheat event after two years it isn't really a issue. Please just get rid of the timer

Quoted 10 lines
Quoted 8 lines

arch/i386/kernel/cpu/mcheck/therm_throt.c line 191

Quoted 6 lines
(quoted diff) +2 -0
+#ifdef CONFIG_HOTPLUG_CPU
+static __cpuinit int thermal_throttle_remove_dev(struct sys_device *

sys_dev) +{

arch/i386/kernel/cpu/mcheck/therm_throt.c new file +3 -0
+     sysfs_remove_group(&sys_dev->kobj, &thermal_throttle_attr_group);
+     return 0;
+}

You don't remove the timer?

Oops. Will fix. At this point, does the PER_CPU variable still exist?

Yes, they always exist for all possible CPUs.

But better just get rid of the timer completely.

Quoted 17 lines
Quoted 8 lines
Quoted 8 lines

+/* mce_log_thermal_throttle_event -
+ * Add an mce_log entry for the thermal throttle event 'evinfo'.
'status' + * should be whatever the user wants to put into the
mce.status field, + * and historically has been the register value
of the
+ * MSR_IA32_THERMAL_STATUS.
+ *
+ * This will typically be called from a thermal throttle interrupt

Quoted 8 lines
Quoted 5 lines

+ * if the event should be logged.
+ */

(quoted diff) +2 -0
+void mce_log_thermal_throttle_event(__u64 status,
+                                 struct therm_throt_event_info

*evinfo) +{

Instead of having this evinfo structure you could just directly fill in struct mces in the caller.

Quoted 5 lines

But the caller won't know what to fill the struct mce with since this function does the logic of figuring out the thermal event info. I can't have this function take "struct mce *" since that doesn't exist on i386. I could have it accept pointers to values as arguments, but that's messy.

Then either define struct mce for i386 or use two different functions for i386/x86-64.

Quoted 11 lines
Quoted 8 lines

arch/x86_64/kernel/mce.c static struct miscdevice mce_log_device · line 520

Quoted 6 lines
arch/x86_64/kernel/mce.c +5 -0
+     struct mce m;
+
+     memset(&m, 0, sizeof(m));
+     m.cpu = evinfo->cpu;
+     m.bank = MCE_THERMAL_BANK;

So how should user space distingush that event from other thermal events?

What do you mean? This is already what x86_64 code does today for thermal throttle events.

It should be just clear where it came from since you added a new way to generate them.

-Andi

#4 lore raw

[dropping Zwane from CC since the address I have is wrong]

Quoted 7 lines
Quoted 1 line

Each cpu can throttle at different times and have different counts, so

The counts should be per CPU, but everything else (timestamp, enabled) etc. can be just global. You just don't want the logs flooded with events, but if the rate limiting is global or local doesn't make much difference That would give much simpler code and I believe
is sufficient for basically everybody

This will make you skip logging events for other CPUs once one has throttled, which is important to me. In that case, one has to rely on just the counts.

But I still think that it is important to report and rate limit on a per CPU basis, which is how it is already done in the upstream kernel (and has for a while), so it wouldn't be spamming logs any more than it already does.

Quoted 3 lines

Currently your patch is a bit too large for the relatively simple things it tries to attempt, so such ways to simplify it and slim it down are needed. If you have other ideas to make it simpler that would be appreciated too.

You are right, it was my newbie mistake. I am splitting it up into logical, smaller chunks that are easier to review.

Quoted 8 lines
Quoted 4 lines

Then the caller has to maintain the thermal counter, which is what I don't want. I wanted the managmente code to be separate. I can make this an inline in the header if you want to get rid of the function call.

The function call doesn't matter, it is just that we don't want overabstracted kernel in the Linux kernel and your patch currently definitely has too many functions doing very simple things. That makes it harder to read etc.

I will roll the counter bump into process_event() so the user just calls that.

Quoted 12 lines
Quoted 11 lines
Quoted 3 lines

I'm not sure what you need the timer for. Can't just just drive that from the interrupt handlers and check time stamps to do rate limiting? Then another timer wouldn't be needed.

Problem is that the timestamps are in jiffies. You could potentially have very long periods of time in between thermal events, assuming the box has a long uptime. On 32bit systems, if the time delta in jiffies between 2 events is >= 0x80000000, the time elapsed will be negative, and will prevent you from logging events until jiffies wrap all the way around (the counter will still increment though). The timer takes care of clearing the flag after 5 minutes.

Quoted 2 lines

But Jiffies wrap is several years. Even if we get a bogus overheat event after two years it isn't really a issue. Please just get rid of the timer

We won't get a bogus overheat event. The problem is that we will SKIP reporting that event, and further events alltogether, until jiffies pass the threshold. Then we will HAVE to rely on someone watching the counter(s) to know that something has happened.

I understand your uneasiness with the timer, and will try to find a different way.

Quoted 1 line

We have a 64bit jiffies for that now on 32bit too.

The problem with using jiffies_64 is that the time_before/time_after macros in linux/jiffies.h always typecast to "long" which is not 64 bits on 32bit systems. So it will get truncated, and behave as 32bits and won't solve the problem

Quoted 11 lines
Quoted 8 lines
Quoted 2 lines

Instead of having this evinfo structure you could just directly fill in struct mces in the caller.

But the caller won't know what to fill the struct mce with since this function does the logic of figuring out the thermal event info. I can't have this function take "struct mce *" since that doesn't exist on i386. I could have it accept pointers to values as arguments, but that's messy.

Then either define struct mce for i386 or use two different functions for i386/x86-64.

I'll add a patch to pull out mcelog related stuff into mce_log.c, and share that between i386/x86_64. Put mcelog.c to arch/i386/kernel/cpu/mcheck, and have x86_64 just compile that in directly. Does that sound like a workable solution? There's no need to maintain identical code in 2 places.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
arch/x86_64/kernel/mce.c +5 -0
+     struct mce m;
+
+     memset(&m, 0, sizeof(m));
+     m.cpu = evinfo->cpu;
+     m.bank = MCE_THERMAL_BANK;
Quoted 7 lines
Quoted 4 lines
Quoted 1 line

So how should user space distingush that event from other thermal events?

What do you mean? This is already what x86_64 code does today for thermal throttle events.

It should be just clear where it came from since you added a new way to generate them.

I didn't add a new way to generate them :). Detection/generation of the log entry is identical, just the actual code moved to a different file.

Thanks.

--Dima

#5 lore raw
Quoted 11 lines · Dmitriy Zavin
Quoted 7 lines
Quoted 1 line

Each cpu can throttle at different times and have different counts, so

The counts should be per CPU, but everything else (timestamp, enabled) etc. can be just global. You just don't want the logs flooded with events, but if the rate limiting is global or local doesn't make much difference That would give much simpler code and I believe
is sufficient for basically everybody

This will make you skip logging events for other CPUs once one has throttled, which is important to me. In that case, one has to rely on just the counts.

Ok if you want complex features like this you'll have to get rid of some other code in your patch.

One possibility is to keep only some state per CPU, but do the sys configuration globally, but that's probably not enough.

Currently the bloat:feature ratio is imho not good enough at least.

Quoted 6 lines
Quoted 3 lines

Currently your patch is a bit too large for the relatively simple things it tries to attempt, so such ways to simplify it and slim it down are needed. If you have other ideas to make it simpler that would be appreciated too.

You are right, it was my newbie mistake. I am splitting it up into logical, smaller chunks that are easier to review.

It's not just smaller chunks, the end result should be simple too

Quoted 6 lines
Quoted 1 line

We have a 64bit jiffies for that now on 32bit too.

The problem with using jiffies_64 is that the time_before/time_after macros in linux/jiffies.h always typecast to "long" which is not 64 bits on 32bit systems. So it will get truncated, and behave as 32bits and won't solve the problem

Then please submit a separate patch to add time_before/after64 instead of trying to work around that.

Quoted 9 lines
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines

Instead of having this evinfo structure you could just directly fill in struct mces in the caller.

But the caller won't know what to fill the struct mce with since this function does the logic of figuring out the thermal event info. I can't have this function take "struct mce *" since that doesn't exist on i386. I could have it accept pointers to values as arguments, but that's messy.

Quoted 6 lines
Quoted 2 lines

Then either define struct mce for i386 or use two different functions for i386/x86-64.

I'll add a patch to pull out mcelog related stuff into mce_log.c, and share that between i386/x86_64. Put mcelog.c to arch/i386/kernel/cpu/mcheck

Please keep it in x86-64.

Quoted 3 lines

, and have x86_64 just compile that in
directly. Does that sound like a workable solution? There's no need to maintain identical code in 2 places.

It depends on how ugly that patch is. Unification is fine as long as it improves something, but if it requires weird hacks again it's a net loss.

-Andi

#6 lore raw
Quoted 2 lines

Ok if you want complex features like this you'll have to get rid of some other code in your patch.

...

Quoted 1 line

Currently the bloat:feature ratio is imho not good enough at least.

You are right. I was trying to do too much, unnecessarily. The count is enough to be exported, and if someone cares to do interval monitoring (etc.) they can write a daemon that watches it (since the original patch was in second granularity, userspace can easily handle that). If it turns out that we do need interval tracking, that can be added at a later time. I'll have an updated series of patches to address the issues.

Quoted 10 lines
Quoted 7 lines
Quoted 2 lines

We have a 64bit jiffies for that now on 32bit too.

The problem with using jiffies_64 is that the time_before/time_after macros in linux/jiffies.h always typecast to "long" which is not 64 bits on 32bit systems. So it will get truncated, and behave as 32bits and won't solve the problem

Then please submit a separate patch to add time_before/after64 instead of trying to work around that.

Will do.

Quoted 16 lines
Quoted 9 lines
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines

Instead of having this evinfo structure you could just directly fill in struct mces in the caller.

But the caller won't know what to fill the struct mce with since this function does the logic of figuring out the thermal event info. I can't have this function take "struct mce *" since that doesn't exist on i386. I could have it accept pointers to values as arguments, but that's messy.

Quoted 6 lines
Quoted 2 lines

Then either define struct mce for i386 or use two different functions for i386/x86-64.

I'll add a patch to pull out mcelog related stuff into mce_log.c, and share that between i386/x86_64. Put mcelog.c to arch/i386/kernel/cpu/mcheck

Quoted 8 lines

Please keep it in x86-64.

Quoted 3 lines

, and have x86_64 just compile that in
directly. Does that sound like a workable solution? There's no need to maintain identical code in 2 places.

It depends on how ugly that patch is. Unification is fine as long as it improves something, but if it requires weird hacks again it's a net loss.

I decided against it. The patch would in fact be pretty ugly. Going back in archives, I saw there were several attempts to do this, and it was never successful. The pain of doing this would outweigh any use i have for it at the moment.

Thanks for your insights.

--Dima