Skip to contentSkip to conversation

2.4.1-ac UP-APIC updates

PATCH In review Mikael Pettersson 3 participants 4 messages Feb 20, 2001

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 2.4.1-ac UP-APIC updates

Alan,

This patch (against 2.4.1-ac19) includes the following UP-APIC updates:

* Power Management: If the kernel's UP-APIC code enabled the local APIC because the BIOS chose not to, it is imperative that the kernel also disables the local APIC before entering apm/acpi suspend mode. Failure to do this causes suspend failures and hard system hangs.

The patch fixes apm suspend problems at least on my ASUS P3B-F and Petr Vandrovec's A7V. (Petr had to use the "standard" UHCI support, the alternate "JE" one dies after resume.)

* Maciej Rozycki's SMP kernel on UP hardware fix: smpboot needs to call APIC_unit_uniprocessor() in order to activate the NMI watchdog.

* Code cleanups in nmi.c. Moved P6/K7 setup code to their own functions. Simplified the P6 code.

* NMI rate reduction for UP-APIC: the 100Hz default rate is excessive for normal systems, 1Hz suffices. It turns out we cannot start at 1Hz due to this interacting badly with check_nmi_watchdog() and the watchdog itself, so the rate is reduced after check_nmi_watchdog() is done.

Some people see much less than 100 NMI/s with the current code in 2.4.1-ac. I _believe_ it has to do with APM being enabled or not: with APM disabled the kernel will 'hlt' when idle, which will stop the CPU clock and thus the perfctr generating NMIs until the next timer irq.

/Mikael

arch/i386/kernel/apic.c +168 -4
diff -ruN linux-2.4.1-ac19/arch/i386/kernel/apic.c linux-2.4.1-ac19-upapic/arch/i386/kernel/apic.c
@@ -9,6 +9,7 @@
  *					and Rolf G. Tews
  *					for testing these extensively.
  *	Maciej W. Rozycki	:	Various updates and fixes.
+ *	Mikael Pettersson	:	Power Management for UP-APIC.
  */
 
 #include <linux/config.h>
@@ -379,14 +380,169 @@
 		setup_apic_nmi_watchdog();
 }
 
+#ifdef CONFIG_PM
+
+#include <linux/slab.h>
+#include <linux/pm.h>
+
+static struct {
+	/* 'active' is true if the local APIC was enabled by us and
+	   not the BIOS; this signifies that we are also responsible
+	   for disabling it before entering apm/acpi suspend */
+	int active;
+	/* 'perfctr_pmdev' is here because the current (2.4.1) PM
+	   callback system doesn't handle hierarchical dependencies */
+	struct pm_dev *perfctr_pmdev;
+	/* r/w apic fields */
+	unsigned int apic_id;
+	unsigned int apic_taskpri;
+	unsigned int apic_ldr;
+	unsigned int apic_dfr;
+	unsigned int apic_spiv;
+	unsigned int apic_lvtt;
+	unsigned int apic_lvtpc;
+	unsigned int apic_lvt0;
+	unsigned int apic_lvt1;
+	unsigned int apic_lvterr;
+	unsigned int apic_tmict;
+	unsigned int apic_tdcr;
+} apic_pm_state;
+
+static void apic_pm_suspend(void *data)
+{
+	unsigned int l, h;
+	unsigned long flags;
+
+	if (apic_pm_state.perfctr_pmdev)
+		pm_send(apic_pm_state.perfctr_pmdev, PM_SUSPEND, data);
+	apic_pm_state.apic_id = apic_read(APIC_ID);
+	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
+	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
+	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
+	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
+	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
+	apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
+	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
+	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
+	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
+	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
+	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
+	__save_flags(flags);
+	__cli();
+	disable_local_APIC();
+	rdmsr(MSR_IA32_APICBASE, l, h);
+	l &= ~MSR_IA32_APICBASE_ENABLE;
+	wrmsr(MSR_IA32_APICBASE, l, h);
+	__restore_flags(flags);
+}
+
+static void apic_pm_resume(void *data)
+{
+	unsigned int l, h;
+	unsigned long flags;
+
+	__save_flags(flags);
+	__cli();
+	rdmsr(MSR_IA32_APICBASE, l, h);
+	l &= ~MSR_IA32_APICBASE_BASE;
+	l |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE;
+	wrmsr(MSR_IA32_APICBASE, l, h);
+	apic_write(APIC_ID, apic_pm_state.apic_id);
+	apic_write(APIC_DFR, apic_pm_state.apic_dfr);
+	apic_write(APIC_LDR, apic_pm_state.apic_ldr);
+	apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
+	apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
+	apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
+	apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
+	apic_write(APIC_ESR, 0);
+	apic_read(APIC_ESR);
+	apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
+	apic_write(APIC_ESR, 0);
+	apic_read(APIC_ESR);
+	apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
+	apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
+	apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
+	apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
+	__restore_flags(flags);
+	if (apic_pm_state.perfctr_pmdev)
+		pm_send(apic_pm_state.perfctr_pmdev, PM_RESUME, data);
+}
+
+static int apic_pm_callback(struct pm_dev *dev, pm_request_t rqst, void *data)
+{
+	switch (rqst) {
+	case PM_SUSPEND:
+		apic_pm_suspend(data);
+		break;
+	case PM_RESUME:
+		apic_pm_resume(data);
+		break;
+	}
+	return 0;
+}
+
+/* perfctr driver should call this instead of pm_register() */
+struct pm_dev *apic_pm_register(pm_dev_t type,
+				unsigned long id,
+				pm_callback callback)
+{
+	struct pm_dev *dev;
+
+	if (!apic_pm_state.active)
+		return pm_register(type, id, callback);
+	if (apic_pm_state.perfctr_pmdev)
+		return NULL;	/* we're busy */
+	dev = kmalloc(sizeof(struct pm_dev), GFP_KERNEL);
+	if (dev) {
+		memset(dev, 0, sizeof(*dev));
+		dev->type = type;
+		dev->id = id;
+		dev->callback = callback;
+		apic_pm_state.perfctr_pmdev = dev;
+	}
+	return dev;
+}
+
+/* perfctr driver should call this instead of pm_unregister() */
+void apic_pm_unregister(struct pm_dev *dev)
+{
+	if (!apic_pm_state.active) {
+		pm_unregister(dev);
+	} else if (dev == apic_pm_state.perfctr_pmdev) {
+		apic_pm_state.perfctr_pmdev = NULL;
+		kfree(dev);
+	}
+}
+
+static void __init apic_pm_init1(void)
+{
+	/* can't pm_register() at this early stage in the boot process
+	   (causes an immediate reboot), so just set the flag */
+	apic_pm_state.active = 1;
+}
+
+static void __init apic_pm_init2(void)
+{
+	if (apic_pm_state.active)
+		pm_register(PM_SYS_DEV, 0, apic_pm_callback);
+}
+
+#else	/* CONFIG_PM */
+
+static inline void apic_pm_init1(void) { }
+static inline void apic_pm_init2(void) { }
+
+#endif	/* CONFIG_PM */
+
 /*
  * Detect and enable local APICs on non-SMP boards.
  * Original code written by Keir Fraser.
  */
 
-int __init detect_init_APIC (void)
+static int __init detect_init_APIC (void)
 {
 	u32 h, l, dummy, features;
+	int needs_pm = 0;
 	extern void get_cpu_vendor(struct cpuinfo_x86*);
 
 	/* Workaround for us being called before identify_cpu(). */
@@ -418,6 +574,7 @@
 			l &= ~MSR_IA32_APICBASE_BASE;
 			l |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE;
 			wrmsr(MSR_IA32_APICBASE, l, h);
+			needs_pm = 1;
 		}
 	}
 	/*
@@ -437,6 +594,9 @@
 
 	printk("Found and enabled local APIC!\n");
 
+	if (needs_pm)
+		apic_pm_init1();
+
 	return 0;
 
 no_apic:
@@ -893,10 +1053,10 @@
  * This initializes the IO-APIC and APIC hardware if this is
  * a UP kernel.
  */
-void __init APIC_init_uniprocessor (void)
+int __init APIC_init_uniprocessor (void)
 {
 	if (!smp_found_config && !cpu_has_apic)
-		return;
+		return -1;
 
 	/*
 	 * Complain if the BIOS pretends there is one.
@@ -904,7 +1064,7 @@
 	if (!cpu_has_apic && APIC_INTEGRATED(apic_version[boot_cpu_id])) {
 		printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
 			boot_cpu_id);
-		return;
+		return -1;
 	}
 
 	verify_local_APIC();
@@ -914,6 +1074,8 @@
 	phys_cpu_present_map = 1;
 	apic_write_around(APIC_ID, boot_cpu_id);
 
+	apic_pm_init2();
+
 	setup_local_APIC();
 
 	if (nmi_watchdog == NMI_LOCAL_APIC)
@@ -923,4 +1085,6 @@
 		setup_IO_APIC();
 #endif
 	setup_APIC_clocks();
+
+	return 0;
 }
arch/i386/kernel/nmi.c +141 -63

261 lines not on this page · raw message

arch/i386/kernel/smpboot.c +4 -2
diff -ruN linux-2.4.1-ac19/arch/i386/kernel/smpboot.c linux-2.4.1-ac19-upapic/arch/i386/kernel/smpboot.c
@@ -869,12 +869,15 @@
 	 * get out of here now!
 	 */
 	if (!smp_found_config) {
-		printk(KERN_NOTICE "SMP motherboard not detected. Using dummy APIC emulation.\n");
+		printk(KERN_NOTICE "SMP motherboard not detected.\n");
 #ifndef CONFIG_VISWS
 		io_apic_irqs = 0;
 #endif
 		cpu_online_map = phys_cpu_present_map = 1;
 		smp_num_cpus = 1;
+		if (APIC_init_uniprocessor())
+			printk(KERN_NOTICE "Local APIC not detected."
+					   " Using dummy APIC emulation.\n");
 		goto smp_done;
 	}
 
@@ -1020,4 +1023,3 @@
 smp_done:
 	zap_low_mappings();
 }
-
include/asm-i386/apic.h +9 -8
diff -ruN linux-2.4.1-ac19/include/asm-i386/apic.h linux-2.4.1-ac19-upapic/include/asm-i386/apic.h
@@ -2,6 +2,7 @@
 #define __ASM_APIC_H
 
 #include <linux/config.h>
+#include <linux/pm.h>
 #include <asm/apicdef.h>
 #include <asm/system.h>
 
@@ -69,7 +70,6 @@
 extern int verify_local_APIC (void);
 extern void cache_APIC_registers (void);
 extern void sync_Arb_IDs (void);
-extern int detect_init_APIC (void);
 extern void init_bsp_APIC (void);
 extern void setup_local_APIC (void);
 extern void init_apic_mappings (void);
@@ -77,18 +77,19 @@
 extern void setup_APIC_clocks (void);
 extern void setup_apic_nmi_watchdog (void);
 extern inline void nmi_watchdog_tick (struct pt_regs * regs);
-extern void APIC_init_uniprocessor (void);
+extern int APIC_init_uniprocessor (void);
+
+extern struct pm_dev *apic_pm_register(pm_dev_t, unsigned long, pm_callback);
+extern void apic_pm_unregister(struct pm_dev*);
 
 extern unsigned int apic_timer_irqs [NR_CPUS];
 extern int check_nmi_watchdog (void);
 
 extern unsigned int nmi_watchdog;
-enum nmi_watchdog_source_types {
-	NMI_NONE = 0,
-	NMI_IO_APIC = 1,
-	NMI_LOCAL_APIC = 2,
-	NMI_INVALID = 3
-};
+#define NMI_NONE	0
+#define NMI_IO_APIC	1
+#define NMI_LOCAL_APIC	2
+#define NMI_INVALID	3
 
 #endif /* CONFIG_X86_LOCAL_APIC */
 
include/asm-i386/msr.h +4 -1
diff -ruN linux-2.4.1-ac19/include/asm-i386/msr.h linux-2.4.1-ac19-upapic/include/asm-i386/msr.h
@@ -67,4 +67,7 @@
 #define		MSR_IA32_MC0_MISC_OFFSET	0x3
 #define		MSR_IA32_MC0_BANK_COUNT		0x4
 
-#endif
+#define MSR_K7_EVNTSEL0			0xC0010000
+#define MSR_K7_PERFCTR0			0xC0010004
+
+#endif /* __ASM_MSR_H */
#1 lore raw
Quoted 4 lines · Mikael Pettersson

* NMI rate reduction for UP-APIC: the 100Hz default rate is excessive for normal systems, 1Hz suffices. It turns out we cannot start at 1Hz due to this interacting badly with check_nmi_watchdog() and the watchdog itself, so the rate is reduced after check_nmi_watchdog() is done.

i dont like this one. 100 times a second makes absolutely no performance difference whatsoever - but eg. i'm driving kernel profiling from the NMI handler to get profiles of eg. IRQ handlers and other cli()-ed code areas.

another reason is that if NMIs intaract with anything else in the system, we'll have much better chances to hit it with 100 Hz than with 1 Hz.

Ingo

#2 lore raw
Quoted 3 lines

i dont like this one. 100 times a second makes absolutely no performance difference whatsoever - but eg. i'm driving kernel profiling from the NMI handler to get profiles of eg. IRQ handlers and other cli()-ed code areas.

So set it to 100Hz as a debugging option like slab debugging

#3 lore raw
Quoted 5 lines · Alan Cox
Quoted 3 lines

i dont like this one. 100 times a second makes absolutely no performance difference whatsoever - but eg. i'm driving kernel profiling from the NMI handler to get profiles of eg. IRQ handlers and other cli()-ed code areas.

So set it to 100Hz as a debugging option like slab debugging

my major gripe right now is that we still have bug reports that say that systems hang when using nmi_watchdog=1 and work if nmi_watchdog=0. Changing the NMI watchdog to be 1 Hz will make these bugreports "Linux hangs once a week" instead of a "Linux hangs after 1-2 hours", which is clearly hiding things and making debugging harder.

(and driving kernel-profiling from the NMI interrupt is a short-term patch, so there is just no point in going to 1 Hz right now just to go back to 100 Hz a few days later.)

the rest of the changes are excellent - it's only the 100 Hz NMI issue i have a problem with.

Ingo