Fix microcode-related suspend problem
PATCH 1/1 Fix microcode-related suspend problem
Hi,
This patch appeard on LMKL six days ago and there have not been any negative comments since then, so I think I can try to make it official.
---
From: Rafael J. Wysocki <[email protected]>
Fix the regression resulting from the recent change of suspend code ordering that causes systems based on Intel x86 CPUs using the microcode driver to hang during the resume.
The problem occurs since the microcode driver uses request_firmware() in its CPU hotplug notifier, which is called after tasks has been frozen and hangs. It can be fixed by telling the microcode driver to use the microcode stored in memory during the resume instead of trying to load it from disk.
--- arch/i386/kernel/microcode.c | 71 ++++++++++++++++++++++++++++++++++++++++--- include/linux/cpu.h | 2 + kernel/cpu.c | 32 +++++++++---------- 3 files changed, 85 insertions(+), 20 deletions(-)
arch/i386/kernel/microcode.c +67 -4
@@ -567,6 +567,53 @@ static int cpu_request_microcode(int cpureturn error; } +static int apply_microcode_on_cpu (int cpu) +{ +struct cpuinfo_x86 *c = cpu_data + cpu; +struct ucode_cpu_info *uci = ucode_cpu_info + cpu; +cpumask_t old; +unsigned int val[2 ]; +int err =0 ; + +if (!uci->mc) +return -EINVAL ; + + old = current->cpus_allowed; +set_cpus_allowed (current,cpumask_of_cpu (cpu)); + +/* Check if the microcode we have in memory matches the CPU */ +if (c->x86_vendor !=X86_VENDOR_INTEL || c->x86 <6 || +cpu_has (c,X86_FEATURE_IA64 ) || uci->sig !=cpuid_eax (0x00000001 )) + err = -EINVAL ; + +if (!err && ((c->x86_model >=5 ) || (c->x86 >6 ))) { +/* get processor flags from MSR 0x17 */ +rdmsr (MSR_IA32_PLATFORM_ID , val[0 ], val[1 ]); +if (uci->pf != (1 << ((val[1 ] >>18 ) &7 ))) + err = -EINVAL ; + } + +if (!err) { +wrmsr (MSR_IA32_UCODE_REV ,0 ,0 ); +/* see notes above for revision 1.07. Apparent chip bug */ +sync_core (); +/* get the current revision from MSR 0x8B */ +rdmsr (MSR_IA32_UCODE_REV , val[0 ], val[1 ]); +if (uci->rev != val[1 ]) + err = -EINVAL ; + } + +if (!err) +apply_microcode (cpu); +else +printk (KERN_ERR "microcode: Could not apply microcode to CPU%d:" +" sig=0x%x, pf=0x%x, rev=0x%x\n" , + cpu, uci->sig, uci->pf, uci->rev); + +set_cpus_allowed (current, old); +return err; +} +static void microcode_init_cpu (int cpu) {cpumask_t old;
@@ -577,7 +624,8 @@ static void microcode_init_cpu(int cpu)set_cpus_allowed (current,cpumask_of_cpu (cpu));mutex_lock (µcode_mutex);collect_cpu_info (cpu); -if (uci->valid && system_state ==SYSTEM_RUNNING ) +if (uci->valid && system_state ==SYSTEM_RUNNING && + !suspend_cpu_hotplug)cpu_request_microcode (cpu);mutex_unlock (µcode_mutex);set_cpus_allowed (current, old);
@@ -663,13 +711,24 @@ static int mc_sysdev_add(struct sys_devireturn 0 ;pr_debug ("Microcode:CPU %d added\n" , cpu); -memset (uci,0 ,sizeof (*uci)); +/* If suspend_cpu_hotplug is set, the system is resuming and we should +* use the data from before the suspend. +*/ +if (suspend_cpu_hotplug) { + err =apply_microcode_on_cpu (cpu); +if (err) +microcode_fini_cpu (cpu); + } +if (!uci->valid) +memset (uci,0 ,sizeof (*uci)); err =sysfs_create_group (&sys_dev->kobj, &mc_attr_group);if (err)return err; -microcode_init_cpu (cpu); +if (!uci->valid) +microcode_init_cpu (cpu); +return 0 ; }
@@ -680,7 +739,11 @@ static int mc_sysdev_remove(struct sys_dif (!cpu_online (cpu))return 0 ;pr_debug ("Microcode:CPU %d removed\n" , cpu); -microcode_fini_cpu (cpu); +/* If suspend_cpu_hotplug is set, the system is suspending and we should +* keep the microcode in memory for the resume. +*/ +if (!suspend_cpu_hotplug) +microcode_fini_cpu (cpu);sysfs_remove_group (&sys_dev->kobj, &mc_attr_group);return 0 ; }
include/linux/cpu.h +2 -0
@@ -127,6 +127,8 @@ static inline int cpu_is_offline(int cpu#endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_SUSPEND_SMP +extern int suspend_cpu_hotplug; +extern int disable_nonboot_cpus (void );extern void enable_nonboot_cpus (void );#else
kernel/cpu.c +16 -16
@@ -254,6 +254,12 @@ int __cpuinit cpu_up(unsigned int cpu) }#ifdef CONFIG_SUSPEND_SMP +/* Needed to prevent the microcode driver from requesting firmware in its CPU +* hotplug notifier during the suspend/resume. +*/ +int suspend_cpu_hotplug; +EXPORT_SYMBOL (suspend_cpu_hotplug); +static cpumask_t frozen_cpus;int disable_nonboot_cpus (void )
@@ -261,16 +267,8 @@ int disable_nonboot_cpus(void)int cpu, first_cpu, error =0 ;mutex_lock (&cpu_add_remove_lock); - first_cpu =first_cpu (cpu_present_map); -if (!cpu_online (first_cpu)) { - error =_cpu_up (first_cpu); -if (error) { -printk (KERN_ERR "Could not bring CPU%d up.\n" , - first_cpu); - goto out; - } - } - + suspend_cpu_hotplug =1 ; + first_cpu =first_cpu (cpu_online_map);/* We take down all of the non-boot CPUs in one shot to avoid races * with the userspace trying to use the CPU hotplug at the same time */
@@ -296,7 +294,7 @@ int disable_nonboot_cpus(void) }else {printk (KERN_ERR "Non-boot CPUs are not disabled\n" ); } -out: + suspend_cpu_hotplug =0 ;mutex_unlock (&cpu_add_remove_lock);return error; }
@@ -308,20 +306,22 @@ void enable_nonboot_cpus(void)/* Allow everyone to use the CPU hotplug again */ mutex_lock (&cpu_add_remove_lock); cpu_hotplug_disabled =0 ; -mutex_unlock (&cpu_add_remove_lock);if (cpus_empty (frozen_cpus)) -return ; + goto out; + suspend_cpu_hotplug =1 ;printk ("Enabling non-boot CPUs ...\n" );for_each_cpu_mask (cpu, frozen_cpus) { - error =cpu_up (cpu); + error =_cpu_up (cpu);if (!error) {printk ("CPU%d is up\n" , cpu);continue ; } -printk (KERN_WARNING "Error taking CPU%d up: %d\n" , - cpu, error); +printk (KERN_WARNING "Error taking CPU%d up: %d\n" , cpu, error); }cpus_clear (frozen_cpus); + suspend_cpu_hotplug =0 ; +out: + mutex_unlock(&cpu_add_remove_lock); }#endif
Quoted 6 lines · On Sat, 31 Mar 2007 22:04:15 +0200 "Rafael J. Wysocki" <[email protected]> wrote:
This patch appeard on LMKL six days ago and there have not been any negative comments since then, so I think I can try to make it official.
---
From: Rafael J. Wysocki <[email protected]>
Quoted 8 lines
Fix the regression resulting from the recent change of suspend code ordering that causes systems based on Intel x86 CPUs using the microcode driver to hang during the resume.
The problem occurs since the microcode driver uses
request_firmware()in its CPU hotplug notifier, which is called after tasks has been frozen and hangs. It can be fixed by telling the microcode driver to use the microcode stored in memory during the resume instead of trying to load it from disk.
CONFIG_SMP=n:
arch/i386/kernel/microcode.c: In function 'microcode_init_cpu':arch/i386/kernel/microcode.c:628: error: 'suspend_cpu_hotplug' undeclared (first use in this function) arch/i386/kernel/microcode.c:628: error: (Each undeclared identifier is reported only once arch/i386/kernel/microcode.c:628: error: for each function it appears in.) arch/i386/kernel/microcode.c: In function 'mc_sysdev_add':arch/i386/kernel/microcode.c:717: error: 'suspend_cpu_hotplug' undeclared (first use in this function) arch/i386/kernel/microcode.c: In function 'mc_sysdev_remove':arch/i386/kernel/microcode.c:745: error: 'suspend_cpu_hotplug' undeclared (first use in this function)
Given this, and the overall intrusiveness of the change, I'd worry about trying to get this into 2.6.21.
Quoted 12 lines · Andrew Morton
Quoted 10 lines · On Sat, 31 Mar 2007 22:04:15 +0200 "Rafael J. Wysocki" <[email protected]> wrote:
This patch appeard on LMKL six days ago and there have not been any negative comments since then, so I think I can try to make it official.
---
From: Rafael J. Wysocki <[email protected]>Fix the regression resulting from the recent change of suspend code ordering that causes systems based on Intel x86 CPUs using the microcode driver to hang during the resume.
Quoted 6 lines
Quoted 4 lines
The problem occurs since the microcode driver uses
request_firmware()in its CPU hotplug notifier, which is called after tasks has been frozen and hangs. It can be fixed by telling the microcode driver to use the microcode stored in memory during the resume instead of trying to load it from disk.
CONFIG_SMP=n:
Ah, sorry. I tend to forget about it ...
Quoted 11 lines
arch/i386/kernel/microcode.c: In function 'microcode_init_cpu':arch/i386/kernel/microcode.c:628: error: 'suspend_cpu_hotplug' undeclared (first use in this function)arch/i386/kernel/microcode.c:628: error: (Each undeclared identifier is reported only oncearch/i386/kernel/microcode.c:628: error: for each function it appears in.)arch/i386/kernel/microcode.c: In function 'mc_sysdev_add':arch/i386/kernel/microcode.c:717: error: 'suspend_cpu_hotplug' undeclared (first use in this function)arch/i386/kernel/microcode.c: In function 'mc_sysdev_remove':arch/i386/kernel/microcode.c:745: error: 'suspend_cpu_hotplug' undeclared (first use in this function)Given this, and the overall intrusiveness of the change, I'd worry about trying to get this into 2.6.21.
Well, in that case I'll try to fix it later in a slightly different way.
Quoted 28 lines · Andrew Morton
Quoted 6 lines · On Sat, 31 Mar 2007 22:04:15 +0200 "Rafael J. Wysocki" <[email protected]> wrote:
This patch appeard on LMKL six days ago and there have not been any negative comments since then, so I think I can try to make it official.
---
From: Rafael J. Wysocki <[email protected]>Quoted 8 lines
Fix the regression resulting from the recent change of suspend code ordering that causes systems based on Intel x86 CPUs using the microcode driver to hang during the resume.
The problem occurs since the microcode driver uses
request_firmware()in its CPU hotplug notifier, which is called after tasks has been frozen and hangs. It can be fixed by telling the microcode driver to use the microcode stored in memory during the resume instead of trying to load it from disk.
CONFIG_SMP=n:
arch/i386/kernel/microcode.c: In function 'microcode_init_cpu':arch/i386/kernel/microcode.c:628: error: 'suspend_cpu_hotplug' undeclared (first use in this function)arch/i386/kernel/microcode.c:628: error: (Each undeclared identifier is reported only oncearch/i386/kernel/microcode.c:628: error: for each function it appears in.)arch/i386/kernel/microcode.c: In function 'mc_sysdev_add':arch/i386/kernel/microcode.c:717: error: 'suspend_cpu_hotplug' undeclared (first use in this function)arch/i386/kernel/microcode.c: In function 'mc_sysdev_remove':arch/i386/kernel/microcode.c:745: error: 'suspend_cpu_hotplug' undeclared (first use in this function)
Quoted 2 lines
Given this, and the overall intrusiveness of the change, I'd worry about trying to get this into 2.6.21.
It fixes a regression, and we are still at least one month away from the release of 2.6.21.
2.6.20 was released with too many known regressions [1], let's not repeat this mistake with 2.6.21.
cu
Adrian
[1] at least the CONFIG_USB_SUSPEND powerdown regression and the pktcdvd+libata not working regression shouldn't have been in a released kernel
Signature · 4 lines
"Is there not promise of rain?" Ling Tan asked suddenly out
of the darkness. There had been need of rain for many days.
"Only a promise," Lao Er said.
Pearl S. Buck - Dragon SeedQuoted 29 lines · Adrian Bunk
Quoted 28 lines · Andrew Morton
Quoted 6 lines · On Sat, 31 Mar 2007 22:04:15 +0200 "Rafael J. Wysocki" <[email protected]> wrote:
This patch appeard on LMKL six days ago and there have not been any negative comments since then, so I think I can try to make it official.
---
From: Rafael J. Wysocki <[email protected]>Quoted 8 lines
Fix the regression resulting from the recent change of suspend code ordering that causes systems based on Intel x86 CPUs using the microcode driver to hang during the resume.
The problem occurs since the microcode driver uses
request_firmware()in its CPU hotplug notifier, which is called after tasks has been frozen and hangs. It can be fixed by telling the microcode driver to use the microcode stored in memory during the resume instead of trying to load it from disk.
CONFIG_SMP=n:
arch/i386/kernel/microcode.c: In function 'microcode_init_cpu':arch/i386/kernel/microcode.c:628: error: 'suspend_cpu_hotplug' undeclared (first use in this function)arch/i386/kernel/microcode.c:628: error: (Each undeclared identifier is reported only oncearch/i386/kernel/microcode.c:628: error: for each function it appears in.)arch/i386/kernel/microcode.c: In function 'mc_sysdev_add':arch/i386/kernel/microcode.c:717: error: 'suspend_cpu_hotplug' undeclared (first use in this function)arch/i386/kernel/microcode.c: In function 'mc_sysdev_remove':arch/i386/kernel/microcode.c:745: error: 'suspend_cpu_hotplug' undeclared (first use in this function)
Quoted 8 lines
Quoted 2 lines
Given this, and the overall intrusiveness of the change, I'd worry about trying to get this into 2.6.21.
It fixes a regression, and we are still at least one month away from the release of 2.6.21.
2.6.20 was released with too many known regressions [1], let's not repeat this mistake with 2.6.21.
Okay, the appended version of the patch compiles with CONFIG_SMP unset too.
I think it could be done in a more elegant way, but that would require some additional infrastructure, which definitely is not 2.6.21 material.
---
From: Rafael J. Wysocki <[email protected]>
Fix the regression resulting from the recent change of suspend code ordering that causes systems based on Intel x86 CPUs using the microcode driver to hang during the resume.
The problem occurs since the microcode driver uses request_firmware() in its CPU hotplug notifier, which is called after tasks has been frozen and hangs. It can be fixed by telling the microcode driver to use the microcode stored in memory during the resume instead of trying to load it from disk.
--- arch/i386/kernel/microcode.c | 71 ++++++++++++++++++++++++++++++++++++++++--- include/linux/cpu.h | 4 ++ kernel/cpu.c | 32 +++++++++---------- 3 files changed, 87 insertions(+), 20 deletions(-)
arch/i386/kernel/microcode.c +67 -4
@@ -567,6 +567,53 @@ static int cpu_request_microcode(int cpureturn error; } +static int apply_microcode_on_cpu (int cpu) +{ +struct cpuinfo_x86 *c = cpu_data + cpu; +struct ucode_cpu_info *uci = ucode_cpu_info + cpu; +cpumask_t old; +unsigned int val[2 ]; +int err =0 ; + +if (!uci->mc) +return -EINVAL ; + + old = current->cpus_allowed; +set_cpus_allowed (current,cpumask_of_cpu (cpu)); + +/* Check if the microcode we have in memory matches the CPU */ +if (c->x86_vendor !=X86_VENDOR_INTEL || c->x86 <6 || +cpu_has (c,X86_FEATURE_IA64 ) || uci->sig !=cpuid_eax (0x00000001 )) + err = -EINVAL ; + +if (!err && ((c->x86_model >=5 ) || (c->x86 >6 ))) { +/* get processor flags from MSR 0x17 */ +rdmsr (MSR_IA32_PLATFORM_ID , val[0 ], val[1 ]); +if (uci->pf != (1 << ((val[1 ] >>18 ) &7 ))) + err = -EINVAL ; + } + +if (!err) { +wrmsr (MSR_IA32_UCODE_REV ,0 ,0 ); +/* see notes above for revision 1.07. Apparent chip bug */ +sync_core (); +/* get the current revision from MSR 0x8B */ +rdmsr (MSR_IA32_UCODE_REV , val[0 ], val[1 ]); +if (uci->rev != val[1 ]) + err = -EINVAL ; + } + +if (!err) +apply_microcode (cpu); +else +printk (KERN_ERR "microcode: Could not apply microcode to CPU%d:" +" sig=0x%x, pf=0x%x, rev=0x%x\n" , + cpu, uci->sig, uci->pf, uci->rev); + +set_cpus_allowed (current, old); +return err; +} +static void microcode_init_cpu (int cpu) {cpumask_t old;
@@ -577,7 +624,8 @@ static void microcode_init_cpu(int cpu)set_cpus_allowed (current,cpumask_of_cpu (cpu));mutex_lock (µcode_mutex);collect_cpu_info (cpu); -if (uci->valid && system_state ==SYSTEM_RUNNING ) +if (uci->valid && system_state ==SYSTEM_RUNNING && + !suspend_cpu_hotplug)cpu_request_microcode (cpu);mutex_unlock (µcode_mutex);set_cpus_allowed (current, old);
@@ -663,13 +711,24 @@ static int mc_sysdev_add(struct sys_devireturn 0 ;pr_debug ("Microcode:CPU %d added\n" , cpu); -memset (uci,0 ,sizeof (*uci)); +/* If suspend_cpu_hotplug is set, the system is resuming and we should +* use the data from before the suspend. +*/ +if (suspend_cpu_hotplug) { + err =apply_microcode_on_cpu (cpu); +if (err) +microcode_fini_cpu (cpu); + } +if (!uci->valid) +memset (uci,0 ,sizeof (*uci)); err =sysfs_create_group (&sys_dev->kobj, &mc_attr_group);if (err)return err; -microcode_init_cpu (cpu); +if (!uci->valid) +microcode_init_cpu (cpu); +return 0 ; }
@@ -680,7 +739,11 @@ static int mc_sysdev_remove(struct sys_dif (!cpu_online (cpu))return 0 ;pr_debug ("Microcode:CPU %d removed\n" , cpu); -microcode_fini_cpu (cpu); +/* If suspend_cpu_hotplug is set, the system is suspending and we should +* keep the microcode in memory for the resume. +*/ +if (!suspend_cpu_hotplug) +microcode_fini_cpu (cpu);sysfs_remove_group (&sys_dev->kobj, &mc_attr_group);return 0 ; }
include/linux/cpu.h +4 -0
@@ -127,9 +127,13 @@ static inline int cpu_is_offline(int cpu#endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_SUSPEND_SMP +extern int suspend_cpu_hotplug; +extern int disable_nonboot_cpus (void );extern void enable_nonboot_cpus (void );#else +#define suspend_cpu_hotplug 0 +static inline int disable_nonboot_cpus (void ) {return 0 ; }static inline void enable_nonboot_cpus (void ) {}#endif
kernel/cpu.c +16 -16
@@ -254,6 +254,12 @@ int __cpuinit cpu_up(unsigned int cpu) }#ifdef CONFIG_SUSPEND_SMP +/* Needed to prevent the microcode driver from requesting firmware in its CPU +* hotplug notifier during the suspend/resume. +*/ +int suspend_cpu_hotplug; +EXPORT_SYMBOL (suspend_cpu_hotplug); +static cpumask_t frozen_cpus;int disable_nonboot_cpus (void )
@@ -261,16 +267,8 @@ int disable_nonboot_cpus(void)int cpu, first_cpu, error =0 ;mutex_lock (&cpu_add_remove_lock); - first_cpu =first_cpu (cpu_present_map); -if (!cpu_online (first_cpu)) { - error =_cpu_up (first_cpu); -if (error) { -printk (KERN_ERR "Could not bring CPU%d up.\n" , - first_cpu); - goto out; - } - } - + suspend_cpu_hotplug =1 ; + first_cpu =first_cpu (cpu_online_map);/* We take down all of the non-boot CPUs in one shot to avoid races * with the userspace trying to use the CPU hotplug at the same time */
@@ -296,7 +294,7 @@ int disable_nonboot_cpus(void) }else {printk (KERN_ERR "Non-boot CPUs are not disabled\n" ); } -out: + suspend_cpu_hotplug =0 ;mutex_unlock (&cpu_add_remove_lock);return error; }
@@ -308,20 +306,22 @@ void enable_nonboot_cpus(void)/* Allow everyone to use the CPU hotplug again */ mutex_lock (&cpu_add_remove_lock); cpu_hotplug_disabled =0 ; -mutex_unlock (&cpu_add_remove_lock);if (cpus_empty (frozen_cpus)) -return ; + goto out; + suspend_cpu_hotplug =1 ;printk ("Enabling non-boot CPUs ...\n" );for_each_cpu_mask (cpu, frozen_cpus) { - error =cpu_up (cpu); + error =_cpu_up (cpu);if (!error) {printk ("CPU%d is up\n" , cpu);continue ; } -printk (KERN_WARNING "Error taking CPU%d up: %d\n" , - cpu, error); +printk (KERN_WARNING "Error taking CPU%d up: %d\n" , cpu, error); }cpus_clear (frozen_cpus); + suspend_cpu_hotplug =0 ; +out: + mutex_unlock(&cpu_add_remove_lock); }#endif