x86: Disable x2apic if nox2apic is specified or intr-remap can not be enabled
PATCH 1/1 x86: Disable x2apic if nox2apic is specified or intr-remap can not be enabled
For
1. x2apic preenabled system
2. first kernel have x2apic enabled, and try to kexec second kernel with "nox2apic"
Will put back cpu with apic id < 255 into xapic mode, instead of panic.
-v2: use variable x2apic_disabled instead of variable nox2apic, Suggested by Thomas
update x2apic_supported with x2apic_disabled, Suggested by Thomas
-v3: add checking for boot cpu apic id > 255. in that case will just panic
--- pointed out by Suresh.
-v4: according to Ingo, for x2apic pre-enabled system, if intr-remap can not
be enabled, try to disable x2apic instead of panic and request to specify nox2apic for next boot.
-v5: fix the lapic mapping for falling path.
-v6: add warning when check_timer fails in strange case when BIOS enable x2apic
and intr_remapping but does not provide dmar table. requested by Suresh.
-v7: fix compiling with x2apic_preenabled
-v8: fix compiling with x2apic_disabled assigned with !X86_X2APIC
--- arch/x86/include/asm/apic.h | 9 +++- arch/x86/include/asm/apicdef.h | 1 arch/x86/include/asm/processor.h | 1 arch/x86/kernel/acpi/boot.c | 10 +++- arch/x86/kernel/apic/apic.c | 79 ++++++++++++++++++++++++++---------- arch/x86/kernel/apic/apic_flat_64.c | 7 ++- arch/x86/kernel/apic/io_apic.c | 4 + arch/x86/kernel/cpu/topology.c | 21 +++++++++ arch/x86/mm/srat.c | 7 ++- 9 files changed, 112 insertions(+), 27 deletions(-)
arch/x86/include/asm/apic.h +8 -2
@@ -176,6 +176,8 @@ static inline u64 native_x2apic_icr_read }extern int x2apic_phys; +extern int x2apic_preenabled; +extern int x2apic_disabled;extern void check_x2apic (void );extern void enable_x2apic (void );extern void x2apic_icr_write (u32 low,u32 id);
@@ -183,7 +185,7 @@ static inline int x2apic_enabled(void) {u64 msr; -if (!cpu_has_x2apic) +if (!cpu_has_x2apic || x2apic_disabled)return 0 ;rdmsrl (MSR_IA32_APICBASE , msr);
@@ -192,12 +194,15 @@ static inline int x2apic_enabled(void) return 0; } -#define x2apic_supported () (cpu_has_x2apic) +#define x2apic_supported () (cpu_has_x2apic && !x2apic_disabled)static inline void x2apic_force_phys (void ) { x2apic_phys =1 ; }#else +static inline void disable_x2apic (void ) +{ +}static inline void check_x2apic (void ) { }
@@ -214,6 +219,7 @@ static inline void x2apic_force_phys(voi#define x2apic_preenabled 0#define x2apic_supported () 0 +#define x2apic_disabled 1#endif extern void enable_IR_x2apic (void );
arch/x86/kernel/acpi/boot.c +8 -2
@@ -219,6 +219,8 @@ static int __init acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end) {struct acpi_madt_local_x2apic *processor =NULL ; +int apic_id; +u8 enabled; processor = (struct acpi_madt_local_x2apic *)header;
@@ -227,6 +229,8 @@ acpi_parse_x2apic(struct acpi_subtable_hacpi_table_print_madt_entry (header); + apic_id = processor->local_apic_id; + enabled = processor->lapic_flags &ACPI_MADT_ENABLED ;#ifdef CONFIG_X86_X2APIC /* * We need to register disabled CPU as well to permit
@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_h* to not preallocating memory for all NR_CPUS * when we use CPU hotplug. */ -acpi_register_lapic (processor->local_apic_id,/* APIC ID */ - processor->lapic_flags &ACPI_MADT_ENABLED ); +if (x2apic_disabled && (apic_id >=0xff ) && enabled) + printk(KERN_WARNING PREFIX "x2apic entry ignored\n"); +else +acpi_register_lapic (apic_id, enabled);#else printk(KERN_WARNING PREFIX "x2apic entry ignored\n");#endif
arch/x86/kernel/apic/apic.c +58 -21
133 lines not on this page · raw message
arch/x86/mm/srat.c +6 -1
@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct acif ((pa->flags &ACPI_SRAT_CPU_ENABLED ) ==0 )return ; pxm = pa->proximity_domain; + apic_id = pa->apic_id; +if (x2apic_disabled && (apic_id >=0xff )) { +printk (KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n" , + pxm, apic_id); +return ; + } node =setup_node (pxm);if (node <0 ) {printk (KERN_ERR "SRAT: Too many proximity domains %x\n" , pxm);
@@ -76,7 +82,6 @@ acpi_numa_x2apic_affinity_init(struct acreturn ; } - apic_id = pa->apic_id;if (apic_id >=MAX_LOCAL_APIC ) {printk (KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n" , pxm, apic_id, node);return ;
arch/x86/include/asm/apicdef.h +1 -0
@@ -144,6 +144,7 @@#define APIC_BASE (fix_to_virt(FIX_APIC_BASE))#define APIC_BASE_MSR 0x800 +#define XAPIC_ENABLE (1UL << 11)#define X2APIC_ENABLE (1UL << 10)#ifdef CONFIG_X86_32
arch/x86/include/asm/processor.h +1 -0
@@ -168,6 +168,7 @@ extern void init_scattered_cpuid_featureextern unsigned int init_intel_cacheinfo (struct cpuinfo_x86 *c);extern unsigned short num_cache_leaves; +u32 x2apic_cpuid_initial_apicid (void );extern void detect_extended_topology (struct cpuinfo_x86 *c);extern void detect_ht (struct cpuinfo_x86 *c);
arch/x86/kernel/cpu/topology.c +21 -0
@@ -21,6 +21,27 @@#define BITS_SHIFT_NEXT_LEVEL (eax) ((eax) & 0x1f)#define LEVEL_MAX_SIBLINGS (ebx) ((ebx) & 0xffff) +u32 x2apic_cpuid_initial_apicid (void ) +{ +unsigned int eax, ebx, ecx, edx; + +if (boot_cpu_data.cpuid_level <0xb ) +return 0 ; + +cpuid_count (0xb ,SMT_LEVEL , &eax, &ebx, &ecx, &edx); + +/* +* check if the cpuid leaf 0xb is actually implemented. +*/ +if (ebx ==0 || (LEAFB_SUBTYPE (ecx) !=SMT_TYPE )) +return 0 ; + +/* +* initial apic id, which also represents 32-bit extended x2apic id. +*/ +return edx; +} +/* * Check for extended topology enumeration cpuid leaf 0xb and if it * exists, use it for populating initial_apicid and cpu topology
arch/x86/kernel/apic/apic_flat_64.c +6 -1
@@ -171,9 +171,14 @@ static int flat_phys_pkg_id(int initial_return initial_apic_id >> index_msb; } +static int flat_probe (void ) +{ +return 1 ; +} +static struct apic apic_flat = { .name ="flat" , - .probe =NULL , + .probe = flat_probe, .acpi_madt_oem_check = flat_acpi_madt_oem_check, .apic_id_registered = flat_apic_id_registered,
arch/x86/kernel/apic/io_apic.c +4 -0
@@ -2947,6 +2947,10 @@ static inline void __init check_timer(vo }local_irq_disable ();apic_printk (APIC_QUIET ,KERN_INFO "..... failed :(.\n" ); +if (x2apic_preenabled && !x2apic_mode) +pr_info ("Your system has x2apic pre-enabled, but kernel can not enable intr-remapping.\n" +"So kernel try to disable x2apic, but interrupt still have problem.\n" +"You could disable x2apic from BIOS setup\n" );panic ("IO-APIC + timer doesn't work! Boot with apic=debug and send a " "report. Then try booting with the 'noapic' option.\n" ); out:
Quoted 6 lines · Yinghai Lu
For
1. x2apic preenabled system
2. first kernel have x2apic enabled, and try to kexec second kernel with "nox2apic"Will put back cpu with apic id < 255 into xapic mode, instead of panic.
Yinghai, I didn't get a chance to review this patch completely. But few things.
a. Need to split this patch into multiple patches, specially the cpuid mechanism to get the initial apic id, flat_probe etc. While I understand the reason behind the flat_probe change, it isn't quite obvious from this changelog.
b. for nox2apic, as you try to disable x2apic even if the bios has enabled it along with interrupt-remapping, this will probably not work unless we disable interrupt-remapping. So either we need to ignore the nox2apic when OS can find that interrupt-remapping is already enabled or disable interrupt-remapping. Let me think a bit more about this and get back to you by monday with detailed review of this.
thanks,
suresh
Yinghai, Here is the detailed review of your patch:
Quoted 14 lines · Yinghai Lu
Index:
linux-2.6/arch/x86/include/asm/apic.h
===================================================================arch/x86/include/asm/apic.h +2
-0@@ -176,6 +176,8 @@ static inline u64 native_x2apic_icr_read }extern int x2apic_phys; +extern int x2apic_preenabled; +extern int x2apic_disabled;extern void check_x2apic (void );extern void enable_x2apic (void );extern void x2apic_icr_write (u32 low,u32 id);
arch/x86/include/asm/apic.h static inline int x2apic_enabled(void) · line 191
Quoted 8 lines
arch/x86/include/asm/apic.h +1
-1@@ -183,7 +185,7 @@ static inline int x2apic_enabled(void) {u64 msr; -if (!cpu_has_x2apic) +if (!cpu_has_x2apic || x2apic_disabled)return 0 ;rdmsrl (MSR_IA32_APICBASE , msr);
Not sure why we need x2apic_disabled check here.
arch/x86/include/asm/apic.h static inline int x2apic_enabled(void) · line 197
Quoted 6 lines
arch/x86/include/asm/apic.h +1
-1@@ -192,12 +194,15 @@ static inline int x2apic_enabled(void)return 0 ; } -#define x2apic_supported () (cpu_has_x2apic) +#define x2apic_supported () (cpu_has_x2apic && !x2apic_disabled)
Here also not sure why we need this. Right thing is to clear the x2apic capability from the cpuid features when the user selects nox2apic.
arch/x86/include/asm/apic.h acpi_parse_x2apic(struct acpi_subtable_h · line 243
Quoted 8 lines
arch/x86/include/asm/apic.h +2
-2@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_h* to not preallocating memory for all NR_CPUS * when we use CPU hotplug. */ -acpi_register_lapic (processor->local_apic_id,/* APIC ID */ - processor->lapic_flags &ACPI_MADT_ENABLED ); +if (x2apic_disabled && (apic_id >=0xff ) && enabled) + printk(KERN_WARNING PREFIX "x2apic entry ignored\n");
x2apic MADT entries are specifically for apic-id's >= 255. So no need to check for the value again. We can ignore all the x2apic entries. Also print once indicating why all x2apic entries are being ignored should be enough.
Also when we disable x2apic in the case not initiated by the user (like no dmar tables etc), we would have already parsed the MADT entries but we need to skip those entries with id >= 255 duing smp boot. I don't think I see that code in this patch.
Quoted 5 lines
arch/x86/include/asm/apic.h +0
-3static __init int setup_nox2apic (char *str) { -if (x2apic_enabled ()) { -pr_warning ("Bios already enabled x2apic, " -"can't enforce nox2apic" );
arch/x86/include/asm/apic.h acpi_parse_x2apic(struct acpi_subtable_h
Quoted 10 lines
arch/x86/include/asm/apic.h +4
-3-return 0 ; - } +if (x2apic_enabled ()) +pr_warning ("Bios already enabled x2apic, will disable it" ); + + x2apic_disabled =1 ; -setup_clear_cpu_cap (X86_FEATURE_X2APIC );return 0 ; }
Not sure why you removed this clear_cpu_cap here. Right thing to do here is to delay the clearing when x2apic is pre-enabled by the bios and clear it here when x2apic is not pre-enabled by the bios.
arch/x86/include/asm/apic.h
Quoted 11 lines
arch/x86/include/asm/apic.h +11
-0+ +static void disable_x2apic (void ) +{ +int msr, msr2; + +if (!cpu_has_x2apic) +return ; + +rdmsr (MSR_IA32_APICBASE , msr, msr2); +if (msr &X2APIC_ENABLE ) { +u32 x2apic_id =x2apic_cpuid_initial_apicid ();
I think we should be able to do read_apic_id() here and thus avoid the need for getting apicid from cpuid.
Quoted 12 lines
arch/x86/include/asm/apic.h +12
-0+ +if (x2apic_id >255 ) +panic ("Can not disable x2apic, id: %08x\n" , x2apic_id); + +pr_info ("Disabling x2apic\n" ); +/* +* Need to disable xapic and x2apic at the same time at first +* then enable xapic +*/ +wrmsr (MSR_IA32_APICBASE , msr & ~(X2APIC_ENABLE |XAPIC_ENABLE ), +0 ); +wrmsr (MSR_IA32_APICBASE , msr & ~X2APIC_ENABLE ,0 );
arch/x86/include/asm/apic.h
Quoted 10 lines
arch/x86/include/asm/apic.h +8
-0+ + x2apic_disabled =1 ; + } +}void check_x2apic (void ) { +if (x2apic_disabled) { +disable_x2apic (); +return ; + }
We shouldn't call disable_x2apic() from here. We should be following the same sequence, like masking interrupts at the cpu, pic, io-apic level before we disable x2apic and re-enable xapic. So the right place to do this disable is at enable_IR_x2apic().
arch/x86/include/asm/apic.h void enable_x2apic(void) · line 1484
Quoted 6 lines
arch/x86/include/asm/apic.h +2
-0@@ -1449,6 +1480,11 @@ void enable_x2apic(void) {int msr, msr2; +if (x2apic_disabled) { +disable_x2apic ();
enable_x2apic() gets called from both BP and AP. AP's need not do the apic-id being >= 255 checks etc. As that is done during smp bringup (which is missing currently from the current patch) or during MADT parsing. So all we need to do is a simple x2apic->xapic conversion.
arch/x86/mm/srat.c acpi_numa_x2apic_affinity_init(struct ac · line 73
Quoted 12 lines
arch/x86/include/asm/apic.h +0
-0#ifdef CONFIG_X86_64 Index:
linux-2.6/arch/x86/mm/srat.c
===================================================================arch/x86/mm/srat.c +2
-0@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct acif ((pa->flags &ACPI_SRAT_CPU_ENABLED ) ==0 )return ; pxm = pa->proximity_domain; + apic_id = pa->apic_id; +if (x2apic_disabled && (apic_id >=0xff )) {
Same comments like in the above MADT parsing.
arch/x86/kernel/cpu/topology.c line 25
Quoted 11 lines
Index:
linux-2.6/arch/x86/kernel/cpu/topology.c
===================================================================arch/x86/kernel/cpu/topology.c +2
-0@@ -21,6 +21,27 @@#define BITS_SHIFT_NEXT_LEVEL (eax) ((eax) & 0x1f)#define LEVEL_MAX_SIBLINGS (ebx) ((ebx) & 0xffff) +u32 x2apic_cpuid_initial_apicid (void ) +{
I think we can do with out this function.
While reviewing the code, I ended up doing some of this cleanup of your patch. I have appended the modified version. Can you please incorporate the smpboot checks before the AP bringup (apic-id should be less than 255 in !x2apic_mode etc) and split the patch into couple of patches (apic_probe etc) along with other needed cleanups.
Thanks.
--- arch/x86/include/asm/apic.h | 6 ++ arch/x86/include/asm/apicdef.h | 1 + arch/x86/kernel/acpi/boot.c | 10 +++- arch/x86/kernel/apic/apic.c | 92 ++++++++++++++++++++++++++--------- arch/x86/kernel/apic/apic_flat_64.c | 7 ++- arch/x86/kernel/apic/io_apic.c | 4 ++ arch/x86/mm/srat.c | 7 ++- 7 files changed, 99 insertions(+), 28 deletions(-)
arch/x86/include/asm/apic.h +6 -0
@@ -176,6 +176,8 @@ static inline u64 native_x2apic_icr_read(void) }extern int x2apic_phys; +extern int x2apic_preenabled; +extern int x2apic_disabled;extern void check_x2apic (void );extern void enable_x2apic (void );extern void x2apic_icr_write (u32 low,u32 id);
@@ -198,6 +200,9 @@ static inline void x2apic_force_phys(void) x2apic_phys = 1; }#else +static inline void disable_x2apic (void ) +{ +}static inline void check_x2apic (void ) { }
@@ -214,6 +219,7 @@ static inline void x2apic_force_phys(void)#define x2apic_preenabled 0#define x2apic_supported () 0 +#define x2apic_disabled 1#endif extern void enable_IR_x2apic (void );
arch/x86/include/asm/apicdef.h +1 -0
@@ -144,6 +144,7 @@#define APIC_BASE (fix_to_virt(FIX_APIC_BASE))#define APIC_BASE_MSR 0x800 +#define XAPIC_ENABLE (1UL << 11)#define X2APIC_ENABLE (1UL << 10)#ifdef CONFIG_X86_32
arch/x86/kernel/acpi/boot.c +8 -2
@@ -219,6 +219,8 @@ static int __init acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end) {struct acpi_madt_local_x2apic *processor =NULL ; +int apic_id; +u8 enabled; processor = (struct acpi_madt_local_x2apic *)header;
@@ -227,6 +229,8 @@ acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end)acpi_table_print_madt_entry (header); + apic_id = processor->local_apic_id; + enabled = processor->lapic_flags &ACPI_MADT_ENABLED ;#ifdef CONFIG_X86_X2APIC /* * We need to register disabled CPU as well to permit
@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end)* to not preallocating memory for all NR_CPUS * when we use CPU hotplug. */ -acpi_register_lapic (processor->local_apic_id,/* APIC ID */ - processor->lapic_flags &ACPI_MADT_ENABLED ); +if (x2apic_disabled && (apic_id >=0xff ) && enabled) + printk(KERN_WARNING PREFIX "x2apic entry ignored\n"); +else +acpi_register_lapic (apic_id, enabled);#else printk(KERN_WARNING PREFIX "x2apic entry ignored\n");#endif
arch/x86/kernel/apic/apic.c +68 -24
155 lines not on this page · raw message
arch/x86/kernel/apic/apic_flat_64.c +6 -1
@@ -171,9 +171,14 @@ static int flat_phys_pkg_id(int initial_apic_id, int index_msb)return initial_apic_id >> index_msb; } +static int flat_probe (void ) +{ +return 1 ; +} +static struct apic apic_flat = { .name ="flat" , - .probe =NULL , + .probe = flat_probe, .acpi_madt_oem_check = flat_acpi_madt_oem_check, .apic_id_registered = flat_apic_id_registered,
arch/x86/kernel/apic/io_apic.c +4 -0
@@ -2948,6 +2948,10 @@ static inline void __init check_timer(void) }local_irq_disable ();apic_printk (APIC_QUIET ,KERN_INFO "..... failed :(.\n" ); +if (x2apic_preenabled && !x2apic_mode) +pr_info ("Your system has x2apic pre-enabled, but kernel can not enable intr-remapping.\n" +"So kernel try to disable x2apic, but interrupt still have problem.\n" +"You could disable x2apic from BIOS setup\n" );panic ("IO-APIC + timer doesn't work! Boot with apic=debug and send a " "report. Then try booting with the 'noapic' option.\n" ); out:
arch/x86/mm/srat.c +6 -1
@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)if ((pa->flags &ACPI_SRAT_CPU_ENABLED ) ==0 )return ; pxm = pa->proximity_domain; + apic_id = pa->apic_id; +if (x2apic_disabled && (apic_id >=0xff )) { +printk (KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n" , + pxm, apic_id); +return ; + } node =setup_node (pxm);if (node <0 ) {printk (KERN_ERR "SRAT: Too many proximity domains %x\n" , pxm);
@@ -76,7 +82,6 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)return ; } - apic_id = pa->apic_id;if (apic_id >=MAX_LOCAL_APIC ) {printk (KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n" , pxm, apic_id, node);return ;
Quoted 20 lines · Suresh Siddha
Yinghai, Here is the detailed review of your patch:
Quoted 17 lines · Yinghai Lu
Index:
linux-2.6/arch/x86/include/asm/apic.h
===================================================================arch/x86/include/asm/apic.h +2
-0@@ -176,6 +176,8 @@ static inline u64 native_x2apic_icr_read } extern int x2apic_phys; +extern int x2apic_preenabled; +extern int x2apic_disabled; extern void check_x2apic(void); extern void enable_x2apic(void); extern void x2apic_icr_write(u32 low, u32 id);@@ -183,7 +185,7 @@ static inline int x2apic_enabled(void) { u64 msr;
Quoted 7 lines
arch/x86/include/asm/apic.h static inline int x2apic_enabled(void) · line 191
Quoted 5 lines
arch/x86/include/asm/apic.h +1
-1- if (!cpu_has_x2apic) + if (!cpu_has_x2apic || x2apic_disabled) return 0; rdmsrl(MSR_IA32_APICBASE, msr);Not sure why we need
x2apic_disabledcheck here.
skip the extra msr reading, because cpu x2apic feature is kept.
Quoted 10 lines
arch/x86/include/asm/apic.h static inline int x2apic_enabled(void) · line 197
Quoted 7 lines
arch/x86/include/asm/apic.h +1
-1@@ -192,12 +194,15 @@ static inline int x2apic_enabled(void) return 0; } -#define x2apic_supported () (cpu_has_x2apic) +#define x2apic_supported () (cpu_has_x2apic && !x2apic_disabled)Here also not sure why we need this. Right thing is to clear the x2apic capability from the cpuid features when the user selects nox2apic.
just want to leave that feature alone.
Quoted 14 lines
arch/x86/include/asm/apic.h acpi_parse_x2apic(struct acpi_subtable_h · line 243
Quoted 9 lines
arch/x86/include/asm/apic.h +2
-2@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_h* to not preallocating memory for all NR_CPUS * when we use CPU hotplug. */ - acpi_register_lapic(processor->local_apic_id, /* APIC ID */ - processor->lapic_flags & ACPI_MADT_ENABLED); + if (x2apic_disabled && (apic_id >= 0xff) && enabled) + printk(KERN_WARNING PREFIX "x2apic entry ignored\n");x2apic MADT entries are specifically for apic-id's >= 255. So no need to check for the value again. We can ignore all the x2apic entries. Also print once indicating why all x2apic entries are being ignored should be enough.
no, some vendor still keep x2apic entries even apic id < 255. they have different understanding of x2apic entries.
Quoted 5 lines
Also when we disable x2apic in the case not initiated by the user (like no dmar tables etc), we would have already parsed the MADT entries but we need to skip those entries with id >= 255 duing smp boot. I don't think I see that code in this patch.
it will panic.
Quoted 14 lines
Quoted 14 lines
arch/x86/include/asm/apic.h +4
-5static __init int setup_nox2apic(char *str) { - if (x2apic_enabled()) { - pr_warning("Bios already enabled x2apic, " - "can't enforce nox2apic"); - return 0; - } + if (x2apic_enabled()) + pr_warning("Bios already enabled x2apic, will disable it"); + + x2apic_disabled = 1;
Quoted 7 lines
Quoted 3 lines
-
setup_clear_cpu_cap(X86_FEATURE_X2APIC);
return 0;
}Not sure why you removed this
clear_cpu_caphere. Right thing to do here is to delay the clearing when x2apic is pre-enabled by the bios and clear it here when x2apic is not pre-enabled by the bios.
that param (nox2apic) is analyzed earlier than check_x2apic()
so can not x2apic_preenabled to check clear the x2apic cpuid features.
Quoted 15 lines
arch/x86/include/asm/apic.h
Quoted 12 lines
arch/x86/include/asm/apic.h +11
-0+ +static void disable_x2apic (void ) +{ + int msr, msr2; + + if (!cpu_has_x2apic) + return; + + rdmsr(MSR_IA32_APICBASE, msr, msr2); + if (msr & X2APIC_ENABLE) { + u32 x2apic_id = x2apic_cpuid_initial_apicid();I think we should be able to do
read_apic_id()here and thus avoid the need for getting apicid from cpuid.
during BP enable_IR_x2apic and later apic ops could be change back to xapic...
that may not read out whole x2apic id.
Quoted 20 lines
Quoted 20 lines
arch/x86/include/asm/apic.h +16
-0+ + if (x2apic_id > 255) + panic("Can not disable x2apic, id: %08x\n", x2apic_id); + + pr_info("Disabling x2apic\n"); + /* + * Need to disable xapic and x2apic at the same time at first + * then enable xapic + */ + wrmsr(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE), + 0); + wrmsr(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE, 0); + + x2apic_disabled = 1; + } +}void
check_x2apic(void)
{
+ if (x2apic_disabled) {
Quoted 8 lines
Quoted 3 lines
+
disable_x2apic();
+ return;
+ }We shouldn't call
disable_x2apic()from here. We should be following the same sequence, like masking interrupts at the cpu, pic, io-apic level before we disable x2apic and re-enable xapic. So the right place to do this disable is atenable_IR_x2apic().
Disabling early as possible looks like more cleanly.
Quoted 5 lines
Quoted 5 lines
arch/x86/include/asm/apic.h +0
-0@@ -1449,6 +1480,11 @@ void enable_x2apic(void) { int msr, msr2;
Quoted 7 lines
arch/x86/include/asm/apic.h void enable_x2apic(void) · line 1484
Quoted 2 lines
arch/x86/include/asm/apic.h +2
-0+ if (x2apic_disabled) { + disable_x2apic();
enable_x2apic()gets called from both BP and AP. AP's need not do the apic-id being >= 255 checks etc. As that is done during smp bringup (which is missing currently from the current patch) or during MADT parsing. So all we need to do is a simplex2apic->xapicconversion.
yes, that is intended for APs too, so it will panic the system. go back to clean up process_info could be extra work.
Quoted 30 lines
arch/x86/mm/srat.c acpi_numa_x2apic_affinity_init(struct ac · line 73
Quoted 13 lines
arch/x86/include/asm/apic.h +0
-0#ifdef CONFIG_X86_64Index:
linux-2.6/arch/x86/mm/srat.c
===================================================================arch/x86/mm/srat.c +2
-0@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct ac if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) return; pxm = pa->proximity_domain; + apic_id = pa->apic_id; + if (x2apic_disabled && (apic_id >= 0xff)) {Same comments like in the above MADT parsing.
arch/x86/kernel/cpu/topology.c line 25
Quoted 11 lines
Index:
linux-2.6/arch/x86/kernel/cpu/topology.c
===================================================================arch/x86/kernel/cpu/topology.c +2
-0@@ -21,6 +21,27 @@ #define BITS_SHIFT_NEXT_LEVEL(eax) ((eax) & 0x1f) #define LEVEL_MAX_SIBLINGS(ebx) ((ebx) & 0xffff) +u32 x2apic_cpuid_initial_apicid (void ) +{I think we can do with out this function.
Quoted 5 lines
While reviewing the code, I ended up doing some of this cleanup of your patch. I have appended the modified version. Can you please incorporate the smpboot checks before the AP bringup (apic-id should be less than 255 in !x2apic_mode etc) and split the patch into couple of patches (
apic_probeetc) along with other needed cleanups.
We could handle AP's apic id > 255 panic problem later if needed in another patch.
Actually on system with apic id > 255, intr-remapping aka DMAR must be working, otherwise those big box can not be sold.
And in this case, nox2apic must be passed to disable x2apic, because dmar is correct in both case kexec or first boot.
when nox2apic is passed, those entries will be skipped early. and BSP x2apic get disabled early.
and code path is more like bios enable xapic only.
Can we let this patch get into tip and have test cycle at first? at least we should not let it break any working set up.
Thanks
Yinghai Lu
Yinghai Lu
Tested-by Berck E. NashQuoted 17 lines · Yinghai Lu
Quoted 5 lines · Suresh Siddha
While reviewing the code, I ended up doing some of this cleanup of your patch. I have appended the modified version. Can you please incorporate the smpboot checks before the AP bringup (apic-id should be less than 255 in !x2apic_mode etc) and split the patch into couple of patches (
apic_probeetc) along with other needed cleanups.We could handle AP's apic id > 255 panic problem later if needed in another patch.
Actually on system with apic id > 255, intr-remapping aka DMAR must be working, otherwise those big box can not be sold.
And in this case, nox2apic must be passed to disable x2apic, because dmar is correct in both case kexec or first boot.
Quoted 6 lines
when nox2apic is passed, those entries will be skipped early. and BSP x2apic get disabled early.
and code path is more like bios enable xapic only.Can we let this patch get into tip and have test cycle at first? at least we should not let it break any working set up.
put your updated version as v9.
like to let ingo or hpa or thomas to decide v8 or v9 to be used.
Changes:
1. clear x2apic cpu features
2. disable x2apic later instead of check_x2apic()
Thanks
Yinghai
From: Yinghai Lu <[email protected]>
Subject: [PATCH -v9] x86: Disable x2apic if nox2apic is specified or intr-remap can not be enabled
For
1. x2apic preenabled system
2. first kernel have x2apic enabled, and try to kexec second kernel with "nox2apic"
Will put back cpu with apic id < 255 into xapic mode, instead of panic.
-v2: use variable x2apic_disabled instead of variable nox2apic, Suggested by Thomas
update x2apic_supported with x2apic_disabled, Suggested by Thomas
-v3: add checking for boot cpu apic id > 255. in that case will just panic
--- pointed out by Suresh.
-v4: according to Ingo, for x2apic pre-enabled system, if intr-remap can not
be enabled, try to disable x2apic instead of panic and request to specify nox2apic for next boot.
-v5: fix the lapic mapping for falling path.
-v6: add warning when check_timer fails in strange case when BIOS enable x2apic
and intr_remapping but does not provide dmar table. requested by Suresh.
-v7: fix compiling with x2apic_preenabled
-v8: fix compiling with x2apic_disabled assigned with !X86_X2APIC
-v9: updated version from Suresh
Changes: 1. clear x2apic cpu features
2. disable x2apic later instead of check_x2apic()
--- arch/x86/include/asm/apic.h | 6 ++ arch/x86/include/asm/apicdef.h | 1 arch/x86/kernel/acpi/boot.c | 10 +++ arch/x86/kernel/apic/apic.c | 92 ++++++++++++++++++++++++++---------- arch/x86/kernel/apic/apic_flat_64.c | 7 ++ arch/x86/kernel/apic/io_apic.c | 4 + arch/x86/mm/srat.c | 7 ++ 7 files changed, 99 insertions(+), 28 deletions(-)
arch/x86/include/asm/apic.h +6 -0
@@ -176,6 +176,8 @@ static inline u64 native_x2apic_icr_read }extern int x2apic_phys; +extern int x2apic_preenabled; +extern int x2apic_disabled;extern void check_x2apic (void );extern void enable_x2apic (void );extern void x2apic_icr_write (u32 low,u32 id);
@@ -198,6 +200,9 @@ static inline void x2apic_force_phys(voi x2apic_phys = 1; }#else +static inline void disable_x2apic (void ) +{ +}static inline void check_x2apic (void ) { }
@@ -214,6 +219,7 @@ static inline void x2apic_force_phys(voi#define x2apic_preenabled 0#define x2apic_supported () 0 +#define x2apic_disabled 1#endif extern void enable_IR_x2apic (void );
arch/x86/include/asm/apicdef.h +1 -0
@@ -144,6 +144,7 @@#define APIC_BASE (fix_to_virt(FIX_APIC_BASE))#define APIC_BASE_MSR 0x800 +#define XAPIC_ENABLE (1UL << 11)#define X2APIC_ENABLE (1UL << 10)#ifdef CONFIG_X86_32
arch/x86/kernel/acpi/boot.c +8 -2
@@ -219,6 +219,8 @@ static int __init acpi_parse_x2apic(struct acpi_subtable_header *header, const unsigned long end) {struct acpi_madt_local_x2apic *processor =NULL ; +int apic_id; +u8 enabled; processor = (struct acpi_madt_local_x2apic *)header;
@@ -227,6 +229,8 @@ acpi_parse_x2apic(struct acpi_subtable_hacpi_table_print_madt_entry (header); + apic_id = processor->local_apic_id; + enabled = processor->lapic_flags &ACPI_MADT_ENABLED ;#ifdef CONFIG_X86_X2APIC /* * We need to register disabled CPU as well to permit
@@ -235,8 +239,10 @@ acpi_parse_x2apic(struct acpi_subtable_h* to not preallocating memory for all NR_CPUS * when we use CPU hotplug. */ -acpi_register_lapic (processor->local_apic_id,/* APIC ID */ - processor->lapic_flags &ACPI_MADT_ENABLED ); +if (x2apic_disabled && (apic_id >=0xff ) && enabled) + printk(KERN_WARNING PREFIX "x2apic entry ignored\n"); +else +acpi_register_lapic (apic_id, enabled);#else printk(KERN_WARNING PREFIX "x2apic entry ignored\n");#endif
arch/x86/kernel/apic/apic.c +68 -24
155 lines not on this page · raw message
arch/x86/kernel/apic/apic_flat_64.c +6 -1
@@ -171,9 +171,14 @@ static int flat_phys_pkg_id(int initial_return initial_apic_id >> index_msb; } +static int flat_probe (void ) +{ +return 1 ; +} +static struct apic apic_flat = { .name ="flat" , - .probe =NULL , + .probe = flat_probe, .acpi_madt_oem_check = flat_acpi_madt_oem_check, .apic_id_registered = flat_apic_id_registered,
arch/x86/kernel/apic/io_apic.c +4 -0
@@ -2947,6 +2947,10 @@ static inline void __init check_timer(vo }local_irq_disable ();apic_printk (APIC_QUIET ,KERN_INFO "..... failed :(.\n" ); +if (x2apic_preenabled && !x2apic_mode) +pr_info ("Your system has x2apic pre-enabled, but kernel can not enable intr-remapping.\n" +"So kernel try to disable x2apic, but interrupt still have problem.\n" +"You could disable x2apic from BIOS setup\n" );panic ("IO-APIC + timer doesn't work! Boot with apic=debug and send a " "report. Then try booting with the 'noapic' option.\n" ); out:
arch/x86/mm/srat.c +6 -1
@@ -69,6 +69,12 @@ acpi_numa_x2apic_affinity_init(struct acif ((pa->flags &ACPI_SRAT_CPU_ENABLED ) ==0 )return ; pxm = pa->proximity_domain; + apic_id = pa->apic_id; +if (x2apic_disabled && (apic_id >=0xff )) { +printk (KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n" , + pxm, apic_id); +return ; + } node =setup_node (pxm);if (node <0 ) {printk (KERN_ERR "SRAT: Too many proximity domains %x\n" , pxm);
@@ -76,7 +82,6 @@ acpi_numa_x2apic_affinity_init(struct acreturn ; } - apic_id = pa->apic_id;if (apic_id >=MAX_LOCAL_APIC ) {printk (KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n" , pxm, apic_id, node);return ;
Quoted 7 lines · Yinghai Lu
put your updated version as v9.
like to let ingo or hpa or thomas to decide v8 or v9 to be used.
Changes:
1. clear x2apic cpu features
2. disable x2apic later instead ofcheck_x2apic()
Yinghai, It is not the question of just cleanup. changing the apic mode has to follow some sequence.
Anyways, I have done all the changes and other minor cleanups. Also split up your patch into multiple patches and posted it now.
Please let me know if you see any issue.
thanks,
suresh
Quoted 9 lines · Suresh Siddha
Quoted 7 lines · Yinghai Lu
put your updated version as v9.
like to let ingo or hpa or thomas to decide v8 or v9 to be used.
Changes:
1. clear x2apic cpu features
2. disable x2apic later instead ofcheck_x2apic()
Quoted 7 lines
Yinghai, It is not the question of just cleanup. changing the apic mode has to follow some sequence.
Anyways, I have done all the changes and other minor cleanups. Also split up your patch into multiple patches and posted it now.
Please let me know if you see any issue.
updated 4/5 for some minor problem with applying and compiling.
Please check that.
Thanks
Yinghai
A message that is not in the archive (sent off-list or not yet received): A75BCAD09CE00A4280CDD4429D85F1F92621BD7595@orsmsx501.amr.corp.intel.com
Quoted 3 lines · Suresh Siddha
b. for nox2apic, as you try to disable x2apic even if the bios has enabled it along with interrupt-remapping, this will probably not work unless we disable interrupt-remapping.
Looking at the code, this condition should be handled correctly by the OS by enabling interrupt-remapping in non extended interrupt-mode. So this case should be ok.
thanks,
suresh