x86: enable x2apic early at the first point
Subsystems (3)
PATCH 1/1 x86: enable x2apic early at the first point
Impact: fix bug.
otherwise will get panic from early_acpi_boot_init() also make disable_x2apic global, so could use it it x2_apic_xxx.c and can get warning if preenabled system using nox2apic.
--- arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)
arch/x86/kernel/apic/x2apic_cluster.c +4 -1
@@ -14,8 +14,11 @@ DEFINE_PER_CPU(u32, x86_cpu_to_logical_astatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic) +if (cpu_has_x2apic && !disable_x2apic) { + x2apic =1 ; +enable_x2apic ();return 1 ; + }return 0 ; }
arch/x86/kernel/apic/x2apic_phys.c +4 -1
@@ -21,8 +21,11 @@ early_param("x2apic_phys", set_x2apic_phstatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic && x2apic_phys) +if (cpu_has_x2apic && !disable_x2apic && x2apic_phys) { + x2apic =1 ; +enable_x2apic ();return 1 ; + }return 0 ; }
arch/x86/kernel/apic/x2apic_uv_x.c +3 -1
@@ -41,8 +41,10 @@ static int uv_acpi_madt_oem_check(char * uv_system_type =UV_LEGACY_APIC ; else if (!strcmp(oem_table_id, "UVX")) uv_system_type =UV_X2APIC ; - else if (!strcmp(oem_table_id, "UVH")) { + else if (!strcmp(oem_table_id, "UVH") && !disable_x2apic) { uv_system_type =UV_NON_UNIQUE_APIC ; + x2apic =1 ; +enable_x2apic ();return 1 ; } }
arch/x86/kernel/apic/apic.c +1 -2
@@ -116,11 +116,10 @@ __setup("apicpmtimer", setup_apicpmtimerint x2apic;/* x2apic enabled before OS handover */ static int x2apic_preenabled; -static int disable_x2apic; +int disable_x2apic;static __init int setup_nox2apic (char *str) { disable_x2apic =1 ; -setup_clear_cpu_cap (X86_FEATURE_X2APIC );return 0 ; }early_param ("nox2apic" , setup_nox2apic);
drivers/pci/dmar.c +2 -1
@@ -472,7 +472,8 @@ void __init detect_intel_iommu(void)* is added, we will not need this any more. */ dmar = (struct acpi_table_dmar *) dmar_tbl; -if (ret && cpu_has_x2apic && dmar->flags &0x1 ) +if (ret && cpu_has_x2apic && !disable_x2apic && + dmar->flags &0x1 )printk (KERN_INFO "Queued invalidation will be enabled to support " "x2apic and Intr-remapping.\n" );
Quoted 21 lines · Yinghai Lu
Impact: fix bug.
otherwise will get panic from
early_acpi_boot_init()also makedisable_x2apicglobal, so could use it itx2_apic_xxx.cand can get warning if preenabled system using nox2apic.Signed-off-by Yinghai Lu <[email protected]>--- arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)Index:
linux-2.6/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
arch/x86/kernel/apic/x2apic_cluster.c DEFINE_PER_CPU(u32, x86_cpu_to_logical_a · line 19
Quoted 7 lines
arch/x86/kernel/apic/x2apic_cluster.c +3
-1@@ -14,8 +14,11 @@ DEFINE_PER_CPU(u32, x86_cpu_to_logical_astatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic) +if (cpu_has_x2apic && !disable_x2apic) { + x2apic =1 ; +enable_x2apic ();
Yinghai, I also ran into couple of x2apic issues on the latest tip and this issue is one of them. I don't like this patch mainly because we are enabling x2apic mode in the cpu, with out checking the intr-remapping support. We should fall back to xapic mode (in a clean fashion) if there is any issue with intr-remapping. So I am planning to post another patch to fix this early hang issue that we both encountered.
thanks,
suresh
Quoted 69 lines
arch/x86/kernel/apic/x2apic_cluster.c +1
-0return 1 ; + }return 0 ; }Index:
linux-2.6/arch/x86/kernel/apic/x2apic_phys.c
===================================================================arch/x86/kernel/apic/x2apic_phys.c +4
-1@@ -21,8 +21,11 @@ early_param("x2apic_phys", set_x2apic_phstatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic && x2apic_phys) +if (cpu_has_x2apic && !disable_x2apic && x2apic_phys) { + x2apic =1 ; +enable_x2apic ();return 1 ; + }return 0 ; }Index:
linux-2.6/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================arch/x86/kernel/apic/x2apic_uv_x.c +3
-1@@ -41,8 +41,10 @@ static int uv_acpi_madt_oem_check(char * uv_system_type =UV_LEGACY_APIC ; else if (!strcmp(oem_table_id, "UVX")) uv_system_type =UV_X2APIC ; - else if (!strcmp(oem_table_id, "UVH")) { + else if (!strcmp(oem_table_id, "UVH") && !disable_x2apic) { uv_system_type =UV_NON_UNIQUE_APIC ; + x2apic =1 ; +enable_x2apic ();return 1 ; } }Index:
linux-2.6/arch/x86/kernel/apic/apic.c
===================================================================arch/x86/kernel/apic/apic.c +1
-2@@ -116,11 +116,10 @@ __setup("apicpmtimer", setup_apicpmtimerint x2apic;/* x2apic enabled before OS handover */ static int x2apic_preenabled; -static int disable_x2apic; +int disable_x2apic;static __init int setup_nox2apic (char *str) { disable_x2apic =1 ; -setup_clear_cpu_cap (X86_FEATURE_X2APIC );return 0 ; }early_param ("nox2apic" , setup_nox2apic);Index:
linux-2.6/drivers/pci/dmar.c
===================================================================drivers/pci/dmar.c +2
-1@@ -472,7 +472,8 @@ void __init detect_intel_iommu(void)* is added, we will not need this any more. */ dmar = (struct acpi_table_dmar *) dmar_tbl; -if (ret && cpu_has_x2apic && dmar->flags &0x1 ) +if (ret && cpu_has_x2apic && !disable_x2apic && + dmar->flags &0x1 )printk (KERN_INFO "Queued invalidation will be enabled to support " "x2apic and Intr-remapping.\n" );
Quoted 23 lines · Suresh Siddha
Quoted 22 lines · Yinghai Lu
Impact: fix bug.
otherwise will get panic from
early_acpi_boot_init()also makedisable_x2apicglobal, so could use it itx2_apic_xxx.cand can get warning if preenabled system using nox2apic.Signed-off-by Yinghai Lu <[email protected]>--- arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)Index:
linux-2.6/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================arch/x86/kernel/apic/x2apic_cluster.c +0
-0@@ -14,8 +14,11 @@ DEFINE_PER_CPU(u32, x86_cpu_to_logical_a
Quoted 13 lines
arch/x86/kernel/apic/x2apic_cluster.c DEFINE_PER_CPU(u32, x86_cpu_to_logical_a · line 19
Quoted 6 lines
arch/x86/kernel/apic/x2apic_cluster.c +3
-1static int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic) +if (cpu_has_x2apic && !disable_x2apic) { + x2apic =1 ; +enable_x2apic ();Yinghai, I also ran into couple of x2apic issues on the latest tip and this issue is one of them. I don't like this patch mainly because we are enabling x2apic mode in the cpu, with out checking the intr-remapping support. We should fall back to xapic mode (in a clean fashion) if there is any issue with intr-remapping. So I am planning to post another patch to fix this early hang issue that we both encountered.
Ingo want to decouple that x2apic and intr_remapping.
it seems it does work with x2apic without intr_remapping in one of setup.
YH
Quoted 2 lines · Yinghai Lu
Ingo want to decouple that x2apic and
intr_remapping.
it seems it does work with x2apic withoutintr_remappingin one of setup.
x2apic with out intr-remapping is not architectural. Even when we have < 255 logical cpu's, logical x2apic id's will be greater than 16 bits even on a single/two socket systems and this will break interrupt delivery. Please look at the x2apic logical destination mode definition in the SDM (Section 9.7.2.3 and 9.7.2.4 in my copy of SDM Vol3a)
While it might work in certain configurations (for example, physical mode with < 8 bit apicids), it is not architectural and implementation dependent, which may break in future generations.
And also, logical x2apic mode has more advantages compared to physical mode (like using lowest priority delivery mode etc).
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
thanks,
suresh
Quoted 16 lines · Suresh Siddha
Quoted 2 lines · Yinghai Lu
Ingo want to decouple that x2apic and
intr_remapping.
it seems it does work with x2apic withoutintr_remappingin one of setup.x2apic with out intr-remapping is not architectural. Even when we have < 255 logical cpu's, logical x2apic id's will be greater than 16 bits even on a single/two socket systems and this will break interrupt delivery. Please look at the x2apic logical destination mode definition in the SDM (Section 9.7.2.3 and 9.7.2.4 in my copy of SDM Vol3a)
While it might work in certain configurations (for example, physical mode with < 8 bit apicids), it is not architectural and implementation dependent, which may break in future generations.
Quoted 5 lines
And also, logical x2apic mode has more advantages compared to physical mode (like using lowest priority delivery mode etc).
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
Instead of the revert, we can zap this patch from tip:x86/apic:
d13d2c3: x86, apic: decouple x2apic from interrupt remapping
But it will all need more cleanups. For example the mandatory coupling between x2apic and intr-remap is not clearly spelled out.
Also, some of the code has also become rather ugly. For example could you please work on eliminating this #ifdef-fest:
earth4:~/tip> grep INTR_REMAP arch/x86/kernel/apic/io_apic.c #ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
#ifdef CONFIG_INTR_REMAP
for example all the #ifdefs around irq_remapped() look bogus to me - dmar.h already defines irq_remapped() to always-0 when !CONFIG_INTR_REMAP.
Ingo
Quoted 7 lines · Ingo Molnar
Quoted 4 lines · Suresh Siddha
Quoted 2 lines · Yinghai Lu
Ingo want to decouple that x2apic and
intr_remapping.
it seems it does work with x2apic withoutintr_remappingin one of setup.
Quoted 7 lines
Quoted 7 lines
x2apic with out intr-remapping is not architectural. Even when we have < 255 logical cpu's, logical x2apic id's will be greater than 16 bits even on a single/two socket systems and this will break interrupt delivery. Please look at the x2apic logical destination mode definition in the SDM (Section 9.7.2.3 and 9.7.2.4 in my copy of SDM Vol3a)
This is what Intel doc says:
It is likely that processor implementations may choose to support less than 16 bits of the cluster ID or less than 16-bits of the Logical ID in the Logical Destination Register. However system software should be agnostic to the number of bits implemented in the cluster ID and logical ID sub-fields. The x2APIC hardware initialization will ensure that the appropriately initialized logical x2APIC IDs are available to system software and reads of non-implemented bits return zero.
KVM will implement 0 bits of cluster ID and will want to use x2apic without implementing IR.
Quoted 5 lines
Quoted 5 lines
While it might work in certain configurations (for example, physical mode with < 8 bit apicids), it is not architectural and implementation dependent, which may break in future generations.
It will not break in future generation of KVM though.
Quoted 6 lines
Quoted 5 lines
And also, logical x2apic mode has more advantages compared to physical mode (like using lowest priority delivery mode etc).
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
--
Gleb.
Quoted 11 lines · Gleb Natapov
Quoted 3 lines
Quoted 3 lines
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
yep, that would be required - because your patch can break real systems right now. Mind sending me a fix for it?
I've applied Yinghai's fix as well, so please base it on latest tip:master.
Ingo
Quoted 15 lines · Ingo Molnar
Quoted 11 lines · Gleb Natapov
Quoted 3 lines
Quoted 3 lines
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
Quoted 6 lines
yep, that would be required - because your patch can break real systems right now. Mind sending me a fix for it?
I've applied Yinghai's fix as well, so please base it on latest tip:master.
OK. Will send next week.
--
Gleb.
Quoted 16 lines · Gleb Natapov
Quoted 15 lines · Ingo Molnar
Quoted 11 lines · Gleb Natapov
Quoted 3 lines
Quoted 3 lines
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
Quoted 7 lines
Quoted 6 lines
yep, that would be required - because your patch can break real systems right now. Mind sending me a fix for it?
I've applied Yinghai's fix as well, so please base it on latest tip:master.
OK. Will send next week.
ok, that's too long of a breakage window - i'll revert it then, please send a new version once you have it.
Thanks,
Ingo
Quoted 22 lines · Ingo Molnar
Quoted 19 lines · Gleb Natapov
Quoted 18 lines · Ingo Molnar
Quoted 11 lines · Gleb Natapov
Quoted 3 lines
Quoted 3 lines
I will post couple of patches, which revert's Gleb's patch and another fix for the early boot failure issue, tomorrow.
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
yep, that would be required - because your patch can break real systems right now. Mind sending me a fix for it?
Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
I've applied Yinghai's fix as well, so please base it on latest tip:master.
OK. Will send next week.
ok, that's too long of a breakage window - i'll revert it then, please send a new version once you have it.
Deal.
--
Gleb.
Quoted 11 lines · Ingo Molnar
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
yep, that would be required - because your patch can break real systems right now. Mind sending me a fix for it?
Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
I've applied Yinghai's fix as well, so please base it on latest tip:master.
OK. Will send next week.
ok, that's too long of a breakage window - i'll revert it then, please send a new version once you have it.
What approach you actually prefer? Enable x2apic without intr-remapping only when running in KVM, or even if running on real HW but all CPUs are in cluster 0? If former then below is updated patch (tested only in KVM) if later then it's definitely for next week :)
From: Gleb Natapov <[email protected]>
Subject: x86, apic: decouple x2apic from interrupt remapping
Currently x2apic is used only if interrupt remapping can be enabled, and although there is not real HW that has x2apic but does not have intr-remapping the decoupling is useful since we want to emulate x2apic interface in KVM (it is more vitalization friendly), but we don't want to emulate intr-remapping just for that. Attached patch enables x2apic when running in kvm guest even if intr-remapping cannot be enabled.
arch/x86/kernel/apic/apic.c +50 -54
@@ -48,6 +48,7 @@#include <asm/idle.h> #include <asm/mtrr.h> #include <asm/smp.h> +#include <asm/kvm_para.h> unsigned int num_processors;
@@ -1296,10 +1297,44 @@ void enable_x2apic(void) } } -void __init enable_IR_x2apic (void ) +static int __init enable_IR (void ) {#ifdef CONFIG_INTR_REMAP int ret; + + ret =dmar_table_init (); +if (ret) { +pr_info ("dmar_table_init() failed with %d:\n" , ret); +return ret; + } + + ret =save_mask_IO_APIC_setup (); +if (ret) { +pr_info ("Saving IO-APIC state failed: %d\n" , ret); +return ret; + } + + ret =enable_intr_remapping (1 ); +if (ret) { +restore_IO_APIC_setup (); +return ret; + } + +reinit_intr_remapped_IO_APIC (x2apic_preenabled); + +return 0 ; +#else +if (!kvm_para_available ()) { +local_irq_restore (flags); +panic ("x2apic enabled prior OS handover," +" enable CONFIG_INTR_REMAP" ); + } +return -ENODEV ; +#endif +} + +void __init enable_IR_x2apic (void ) +{unsigned long flags;if (!cpu_has_x2apic)
@@ -1320,72 +1355,33 @@ void __init enable_IR_x2apic(void)return ; } - ret =dmar_table_init (); -if (ret) { -pr_info ("dmar_table_init() failed with %d:\n" , ret); - -if (x2apic_preenabled) -panic ("x2apic enabled by bios. But IR enabling failed" ); -else -pr_info ("Not enabling x2apic,Intr-remapping\n" ); -return ; - } -local_irq_save (flags);mask_8259A (); - ret =save_mask_IO_APIC_setup (); -if (ret) { -pr_info ("Saving IO-APIC state failed: %d\n" , ret); - goto end; - } - - ret =enable_intr_remapping (1 ); - -if (ret && x2apic_preenabled) { -local_irq_restore (flags); -panic ("x2apic enabled by bios. But IR enabling failed" ); +if (enable_IR ()) { +if (!kvm_para_available ()) { +if (x2apic_preenabled) { +local_irq_restore (flags); +panic ("x2apic enabled by bios. " +"But IR enabling failed" ); + } + goto no_x2apic; + } + }else { +pr_info ("Enabled interrupt remapping for x2apic\n" ); } -if (ret) - goto end_restore; -if (!x2apic) { x2apic =1 ;enable_x2apic (); } -end_restore: -if (ret) -/* -* IR enabling failed -*/ -restore_IO_APIC_setup (); -else -reinit_intr_remapped_IO_APIC (x2apic_preenabled); - -end: +no_x2apic:unmask_8259A ();local_irq_restore (flags); -if (!ret) { -if (!x2apic_preenabled) -pr_info ("Enabled x2apic and interrupt-remapping\n" ); -else -pr_info ("Enabled Interrupt-remapping\n" ); - }else -pr_err ("Failed to enable Interrupt-remapping and x2apic\n" ); -#else -if (!cpu_has_x2apic) -return ; - -if (x2apic_preenabled) -panic ("x2apic enabled prior OS handover," -" enable CONFIG_INTR_REMAP" ); - -pr_info ("Enable CONFIG_INTR_REMAP for enabling intr-remapping " -" and x2apic\n" ); -#endif +if (x2apic && !x2apic_preenabled) +pr_info ("Enabled x2apic\n" );return ; }
arch/x86/kernel/apic/probe_64.c +2 -1
@@ -22,6 +22,7 @@#include <asm/apic.h> #include <asm/ipi.h> #include <asm/setup.h> +#include <asm/kvm_para.h> extern struct apic apic_flat;extern struct apic apic_physflat;
@@ -51,7 +52,7 @@ void __init default_setup_apic_routing(void) {#ifdef CONFIG_X86_X2APIC if (apic == &apic_x2apic_phys || apic == &apic_x2apic_cluster) { -if (!intr_remapping_enabled) +if (!intr_remapping_enabled && !kvm_para_available ()) apic = &apic_flat; }#endif
arch/x86/kernel/apic/x2apic_cluster.c +34 -22
78 lines not on this page · raw message
--
Gleb.
Quoted 21 lines · Gleb Natapov
Quoted 11 lines · Ingo Molnar
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
If you'll revert my patch it will not be possible to use x2apic in KVM (at least without KVM implementing interrupt remapping which is unneeded otherwise) and x2apic interface is much better for vitalization. Instead of reverting the patch it will be better to add check if x2apic can be used without intr-remmaping (all CPUs belong to cluster 0) or allow enabling of x2apic without IR if running as a guest.
yep, that would be required - because your patch can break real systems right now. Mind sending me a fix for it?
Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
I've applied Yinghai's fix as well, so please base it on latest tip:master.
OK. Will send next week.
ok, that's too long of a breakage window - i'll revert it then, please send a new version once you have it.
Quoted 5 lines
What approach you actually prefer? Enable x2apic without intr-remapping only when running in KVM, or even if running on real HW but all CPUs are in cluster 0? If former then below is updated patch (tested only in KVM) if later then it's definitely for next week :)
The latter would be the ideal solution - i.e. next week :)
x2apic is a rare feature, and the more exposure we give it the more tested it becomes. It's also obviously good for general code structure if core CPU features are separated from chipset/iommu features.
Plus, x2apic accesses could actually be faster than UC accesses to the lapic, so whenever we can we should enter x2apic mode - even if intr-remap is not turned on (because not needed).
Ingo
Quoted 8 lines · Yinghai Lu
Impact: fix bug.
otherwise will get panic from
early_acpi_boot_init()also makedisable_x2apicglobal, so could use it itx2_apic_xxx.cand can get warning if preenabled system using nox2apic.Signed-off-by Yinghai Lu <[email protected]>
Quoted 7 lines
--- arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)
I've applied it because it fixes a real bug, but this code really needs a cleanup. Look at the repeat patterns:
Quoted 40 lines
Index:
linux-2.6/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================arch/x86/kernel/apic/x2apic_cluster.c +4
-1@@ -14,8 +14,11 @@ DEFINE_PER_CPU(u32, x86_cpu_to_logical_astatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic) +if (cpu_has_x2apic && !disable_x2apic) { + x2apic =1 ; +enable_x2apic ();return 1 ; + }return 0 ; }Index:
linux-2.6/arch/x86/kernel/apic/x2apic_phys.c
===================================================================arch/x86/kernel/apic/x2apic_phys.c +4
-1@@ -21,8 +21,11 @@ early_param("x2apic_phys", set_x2apic_phstatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic && x2apic_phys) +if (cpu_has_x2apic && !disable_x2apic && x2apic_phys) { + x2apic =1 ; +enable_x2apic ();return 1 ; + }return 0 ; }Index:
linux-2.6/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================arch/x86/kernel/apic/x2apic_uv_x.c +0
-0@@ -41,8 +41,10 @@ static int uv_acpi_madt_oem_check(char * uv_system_type =UV_LEGACY_APIC ;
arch/x86/kernel/apic/x2apic_uv_x.c static int uv_acpi_madt_oem_check(char * · line 50
Quoted 10 lines
arch/x86/kernel/apic/x2apic_uv_x.c +3
-1else if (!strcmp (oem_table_id,"UVX" )) uv_system_type =UV_X2APIC ; -else if (!strcmp (oem_table_id,"UVH" )) { +else if (!strcmp (oem_table_id,"UVH" ) && !disable_x2apic) { uv_system_type =UV_NON_UNIQUE_APIC ; + x2apic =1 ; +enable_x2apic ();return 1 ; } }
Such repeat patterns with small variations are always the sign of an inefficient code structure.
The clean approach would be to have one generic helper function concentrated into enable_x2apic(). So instead of:
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
if (cpu_has_x2apic && !disable_x2apic && x2apic_phys) {
x2apic = 1;
enable_x2apic();
return 1;
}
return 0;}
We'd have:
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
if (!x2apic_phys)
return 0;
return x86_enable_x2apic();}
That way all the repeat and common functionality (and the return code logic) is concentrated into enable_x2apic(), and the x2apic_acpi_madt_oem_check() function has _only_ its own special check open-coded. (namely, whether physical ID delivery mode is forced off.)
Okay?
Ingo
Quoted 9 lines · Ingo Molnar
Quoted 6 lines
arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)I've applied it because it fixes a real bug, but this code really needs a cleanup. Look at the repeat patterns:
unapplied it again as it breaks the build:
arch/x86/kernel/apic/x2apic_cluster.c: In function 'x2apic_acpi_madt_oem_check':arch/x86/kernel/apic/x2apic_cluster.c:17: error: 'disable_x2apic' undeclared (first use in this function) arch/x86/kernel/apic/x2apic_cluster.c:17: error: (Each undeclared identifier is reported only once arch/x86/kernel/apic/x2apic_cluster.c:17: error: for each function it appears in.)
so please resend the fixed and cleaned up version.
Thanks,
Ingo
Quoted 12 lines · Ingo Molnar
Quoted 9 lines · Ingo Molnar
Quoted 6 lines
arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)I've applied it because it fixes a real bug, but this code really needs a cleanup. Look at the repeat patterns:
Quoted 8 lines
unapplied it again as it breaks the build:
arch/x86/kernel/apic/x2apic_cluster.c: In function 'x2apic_acpi_madt_oem_check':arch/x86/kernel/apic/x2apic_cluster.c:17: error: 'disable_x2apic' undeclared (first use in this function)arch/x86/kernel/apic/x2apic_cluster.c:17: error: (Each undeclared identifier is reported only oncearch/x86/kernel/apic/x2apic_cluster.c:17: error: for each function it appears in.)so please resend the fixed and cleaned up version.
From: Suresh Siddha <[email protected]>
Subject: x86: select x2apic ops in early apic probe only if x2apic mode is enabled
If BIOS hands over the control to OS in legacy xapic mode, select legacy xapic related ops in the early apic probe and shift to x2apic ops later in the boot sequence, only after enabling x2apic mode.
If BIOS hands over the control in x2apic mode, select x2apic related ops in the early apic probe.
This fixes the early boot panic, where we were selecting x2apic ops, while the cpu is still in legacy xapic mode.
---
arch/x86/include/asm/apic.h renamed from arch/x86/include/asm/apic.h.orig +1 -1
@@ -146,7 +146,7 @@ static inline u64 native_x2apic_icr_readreturn val; } -extern int x2apic; +extern int x2apic, x2apic_phys;extern void check_x2apic (void );extern void enable_x2apic (void );extern void enable_IR_x2apic (void );
arch/x86/kernel/apic/probe_64.c renamed from arch/x86/kernel/apic/probe_64.c.orig +10 -3
@@ -50,9 +50,16 @@ static struct apic *apic_probe[] __initdvoid __init default_setup_apic_routing (void ) {#ifdef CONFIG_X86_X2APIC -if (apic == &apic_x2apic_phys || apic == &apic_x2apic_cluster) { -if (!intr_remapping_enabled) - apic = &apic_flat; +if (x2apic && (apic != &apic_x2apic_phys && +#ifdef CONFIG_X86_UV + apic != &apic_x2apic_uv_x && +#endif + apic != &apic_x2apic_cluster)) { +if (x2apic_phys) + apic = &apic_x2apic_phys; +else + apic = &apic_x2apic_cluster; +printk (KERN_INFO "Setting APIC routing to %s\n" , apic->name); }#endif
arch/x86/kernel/apic/x2apic_cluster.c renamed from arch/x86/kernel/apic/x2apic_cluster.c.orig +1 -4
@@ -14,10 +14,7 @@ DEFINE_PER_CPU(u32, x86_cpu_to_logical_astatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic) -return 1 ; - -return 0 ; +return x2apic_enabled (); }/* Start with all IRQs pointing to boot CPU. IRQ balancing will shift them. */
arch/x86/kernel/apic/x2apic_phys.c renamed from arch/x86/kernel/apic/x2apic_phys.c.orig +5 -5
@@ -10,7 +10,7 @@#include <asm/apic.h> #include <asm/ipi.h> -static int x2apic_phys; +int x2apic_phys;static int set_x2apic_phys_mode (char *arg) {
@@ -21,10 +21,10 @@ early_param("x2apic_phys", set_x2apic_phstatic int x2apic_acpi_madt_oem_check (char *oem_id,char *oem_table_id) { -if (cpu_has_x2apic && x2apic_phys) -return 1 ; - -return 0 ; +if (x2apic_phys) +return x2apic_enabled (); +else +return 0 ; }/* Start with all IRQs pointing to boot CPU. IRQ balancing will shift them. */
arch/x86/kernel/apic/apic.c renamed from arch/x86/kernel/apic/apic.c.orig +1 -8
@@ -1269,14 +1269,7 @@ void __cpuinit end_local_APIC_setup(void#ifdef CONFIG_X86_X2APIC void check_x2apic (void ) { -int msr, msr2; - -if (!cpu_has_x2apic) -return ; - -rdmsr (MSR_IA32_APICBASE , msr, msr2); - -if (msr &X2APIC_ENABLE ) { +if (x2apic_enabled ()) {pr_info ("x2apic enabled by BIOS, switching to x2apic ops\n" ); x2apic_preenabled = x2apic =1 ; }
Quoted 27 lines · Suresh Siddha
Quoted 10 lines · Ingo Molnar
Quoted 8 lines · Ingo Molnar
Quoted 6 lines
arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)I've applied it because it fixes a real bug, but this code really needs a cleanup. Look at the repeat patterns:
Quoted 8 lines
unapplied it again as it breaks the build:
arch/x86/kernel/apic/x2apic_cluster.c: In function 'x2apic_acpi_madt_oem_check':arch/x86/kernel/apic/x2apic_cluster.c:17: error: 'disable_x2apic' undeclared (first use in this function)arch/x86/kernel/apic/x2apic_cluster.c:17: error: (Each undeclared identifier is reported only oncearch/x86/kernel/apic/x2apic_cluster.c:17: error: for each function it appears in.)so please resend the fixed and cleaned up version.
From: Suresh Siddha <[email protected]>
Subject: x86: select x2apic ops in early apic probe only if x2apic mode is enabledIf BIOS hands over the control to OS in legacy xapic mode, select legacy xapic related ops in the early apic probe and shift to x2apic ops later in the boot sequence, only after enabling x2apic mode.
Quoted 5 lines
If BIOS hands over the control in x2apic mode, select x2apic related ops in the early apic probe.
This fixes the early boot panic, where we were selecting x2apic ops, while the cpu is still in legacy xapic mode.
good, other than that.
for x2apic preenabled system,
when nox2apic is used, cpu_has_x2apic will be cleared, apic will be xapic phys_flat or flat. is that expected?
should
1. ignore nox2apic
2. or try to disable x2apic?
YH
Quoted 7 lines · Yinghai Lu
for x2apic preenabled system,
when nox2apic is used,cpu_has_x2apicwill be cleared, apic will be xapicphys_flator flat. is that expected?should
1. ignore nox2apic
2. or try to disable x2apic?
This scenario might be useful for debug purposes? But it might
not be simple/straight fwd in OS to implement this, as we need to do two things.
1. Go back to xapic mode using the state transition diagram in SDM.
2. And also, we need to disable the interrupt-remapping setup by the bios, so that chipset and cpu's are in same mode.
If BIOS has enabled x2apic, it is for a reason (mostly platform has more logical cpus and hence need x2apic to brinup all the AP's etc). And typically other than very high end platforms, I don't expect bios to turn on x2apic.
thanks,
suresh
Ingo Molnar
AppliedQuoted 20 lines · Suresh Siddha
Quoted 19 lines · Ingo Molnar
Quoted 9 lines · Ingo Molnar
Quoted 6 lines
arch/x86/kernel/apic/apic.c | 3 +-- arch/x86/kernel/apic/x2apic_cluster.c | 5 ++++- arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- arch/x86/kernel/apic/x2apic_uv_x.c | 4 +++- drivers/pci/dmar.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-)I've applied it because it fixes a real bug, but this code really needs a cleanup. Look at the repeat patterns:
unapplied it again as it breaks the build:
arch/x86/kernel/apic/x2apic_cluster.c: In function 'x2apic_acpi_madt_oem_check':arch/x86/kernel/apic/x2apic_cluster.c:17: error: 'disable_x2apic' undeclared (first use in this function)arch/x86/kernel/apic/x2apic_cluster.c:17: error: (Each undeclared identifier is reported only oncearch/x86/kernel/apic/x2apic_cluster.c:17: error: for each function it appears in.)
Quoted 4 lines
Quoted 1 line
so please resend the fixed and cleaned up version.
From: Suresh Siddha <[email protected]>
Subject: x86: select x2apic ops in early apic probe only if x2apic mode is enabled
Applied to tip:x86/apic, thanks Suresh!
Ingo