enable x2APIC without interrupt remapping under KVM

KVM would like to provide x2APIC interface to a guest without emulating interrupt remapping device. The reason KVM prefers guest to use x2APIC is that x2APIC interface is better virtualizable and provides better performance than mmio xAPIC interface:

- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface

Included patch changes x2APIC enabling logic to enable it even if IR initialization failed, but kernel runs under KVM and no apic id is greater than 255 (if there is one spec requires BIOS to move to x2apic mode before starting an OS).

Signed-off-by Gleb Natapov <[email protected]>

arch/x86/kernel/apic/apic.c +4 -6

index 8c7c042..351d55a 100644
@@ -49,6 +49,7 @@
  #include <asm/mtrr.h>
  #include <asm/smp.h>
  #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
  unsigned int num_processors;
 

@@ -1363,52 +1364,75 @@ void enable_x2apic(void)
  }
  #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
  {
  #ifdef CONFIG_INTR_REMAP
        int ret;
-       unsigned long flags;
-       struct IO_APIC_route_entry **ioapic_entries = NULL;
 
        ret = dmar_table_init();
        if (ret) {
                pr_debug("dmar_table_init() failed with %d:\n", ret);
-               goto ir_failed;
+               return 0;
        }
 
        if (!intr_remapping_supported()) {
                pr_debug("intr-remapping not supported\n");
-               goto ir_failed;
+               return 0;
        }
 
-
        if (!x2apic_preenabled && skip_ioapic_setup) {

               pr_info("Skipped enabling intr-remap because of skipping " "io-apic setup\n");
-               return;
+               return 0;
       }

+       if (enable_intr_remapping(x2apic_supported()))
+               return 0;

arch/x86/kernel/apic/apic.c +15 -0

+
+       pr_info("Enabled Interrupt-remapping\n");
+
+       return 1;
+
+#endif
+       return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+       unsigned long flags;
+       struct IO_APIC_route_entry **ioapic_entries = NULL;
+       int ret, x2apic_enabled = 0;
+

       ioapic_entries = alloc_ioapic_entries();
       if (!ioapic_entries) {
-               pr_info("Allocate ioapic_entries failed: %d\n", ret);
-               goto end;
+                pr_info("Allocate ioapic_entries failed\n");
+                goto out;
       }

       ret = save_IO_APIC_setup(ioapic_entries);
       if (ret) {
               pr_info("Saving IO-APIC state failed: %d\n", ret);
-               goto end;
+               goto out;
       }

       local_irq_save(flags);
-       mask_IO_APIC_setup(ioapic_entries);
       mask_8259A();
+       mask_IO_APIC_setup(ioapic_entries);

-       ret = enable_intr_remapping(x2apic_supported());
-       if (ret)
-               goto end_restore;
+       ret = enable_IR();
+       if (!ret) {
+               /* IR is required if x2apic is enabled by BIOS even
+                * when running in kvm since this indicates present
+                * of APIC ID > 255 */
+               if (x2apic_preenabled || !kvm_para_available())
+                       goto nox2apic;
+               else
+                       /* without IR all CPUs can be addressed by IOAPIC/MSI
+                        * only in physical mode */
+                       x2apic_phys = 1;
+       }

Quoted 98 lines · Gleb Natapov

KVM would like to provide x2APIC interface to a guest without emulating interrupt remapping device. The reason KVM prefers guest to use x2APIC is that x2APIC interface is better virtualizable and provides better performance than mmio xAPIC interface:

- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface

Included patch changes x2APIC enabling logic to enable it even if IR initialization failed, but kernel runs under KVM and no apic id is greater than 255 (if there is one spec requires BIOS to move to x2apic mode before starting an OS).

Signed-off-by Gleb Natapov <[email protected]>

arch/x86/kernel/apic/apic.c +4 -6

index 8c7c042..351d55a 100644
@@ -49,6 +49,7 @@
  #include <asm/mtrr.h>
  #include <asm/smp.h>
  #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
  unsigned int num_processors;
 

@@ -1363,52 +1364,75 @@ void enable_x2apic(void)
  }
  #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
  {
  #ifdef CONFIG_INTR_REMAP
        int ret;
-       unsigned long flags;
-       struct IO_APIC_route_entry **ioapic_entries = NULL;
 
        ret = dmar_table_init();
        if (ret) {
                pr_debug("dmar_table_init() failed with %d:\n", ret);
-               goto ir_failed;
+               return 0;
        }
 
        if (!intr_remapping_supported()) {
                pr_debug("intr-remapping not supported\n");
-               goto ir_failed;
+               return 0;
        }
 
-
        if (!x2apic_preenabled && skip_ioapic_setup) {

               pr_info("Skipped enabling intr-remap because of skipping " "io-apic setup\n");
-               return;
+               return 0;
       }

+       if (enable_intr_remapping(x2apic_supported()))
+               return 0;

arch/x86/kernel/apic/apic.c +15 -0

+
+       pr_info("Enabled Interrupt-remapping\n");
+
+       return 1;
+
+#endif
+       return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+       unsigned long flags;
+       struct IO_APIC_route_entry **ioapic_entries = NULL;
+       int ret, x2apic_enabled = 0;
+

       ioapic_entries = alloc_ioapic_entries();
       if (!ioapic_entries) {
-               pr_info("Allocate ioapic_entries failed: %d\n", ret);
-               goto end;
+                pr_info("Allocate ioapic_entries failed\n");
+                goto out;
       }

       ret = save_IO_APIC_setup(ioapic_entries);
       if (ret) {
               pr_info("Saving IO-APIC state failed: %d\n", ret);
-               goto end;
+               goto out;
       }

       local_irq_save(flags);
-       mask_IO_APIC_setup(ioapic_entries);
       mask_8259A();
+       mask_IO_APIC_setup(ioapic_entries);

Quoted 15 lines

-       ret = enable_intr_remapping(x2apic_supported());
-       if (ret)
-               goto end_restore;
+       ret = enable_IR();
+       if (!ret) {
+               /* IR is required if x2apic is enabled by BIOS even
+                * when running in kvm since this indicates present
+                * of APIC ID > 255 */
+               if (x2apic_preenabled || !kvm_para_available())
+                       goto nox2apic;
+               else
+                       /* without IR all CPUs can be addressed by IOAPIC/MSI
+                        * only in physical mode */
+                       x2apic_phys = 1;
+       }

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

Quoted 5 lines

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

Yes, bummer. But the similar problem exist now and without KVM. After running x2apic enabled kernel you can't kexec kernel without x2apic support.

2009/6/30 Gleb Natapov <[email protected]>:

Quoted 5 lines

Quoted 5 lines

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

Quoted 3 lines

Yes, bummer. But the similar problem exist now and without KVM. After running x2apic enabled kernel you can't kexec kernel without x2apic support.

if the first kernel enable x2apic and intr_remapping, and second kernel has x2apic and intr_remap support.
we can kexec the second kernel.

restoring to original state is not possible for kdump case.

maybe with minor change to your patch, you could kexec the in kexec second kernel with x2apic support.

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

Quoted 4 lines · Yinghai Lu

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

By the way anybody knows why kexec does not use BIOS reset code (cmos 0xf offset) to jump into new kernel after hard reset?

Quoted 7 lines · Gleb Natapov

Quoted 5 lines · Yinghai Lu

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

Quoted 3 lines

By the way anybody knows why kexec does not use BIOS reset code (cmos 0xf offset) to jump into new kernel after hard reset?

After a hard reset. That simply isn't possible. A hard reset clears everything even memory.

You might be able to get a full cpu reset but not a reset of the I/O devices.

The premise of kexec is that we are doing things on our own, and don't get a 3rd piece of software involved that has not been heavily tested on the path we want to use. Occassionally it is a pain to do everything ourselves but at least when we do and we test it we know it is going to work.

Cpu designers lately seem fond of adding features that require all kind of coordination to turn on and off. We handle the hardware virtualization mode features now, and if x2apic has similar problems being turned on and off I am certain we can handle that case in a similar fashion.

When we can my preference is to keep code like that out of the kexec on panic path if we can figure out how to write the software to do something reasonable.

Once we figure out how to work without putting the interrupt controllers in legacy mode to handle the timer interrupts I expect all kinds of things will become simpler.

So the jump-to-vector reset code cannot work on real hardware? It works in qemu, but of course that's easy.

Quoted 19 lines

You might be able to get a full cpu reset but not a reset of the I/O devices.

The premise of kexec is that we are doing things on our own, and don't get a 3rd piece of software involved that has not been heavily tested on the path we want to use. Occassionally it is a pain to do everything ourselves but at least when we do and we test it we know it is going to work.

Cpu designers lately seem fond of adding features that require all kind of coordination to turn on and off. We handle the hardware virtualization mode features now, and if x2apic has similar problems being turned on and off I am certain we can handle that case in a similar fashion.

When we can my preference is to keep code like that out of the kexec on panic path if we can figure out how to write the software to do something reasonable.

Quoted 4 lines

Once we figure out how to work without putting the interrupt controllers in legacy mode to handle the timer interrupts I expect all kinds of things will become simpler.

I think it makes sense to add full reset support as a non-default option. Leaving hardware running and dmaing left and right has its risks.

Quoted 3 lines · Gleb Natapov

Quoted 2 lines · Yinghai Lu

how about kexec second kernel in KVM ?

Quoted 4 lines

Quoted 2 lines

x2apic_preenabled will be set in second kernel.

By the way anybody knows why kexec does not use BIOS reset code (cmos 0xf offset) to jump into new kernel after hard reset?

After a hard reset. That simply isn't possible. A hard reset clears everything even memory.

You might be able to get a full cpu reset but not a reset of the I/O devices.

The premise of kexec is that we are doing things on our own, and don't get a 3rd piece of software involved that has not been heavily tested on the path we want to use. Occassionally it is a pain to do everything ourselves but at least when we do and we test it we know it is going to work.

Cpu designers lately seem fond of adding features that require all kind of coordination to turn on and off. We handle the hardware virtualization mode features now, and if x2apic has similar problems being turned on and off I am certain we can handle that case in a similar fashion.

When we can my preference is to keep code like that out of the kexec on panic path if we can figure out how to write the software to do something reasonable.

Once we figure out how to work without putting the interrupt controllers in legacy mode to handle the timer interrupts I expect all kinds of things will become simpler.

Quoted 4 lines · Yinghai Lu

how about kexec second kernel in KVM ?

x2apic_preenabled will be set in second kernel.

By the way anybody knows why kexec does not use BIOS reset code (cmos 0xf offset) to jump into new kernel after hard reset?