QEMU maxcpus support v2
cover QEMU maxcpus support v2
Hi,
Here is an updated version of the ,maxcpus patch, now adding it as a sub-option to -smp, ie, -smp <n>,maxcpus=<y>. In addition it has the QEMU BOCHS patches to make the BIOS use the value passed.
Specifying maxcpus= allows the user to cap the limit of possibly cpus of the system, as opposed to -smp which specifies the number of running cpus at startup. This allows us to build BIOS tables accordingly when trying to scale up in size, and it also allows for the BIOS to build smaller tables when not needed.
I am going to submit a set of patches directly for BOCHS as well.
Cheers,
Jes
PATCH 1/2 Introduce -smp , maxcpus= flag to specify maximum number of CPUS.
Follow on patch will use it to determine the size of the MADT and other BIOS tables.
--- hw/fw_cfg.c | 1 + hw/fw_cfg.h | 1 + qemu-options.hx | 5 ++++- sysemu.h | 1 + vl.c | 20 +++++++++++++++++++- 5 files changed, 26 insertions(+), 2 deletions(-)
hw/fw_cfg.c +1 -0
@@ -279,6 +279,7 @@ void *fw_cfg_init(uint32_t ctl_port, uinfw_cfg_add_bytes (s,FW_CFG_UUID , qemu_uuid,16 );fw_cfg_add_i16 (s,FW_CFG_NOGRAPHIC , (uint16_t )(display_type ==DT_NOGRAPHIC ));fw_cfg_add_i16 (s,FW_CFG_NB_CPUS , (uint16_t )smp_cpus); +fw_cfg_add_i16 (s,FW_CFG_MAX_CPUS , (uint16_t )max_cpus);register_savevm ("fw_cfg" ,-1 ,1 , fw_cfg_save, fw_cfg_load, s);qemu_register_reset (fw_cfg_reset,0 , s);
hw/fw_cfg.h +1 -0
@@ -15,6 +15,7 @@#define FW_CFG_INITRD_SIZE 0x0b#define FW_CFG_BOOT_DEVICE 0x0c#define FW_CFG_NUMA 0x0d +#define FW_CFG_MAX_CPUS 0x0e#define FW_CFG_MAX_ENTRY 0x10#define FW_CFG_WRITE_CHANNEL 0x4000
qemu-options.hx +4 -1
@@ -39,7 +39,10 @@ Select CPU model (-cpu ? for list and ad ETEXI DEF("smp", HAS_ARG, QEMU_OPTION_smp, - "-smp n set the number of CPUs to 'n' [default=1]\n") + "-smp n[,maxcpus=cpus]\n" + " set the number of CPUs to 'n' [default=1]\n" + " maxcpus= maximum number of total cpus, including\n" + " offline CPUs for hotplug etc.\n") STEXI @item -smp @var{n} Simulate an SMP system with @var{n} CPUs. On the PC target, up to 255
sysemu.h +1 -0
@@ -118,6 +118,7 @@ extern int alt_grab;extern int usb_enabled;extern int no_virtio_balloon;extern int smp_cpus; +extern int max_cpus;extern int cursor_hide;extern int graphic_rotate;extern int no_quit;
vl.c +19 -1
@@ -241,6 +241,7 @@ int rtc_td_hack = 0;int usb_enabled =0 ;int singlestep =0 ;int smp_cpus =1 ; +int max_cpus =16 ;const char *vnc_display;int acpi_enabled =1 ;int no_hpet =0 ;
@@ -5550,12 +5551,29 @@ int main(int argc, char **argv, char **e usb_devices_index++;break ;case QEMU_OPTION_smp: - smp_cpus =atoi (optarg); + { +char *p; +char option[128 ]; + smp_cpus =strtol (optarg, &p,10 );if (smp_cpus <1 ) {fprintf (stderr,"Invalid number of CPUs\n" );exit (1 ); } +if (*p++ !=',' ) +break ; +if (get_param_value (option,128 ,"maxcpus" , p)) + max_cpus =strtol (option,NULL ,0 ); +if (max_cpus < smp_cpus) { +fprintf (stderr,"maxcpus must be equal to or greater than " +"smp\n" ); +exit (1 ); + } +if (max_cpus >255 ) { +fprintf (stderr,"Unsupported number of maxcpus\n" ); +exit (1 ); + }break ; + }case QEMU_OPTION_vnc: display_type =DT_VNC ; vnc_display = optarg;
PATCH 2/2 QEMU BOCHS bios patches to use maxcpus value.
In reply to Jes Sorensen--- pc-bios/bios-pq/0017-qemu-kvm-cfg-maxcpus.patch | 57 +++++++++++++ pc-bios/bios-pq/0018-qemu-madt-maxcpus.patch | 105 ++++++++++++++++++++++++ pc-bios/bios-pq/series | 2 3 files changed, 164 insertions(+)
pc-bios/bios-pq/0017-qemu-kvm-cfg-maxcpus.patch new file +57 -0
@@ -0,0 +1,57 @@ +Read max_cpus variable from QEMU_CFG. If not provided, use value of +smp_cpus. + +Signed-off-by: Jes Sorensen <[email protected]> + +diff --git a/bios/rombios32.c b/bios/rombios32.c +index f861f81..f4a6300 100644 +--- a/bios/rombios32.c ++++ b/bios/rombios32.c +@@ -436,6 +436,7 @@ void delay_ms(int n) + } + + uint16_t smp_cpus; ++uint16_t max_cpus; + uint32_t cpuid_signature; + uint32_t cpuid_features; + uint32_t cpuid_ext_features; +@@ -474,6 +475,7 @@ void wrmsr_smp(uint32_t index, uint64_t val) + #define QEMU_CFG_ID 0x01 + #define QEMU_CFG_UUID 0x02 + #define QEMU_CFG_NUMA 0x0D ++#define QEMU_CFG_MAX_CPUS 0x0E + #define QEMU_CFG_ARCH_LOCAL 0x8000 + #define QEMU_CFG_ACPI_TABLES (QEMU_CFG_ARCH_LOCAL + 0) + #define QEMU_CFG_SMBIOS_ENTRIES (QEMU_CFG_ARCH_LOCAL + 1) +@@ -536,6 +538,19 @@ static uint16_t smbios_entries(void) + return cnt; + } + ++static uint16_t get_max_cpus(void) ++{ ++ uint16_t cnt; ++ ++ qemu_cfg_select(QEMU_CFG_MAX_CPUS); ++ qemu_cfg_read((uint8_t*)&cnt, sizeof(cnt)); ++ ++ if (!cnt) ++ cnt = smp_cpus; ++ ++ return cnt; ++} ++ + uint64_t qemu_cfg_get64 (void) + { + uint64_t ret; +@@ -1655,6 +1670,11 @@ void acpi_bios_init(void) + uint16_t i, external_tables; + int nb_numa_nodes; + ++#ifdef BX_QEMU ++ max_cpus = get_max_cpus(); ++#else ++ max_cpus = smp_cpus; ++#endif + /* reserve memory space for tables */ + #ifdef BX_USE_EBDA_TABLES + ebda_cur_addr = align(ebda_cur_addr, 16);
pc-bios/bios-pq/0018-qemu-madt-maxcpus.patch new file +105 -0
105 lines not on this page · raw message
pc-bios/bios-pq/series +2 -0
@@ -14,3 +14,5 @@ 0014_add-srat-acpi-table-support.patch 0015_enable-power-button-even-generation.patch 0016-use-correct-mask-to-size-pci-option-rom-bar.patch +0017-qemu-kvm-cfg-maxcpus.patch +0018-qemu-madt-maxcpus.patch
[Qemu-devel] Re: [patch 1/2] Introduce -smp , maxcpus= flag to specify maximum number of CPUS.
In reply to Jes SorensenQuoted 5 lines · Jes Sorensen
Follow on patch will use it to determine the size of the MADT and other BIOS tables.
+int
max_cpus= 16;
Again I should have noticed this before, but this and the other constants should come from the QEMUMachine we're targeting, since each target has different limits.
Signature · 1 line
error compiling committee.c: too many arguments to function
[Qemu-devel] Re: [patch 1/2] Introduce -smp , maxcpus= flag to specify maximum number of CPUS.
Quoted 9 lines · Avi Kivity
Quoted 4 lines · Jes Sorensen
Follow on patch will use it to determine the size of the MADT and other BIOS tables.
+int
max_cpus= 16;Again I should have noticed this before, but this and the other constants should come from the QEMUMachine we're targeting, since each target has different limits.
Actually this is on purpose. The PC machine has a max limit of 255, if we use that as the max per default, you are going to get a BIOS declaring 255 possible CPUs and certain broken operating systems are not going to boot.
So I will argue the code is correct as is.
Cheers,
Jes
[Qemu-devel] Re: [patch 1/2] Introduce -smp , maxcpus= flag to specify maximum number of CPUS.
Quoted 11 lines · Jes Sorensen
Quoted 9 lines · Avi Kivity
Quoted 4 lines · Jes Sorensen
Follow on patch will use it to determine the size of the MADT and other BIOS tables.
+int
max_cpus= 16;Again I should have noticed this before, but this and the other constants should come from the QEMUMachine we're targeting, since each target has different limits.
Quoted 7 lines
Actually this is on purpose. The PC machine has a max limit of 255, if we use that as the max per default, you are going to get a BIOS declaring 255 possible CPUs and certain broken operating systems are not going to boot.
So I will argue the code is correct as is.
What about the various ARMs, powerpcs, and sparcs that we support?
Signature · 1 line
error compiling committee.c: too many arguments to function
[Qemu-devel] Re: [patch 1/2] Introduce -smp , maxcpus= flag to specify maximum number of CPUS.
Quoted 9 lines · Avi Kivity
Quoted 6 lines · Jes Sorensen
Actually this is on purpose. The PC machine has a max limit of 255, if we use that as the max per default, you are going to get a BIOS declaring 255 possible CPUs and certain broken operating systems are not going to boot.
So I will argue the code is correct as is.
What about the various ARMs, powerpcs, and sparcs that we support?
None of them are affected by the maxcpus patch. They still have their check making sure that the -smp argument doesn't exceed whats specified in their machine description.
Jes
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
In reply to Jes SorensenThese changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Regards,
Anthony Liguori
Jes Sorensen wrote:
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
Quoted 4 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Why shouldn't we do cpu hotplug?
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
Quoted 7 lines · Avi Kivity
Quoted 4 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Why shouldn't we do cpu hotplug?
I don't think we should do cpu hotplug via ACPI. I don't think ACPI actually models CPU hotplug and the fact that this works with Linux in KVM is a happy accident. VMware only supports CPU hotplug for Windows 7/2k8 guests so I'm assuming their using Viridian PV extensions to do it.
I think we should go the PV route for Linux too. I'd rather see us create all vcpu threads at once and then let the guest offline each vcpu via a PV notification. I don't see a lot of value in spawning/terminating vcpu threads dynamically and it adds an awful lot of complexity. There's very little overhead to an idle thread.
Regards,
Anthony Liguori
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Quoted 2 lines
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Why shouldn't we do cpu hotplug?
I don't think we should do cpu hotplug via ACPI.
Well, that's a different story from "we shouldn't do cou hotplug".
Quoted 1 line
I don't think ACPI actually models CPU hotplug and the fact that this works with Linux in KVM is a happy accident.
I think it's based on the Unisys machines and thus no accident.
Quoted 1 line
VMware only supports CPU hotplug for Windows 7/2k8 guests so I'm assuming their using Viridian PV extensions to do it.
I don't recall seeing hotplug support in Viridian. Further, Windows 2008 appears to support cpu hotplug on bare metal. My guess is that it uses ACPI, perhaps with an additional vendor driver.
Quoted 1 line
I think we should go the PV route for Linux too.
Seems like needless churn, plus disabling that functionality for older kernels.
Quoted 1 line
I'd rather see us create all vcpu threads at once and then let the guest offline each vcpu via a PV notification.
That doesn't work, for example, when the guest reboots into an older version of itself. It also gives control of resource usage to the guest instead of the host. The latter issue can be fixed using control groups, but I'd rather not break it in the first place.
Quoted 1 line
I don't see a lot of value in spawning/terminating vcpu threads dynamically and it adds an awful lot of complexity. There's very little overhead to an idle thread.
Can you elaborate on the complexity you think dynamic vcpu threads add? IIRC there's some synchronization to be done at startup, but nothing that merits the label "awful".
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
In reply to Anthony LiguoriQuoted 5 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Anthony,
Sorry haven't gotten back to you earlier as I was on vacation. Are you saying the Ubuntu kernel doesn't like having more CPU entries in the ACPI table than it actually boots on?
Does the same guest boot using an older KVM setup? Curious since it does have the larger CPU table in the DSDT.
Cheers,
Jes
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
Quoted 14 lines · Jes Sorensen
Quoted 6 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Anthony,
Sorry haven't gotten back to you earlier as I was on vacation. Are you saying the Ubuntu kernel doesn't like having more CPU entries in the ACPI table than it actually boots on?
Quoted 6 lines
Does the same guest boot using an older KVM setup? Curious since it does have the larger CPU table in the DSDT.
Cheers,
Jes
BTW, many other guests complain when ACPI describes more processors than actually present in machine. That's why I implemented the dynamic DSDT generation in Bochs BIOS in the first place. One that comes to mind is MacOS X, or the Darwin kernel respectively.
Best regards,
Filip Navara
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
Quoted 18 lines · Filip Navara
Quoted 17 lines · Jes Sorensen
Quoted 6 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Anthony,
Sorry haven't gotten back to you earlier as I was on vacation. Are you saying the Ubuntu kernel doesn't like having more CPU entries in the ACPI table than it actually boots on?
Does the same guest boot using an older KVM setup? Curious since it does have the larger CPU table in the DSDT.
Quoted 9 lines
Quoted 3 lines
Cheers,
JesBTW, many other guests complain when ACPI describes more processors than actually present in machine. That's why I implemented the dynamic DSDT generation in Bochs BIOS in the first place. One that comes to mind is MacOS X, or the Darwin kernel respectively.
There is nothing wrong in describing more processors than actually present. The disable flag is defined by ACPI for a reason. My real HW IBM server does this.
--
Gleb.
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
Quoted 23 lines · Gleb Natapov
Quoted 22 lines · Filip Navara
Quoted 20 lines · Jes Sorensen
Quoted 6 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Anthony,
Sorry haven't gotten back to you earlier as I was on vacation. Are you saying the Ubuntu kernel doesn't like having more CPU entries in the ACPI table than it actually boots on?
Does the same guest boot using an older KVM setup? Curious since it does have the larger CPU table in the DSDT.
Cheers,
Jes
Quoted 8 lines
Quoted 5 lines
BTW, many other guests complain when ACPI describes more processors than actually present in machine. That's why I implemented the dynamic DSDT generation in Bochs BIOS in the first place. One that comes to mind is MacOS X, or the Darwin kernel respectively.
There is nothing wrong in describing more processors than actually present. The disable flag is defined by ACPI for a reason. My real HW IBM server does this.
Yeah, last time I tried MacOS X was happy with more CPU descriptions than actual CPUs too as long as they were in disabled state. Has anything changed there I should know about?
Alex
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
In reply to Filip NavaraQuoted 4 lines · Filip Navara
BTW, many other guests complain when ACPI describes more processors than actually present in machine. That's why I implemented the dynamic DSDT generation in Bochs BIOS in the first place. One that comes to mind is MacOS X, or the Darwin kernel respectively.
Hi Filip,
As Gleb mentioned, this is pretty common behavior on real hardware. Most of the boards I have here, declare the maximum number of possible CPUs, even when they are not all active (like if you disable hyper threading).
I like the idea of dynamic declerations, but it's problematic as the ACPI code gets more complicated.
I'd like to try and find out what broke in Anthony's case.
Cheers,
Jes
Re: [Qemu-devel] [patch 2/2] QEMU BOCHS bios patches to use maxcpus value.
In reply to Alexander GrafQuoted 37 lines · Alexander Graf
Quoted 25 lines · Gleb Natapov
Quoted 24 lines · Filip Navara
Quoted 21 lines · Jes Sorensen
Quoted 6 lines · Anthony Liguori
These changes make my Ubuntu server guest very unhappy. I get a bunch of messages about "Not responding." on startup.
If nothing else, maxcpus ==smp_cpus under QEMU because we don't do CPU hotplug (and I don't think we should).
Anthony,
Sorry haven't gotten back to you earlier as I was on vacation. Are you saying the Ubuntu kernel doesn't like having more CPU entries in the ACPI table than it actually boots on?
Does the same guest boot using an older KVM setup? Curious since it does have the larger CPU table in the DSDT.
Cheers,
JesQuoted 8 lines
Quoted 5 lines
BTW, many other guests complain when ACPI describes more processors than actually present in machine. That's why I implemented the dynamic DSDT generation in Bochs BIOS in the first place. One that comes to mind is MacOS X, or the Darwin kernel respectively.
There is nothing wrong in describing more processors than actually present. The disable flag is defined by ACPI for a reason. My real HW IBM server does this.
Quoted 5 lines
Yeah, last time I tried MacOS X was happy with more CPU descriptions than actual CPUs too as long as they were in disabled state. Has anything changed there I should know about?
Alex
Not really... it just spits warnings that are not seen unless you explicitly ask for complete debug/io logs (not sure which one it is in). In any case the disabled flag was not set in QEMU when I was debugging the code.
Best regards,
Filip Navara