qemu-kvm introduce -maxcpus argument
PATCH 1/1 qemu-kvm introduce -maxcpus argument
Hi,
This one introduces a -maxcpus setting, allowing the user to specify the maximum number of vCPUs the system can have, as discussed with Gleb earlier in the week.
Cheers,
Jes
Introduce -maxcpus flag to QEMU and pass the value to the BIOS through FW_CFG.
Follow on patch will use it to determine the size of the MADT.
--- hw/fw_cfg.c | 1 + hw/fw_cfg.h | 1 + kvm/bios/rombios32.c | 16 ++++++++++++++++ qemu-options.hx | 9 +++++++++ sysemu.h | 1 + vl.c | 8 ++++++++ 6 files changed, 36 insertions(+)
hw/fw_cfg.c +1 -0
@@ -279,6 +279,7 @@ void *fw_cfg_init(uint32_t ctl_port, uinfw_cfg_add_bytes (s,FW_CFG_UUID , qemu_uuid,16 );fw_cfg_add_i16 (s,FW_CFG_NOGRAPHIC , (uint16_t )(display_type ==DT_NOGRAPHIC ));fw_cfg_add_i16 (s,FW_CFG_NB_CPUS , (uint16_t )smp_cpus); +fw_cfg_add_i16 (s,FW_CFG_MAX_CPUS , (uint16_t )max_cpus);register_savevm ("fw_cfg" ,-1 ,1 , fw_cfg_save, fw_cfg_load, s);qemu_register_reset (fw_cfg_reset,0 , s);
hw/fw_cfg.h +1 -0
@@ -15,6 +15,7 @@#define FW_CFG_INITRD_SIZE 0x0b#define FW_CFG_BOOT_DEVICE 0x0c#define FW_CFG_NUMA 0x0d +#define FW_CFG_MAX_CPUS 0x0e#define FW_CFG_MAX_ENTRY 0x10#define FW_CFG_WRITE_CHANNEL 0x4000
kvm/bios/rombios32.c +16 -0
@@ -441,6 +441,7 @@ void delay_ms(int n) }uint16_t smp_cpus; +uint16_t max_cpus =MAX_CPUS ;uint32_t cpuid_signature;uint32_t cpuid_features;uint32_t cpuid_ext_features;
@@ -484,6 +485,7 @@ void wrmsr_smp(uint32_t index, uint64_t#define QEMU_CFG_ID 0x01#define QEMU_CFG_UUID 0x02#define QEMU_CFG_NUMA 0x0D +#define QEMU_CFG_MAX_CPUS 0x0E#define QEMU_CFG_ARCH_LOCAL 0x8000#define QEMU_CFG_ACPI_TABLES (QEMU_CFG_ARCH_LOCAL + 0)#define QEMU_CFG_SMBIOS_ENTRIES (QEMU_CFG_ARCH_LOCAL + 1)
@@ -546,6 +548,19 @@ static uint16_t smbios_entries(void)return cnt; } +static uint16_t get_max_cpus (void ) +{ +uint16_t cnt; + +qemu_cfg_select (QEMU_CFG_MAX_CPUS ); +qemu_cfg_read ((uint8_t *)&cnt,sizeof (cnt)); + +if (!cnt) + cnt =MAX_CPUS ; + +return cnt; +} +uint64_t qemu_cfg_get64 (void ) {uint64_t ret;
@@ -1655,6 +1670,7 @@ void acpi_bios_init(void) addr +=sizeof (SSDTCode);#ifdef BX_QEMU + max_cpus =get_max_cpus ();qemu_cfg_select (QEMU_CFG_NUMA ); nb_numa_nodes =qemu_cfg_get64 ();#else
qemu-options.hx +9 -0
@@ -47,6 +47,15 @@ CPUs are supported. On Sparc32 target, L to 4. ETEXI +DEF("maxcpus", HAS_ARG, QEMU_OPTION_maxcpus, + "-maxcpus n set maximumthe number of possibly CPUs to 'n'\n") +STEXI +@item -maxcpus @var{n} +Set the maximum number of possible CPUs to @var(n). @var(n) has to be +bigger or equal to the value of -smp. If @var(n) is equal to -smp, +there will be no space for hotplug cpus to be added later. +ETEXI + DEF("numa", HAS_ARG, QEMU_OPTION_numa, "-numa node[,mem=size][,cpus=cpu[-cpu]][,nodeid=node]\n") STEXI
sysemu.h +1 -0
@@ -119,6 +119,7 @@ extern int alt_grab;extern int usb_enabled;extern int no_virtio_balloon;extern int smp_cpus; +extern int max_cpus;extern int cursor_hide;extern int graphic_rotate;extern int no_quit;
vl.c +8 -0
@@ -246,6 +246,7 @@ int singlestep = 0;const char *assigned_devices[MAX_DEV_ASSIGN_CMDLINE ];int assigned_devices_index;int smp_cpus =1 ; +int max_cpus =16 ;const char *vnc_display;int acpi_enabled =1 ;int no_hpet =0 ;
@@ -5666,6 +5667,13 @@ int main(int argc, char **argv, char **eexit (1 ); }break ; +case QEMU_OPTION_maxcpus: + max_cpus =atoi (optarg); +if ((max_cpus <1 ) || (max_cpus > machine->max_cpus)) { +fprintf (stderr,"Invalid number of CPUs\n" ); +exit (1 ); + } +break ;case QEMU_OPTION_vnc: display_type =DT_VNC ; vnc_display = optarg;
Quoted 5 lines · Jes Sorensen
Hi,
This one introduces a -maxcpus setting, allowing the user to specify the maximum number of vCPUs the system can have, as discussed with Gleb earlier in the week.
What is the maximum value for the 'maxcpus' setting for KVM ?
libvirt currently does
fd = open("/dev/kvm")
r = ioctl(fd, KVM_CHECK_EXTENSION, KVM_CAP_NR_VCPUS);to figure out what the maximum allowed vCPUs will be for KVM, and currently it is returning 16 IIRC.
Quoted 4 lines
vl.c +0
-0@@ -5666,6 +5667,13 @@ int main(int argc, char **argv, char **eexit (1 ); }break ;
vl.c int main(int argc, char **argv, char **e · line 5679
Quoted 10 lines
vl.c +7
-0+case QEMU_OPTION_maxcpus: + max_cpus =atoi (optarg); +if ((max_cpus <1 ) || (max_cpus > machine->max_cpus)) { +fprintf (stderr,"Invalid number of CPUs\n" ); +exit (1 ); + } +break ;case QEMU_OPTION_vnc: display_type =DT_VNC ; vnc_display = optarg;
This implies the limit (for x86 pc machine at least) is now 255. Is that the correct interpretation on my part ?
Regards,
Daniel
Signature · 4 lines
|: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
Quoted 8 lines · Daniel P. Berrange
Quoted 5 lines · Jes Sorensen
Hi,
This one introduces a -maxcpus setting, allowing the user to specify the maximum number of vCPUs the system can have, as discussed with Gleb earlier in the week.
What is the maximum value for the 'maxcpus' setting for KVM ?
Right now it is still 16, however I plan to change this.
Quoted 7 lines
libvirt currently does
fd = open("/dev/kvm") r = ioctl(fd, KVM_CHECK_EXTENSION, KVM_CAP_NR_VCPUS);to figure out what the maximum allowed vCPUs will be for KVM, and currently it is returning 16 IIRC.
Interesting, this will need to be addressed as well. I have plans to introduce a mechanism telling the kernel where the limit will be, in order to allow it to allocate data structures in a reasonable manner.
Quoted 5 lines
Quoted 5 lines
vl.c +1
-0@@ -5666,6 +5667,13 @@ int main(int argc, char **argv, char **eexit (1 ); }break ; +case QEMU_OPTION_maxcpus:
Quoted 8 lines
vl.c int main(int argc, char **argv, char **e · line 5675
Quoted 5 lines
vl.c +5
-0+ max_cpus =atoi (optarg); +if ((max_cpus<1 ) || (max_cpus> machine->max_cpus)) { +fprintf (stderr,"Invalid number of CPUs\n" ); +exit (1 ); + }This implies the limit (for x86 pc machine at least) is now 255. Is that the correct interpretation on my part ?
Actually the 255 limit is tied to ACPI. Once we support ACPI 3.0 and x2apic, it will get much worse. Be afraid, be very afraid :-) To be honest, that is going to be a while before we get to that, but I hope to get to it eventually.
I strongly recommend you try not to impose static limits within libvirt for the number of vCPUs.
I guess it will become a tricky issue who is telling who what the limit is. Ideally I would like to see the kernel limit becoming unlimited and the restrictions being set by userland.
Cheers,
Jes
Quoted 6 lines · Jes Sorensen
Quoted 6 lines
libvirt currently does
fd = open("/dev/kvm") r = ioctl(fd, KVM_CHECK_EXTENSION, KVM_CAP_NR_VCPUS);
Quoted 6 lines
Quoted 2 lines
to figure out what the maximum allowed vCPUs will be for KVM, and currently it is returning 16 IIRC.
Interesting, this will need to be addressed as well. I have plans to introduce a mechanism telling the kernel where the limit will be, in order to allow it to allocate data structures in a reasonable manner.
I prefer to have the kernel adjust dynamically. The vcpu array wants 8 bytes per vcpu, so for 256 vcpus we're still a very reasonable 2K. For (say) 4K vcpus, we'll need to reallocate the array dynamically, for which rcu will be perfect or better.
Quoted 8 lines
Quoted 2 lines
This implies the limit (for x86 pc machine at least) is now 255. Is that the correct interpretation on my part ?
Actually the 255 limit is tied to ACPI. Once we support ACPI 3.0 and x2apic, it will get much worse. Be afraid, be very afraid :-) To be honest, that is going to be a while before we get to that, but I hope to get to it eventually.
Quoted 6 lines
I strongly recommend you try not to impose static limits within libvirt for the number of vCPUs.
I guess it will become a tricky issue who is telling who what the limit is. Ideally I would like to see the kernel limit becoming unlimited and the restrictions being set by userland.
The kernel needs some kind of limit to avoid resource exhaustion. Could be set quite high. Beyond that there should be additional setup.
Note for 4K vcpus you'll need to increase the maximum opened files opened by a process, threads, etc.
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 5 lines · Jes Sorensen
Hi,
This one introduces a -maxcpus setting, allowing the user to specify the maximum number of vCPUs the system can have, as discussed with Gleb earlier in the week.
ACK, but please fix this:
+DEF("maxcpus", HAS_ARG, QEMU_OPTION_maxcpus,
+ "-maxcpus n set maximumthe number of possibly CPUs to 'n'\n")
+STEXI
Amit
Review
Quoted 6 lines · Amit Shah
On (Fri) Jun 19 2009 [15:59:57], Jes Sorensen wrote:
ACK, but please fix this:(quoted diff) +3
-0+DEF("maxcpus", HAS_ARG, QEMU_OPTION_maxcpus, + "-maxcpus n set maximumthe number of possibly CPUs to 'n'\n") +STEXI
Argh, thanks, dunno how I managed to create that.
Here is an updated version.
Cheers,
Jes
Build madt table based on max_cpus count.
--- kvm/bios/rombios32.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
kvm/bios/rombios32.c +4 -4
@@ -1168,13 +1168,13 @@ static void mptable_init(void)putstr (&q,"0.1 " );/* vendor id */ putle32 (&q,0 );/* OEM table ptr */ putle16 (&q,0 );/* OEM table size */ -putle16 (&q,MAX_CPUS +18 );/* entry count */ +putle16 (&q, max_cpus +18 );/* entry count */ putle32 (&q,0xfee00000 );/* local APIC addr */ putle16 (&q,0 );/* ext table length */ putb (&q,0 );/* ext table checksum */ putb (&q,0 );/* reserved */ -for (i =0 ; i <MAX_CPUS ; i++) { +for (i =0 ; i < max_cpus ; i++) {putb (&q,0 );/* entry type = processor */ putb (&q, i);/* APIC id */ putb (&q,0x11 );/* local APIC version number */
@@ -1700,7 +1700,7 @@ void acpi_bios_init(void) addr = (addr +7 ) & ~7 ; madt_addr = addr; madt_size = sizeof(*madt) + - sizeof(struct madt_processor_apic) * MAX_CPUS + + sizeof(struct madt_processor_apic) * max_cpus +#ifdef BX_QEMU sizeof (struct madt_io_apic ) +sizeof (struct madt_int_override ) *MAX_INT_OVERRIDES ;#else
@@ -1778,7 +1778,7 @@ void acpi_bios_init(void) madt->local_apic_address =cpu_to_le32 (0xfee00000 ); madt->flags =cpu_to_le32 (1 ); *(uint32_t *)APIC_MADT_PTR = apic = (void *)(madt +1 ); -for (i=0 ;i<MAX_CPUS ;i++) { +for (i=0 ; i < max_cpus; i++) { apic->type =APIC_PROCESSOR ; apic->length =sizeof (*apic); apic->processor_id = i;
Quoted 6 lines · Jes Sorensen
Hi,
This one introduces a -maxcpus setting, allowing the user to specify the maximum number of vCPUs the system can have, as discussed with Gleb earlier in the week.
Patch looks good. There is nothing kvm-specific about it, so please send it to qemu-devel@. Also, please split the qemu part from the bios part.
Signature · 1 line
error compiling committee.c: too many arguments to function