x86: Reduce MAX_LOCAL_APIC and MAX_IO_APICS
RFC PATCH 1/1 x86: Reduce MAX_LOCAL_APIC and MAX_IO_APICS
Before this change MAX_LOCAL_APIC had the fixed value of 32*1024. Such a big value causes several data arrays to be quite oversized:
phys_cpu_present_map is 4 kbytes (one bit per apic id), __apicid_to_node[] is 64 kbytes, apic_version[] is 128 kbytes.
On "usual" systems, APIC ids simply go from zero to maximum logical CPU number, mirroring CPU ids.
On broken and unusual multi-socket systems
APIC ids can be non-contiguous.
This patch changes MAX_LOCAL_APIC definition as follows:
= It is guaranteed to be at least 16.
= If NR_CPUS > 16, then it's equal to NR_CPUS. = A new CONFIG_MAX_LAPIC_ID can be used to increase it
(but not decrease).
MAX_IO_APICS was 128. This is a bit large too, making, for example, ioapics[] array 9216 bytes big.
After this patch, MAX_IO_APICS is at least 8, at most 128. If NR_CPUS is in this range, then MAX_IO_APICS = NR_CPUS.
apic_version[] array is changed from int to u8 - APIC version values as of year 2015 are no larger than 0x1f on all known CPUs.
A bit of code added to ensure that the statement
apic_version[apicid] = version;
in generic_processor_info() is safe wrt bad values in both 'apicid' and 'version' variables.
This change reduces NR_CPUS=64 kernel's data size by 204661 bytes:
text data bss dec hex filename
91353669 13825744 19021824 124201237 7672915 vmlinux.before 91353680 13760336 18882560 123996576 76409a0 vmlinux
--- arch/x86/Kconfig | 11 +++++++++++ arch/x86/include/asm/apicdef.h | 23 +++++++++++++++++------ arch/x86/include/asm/mpspec.h | 2 +- arch/x86/kernel/apic/apic.c | 19 ++++++++++++++++++- 4 files changed, 47 insertions(+), 8 deletions(-)
arch/x86/Kconfig +11 -0
@@ -872,6 +872,17 @@ config NR_CPUS This is purely to save memory - each supported CPU adds approximately eight kilobytes to the kernel image. +config MAX_LAPIC_ID + int "Maximum APIC ID" + range 8 32768 + default "8" + ---help--- + Use this option to set maximum allowed Local APIC ID higher than + maximum number of CPUs. This may be necessary for machines + with large number of processor sockets and non-contiguous + LAPIC numbering. + This setting will be automatically rounded up, if necessary. + config SCHED_SMT bool "SMT (Hyperthreading) scheduler support" depends on SMP
arch/x86/include/asm/apicdef.h +17 -6
@@ -147,15 +147,26 @@#define XAPIC_ENABLE (1UL << 11)#define X2APIC_ENABLE (1UL << 10) -#ifdef CONFIG_X86_32 -# define MAX_IO_APICS 64 -# define MAX_LOCAL_APIC 256 -#else -# define MAX_IO_APICS 128 -# define MAX_LOCAL_APIC 32768 +/* +* Allow non-contiguous APIC IDs for small machines: +* APIC ids 0..15 are valid in any config. +* Typical SMP machines have contiguous APIC IDs: 0..NR_CPUS-1. +* CONFIG_MAX_LAPIC_ID can override. +*/ +#define MAX_LOCAL_APIC (NR_CPUS < 16 ? 16 : NR_CPUS) +#if MAX_LOCAL_APIC <CONFIG_MAX_LAPIC_ID +# undef MAX_LOCAL_APIC +# define MAX_LOCAL_APIC CONFIG_MAX_LAPIC_ID#endif /* +* Minimum is 8. +* For largish NR_CPUS, we expect to have no more IOAPICs than CPUs. +* No matter how large NR_CPUS is, max is 128. +*/ +#define MAX_IO_APICS (NR_CPUS < 8 ? 8 : NR_CPUS < 128 ? NR_CPUS : 128) + +/* * All x86-64 systems are xAPIC compatible. * In the following, "apicid" is a physical APIC ID. */
arch/x86/include/asm/mpspec.h +1 -1
@@ -6,7 +6,7 @@#include <asm/x86_init.h> #include <asm/apicdef.h> -extern int apic_version[]; +extern u8 apic_version[];extern int pic_mode;#ifdef CONFIG_X86_32
arch/x86/kernel/apic/apic.c +18 -1
@@ -1798,7 +1798,7 @@ void __init register_lapic_address(unsigned long address) } } -int apic_version[MAX_LOCAL_APIC ]; +u8 apic_version[MAX_LOCAL_APIC ];/* * Local APIC interrupts
@@ -2054,6 +2054,23 @@ int generic_processor_info(int apicid, int version)return -EINVAL ; } +if ((unsigned )apicid >=ARRAY_SIZE (apic_version)) { +int thiscpu = max + disabled_cpus; +pr_warning ("APIC: APIC id 0x%x is too large." +" Processor %d ignored.\n" , + apicid, thiscpu); + disabled_cpus++; +return -EINVAL ; + } +if ((unsigned )version >255 ) { +int thiscpu = max + disabled_cpus; +pr_warning ("APIC: APIC version 0x%x is too large." +" Processor %d ignored.\n" , + version, thiscpu); + disabled_cpus++; +return -EINVAL ; + } + num_processors++;if (apicid == boot_cpu_physical_apicid) {/*
Signature · 1 line
1.8.1.4
arch/x86/Kconfig config NR_CPUS · line 884
Quoted 11 lines · Denys Vlasenko
arch/x86/Kconfig +10
-0+config MAX_LAPIC_ID + int "Maximum APIC ID" + range 8 32768 + default "8" + ---help--- + Use this option to set maximum allowed Local APIC ID higher than + maximum number of CPUs. This may be necessary for machines + with large number of processor sockets and non-contiguous + LAPIC numbering. + This setting will be automatically rounded up, if necessary.
This is wrong. If you would limit the APIC IDs then you really break stuff. You can only limit the number of APICs.
ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
ACPI: LAPIC (acpi_id[0x01] lapic_id[0x02] enabled)
And that's not a really large machine..
Thanks,
tglx
Quoted 15 lines · Thomas Gleixner
arch/x86/Kconfig config NR_CPUS · line 884
Quoted 11 lines · Denys Vlasenko
arch/x86/Kconfig +10
-0+config MAX_LAPIC_ID + int "Maximum APIC ID" + range 8 32768 + default "8" + ---help--- + Use this option to set maximum allowed Local APIC ID higher than + maximum number of CPUs. This may be necessary for machines + with large number of processor sockets and non-contiguous + LAPIC numbering. + This setting will be automatically rounded up, if necessary.This is wrong. If you would limit the APIC IDs then you really break stuff. You can only limit the number of APICs.
This CONFIG setting allows to _increase_ max APIC ID.
Check out this part of the patch:
+/*
+ * Allow non-contiguous APIC IDs for small machines:
+ * APIC ids 0..15 are valid in any config.
+ * Typical SMP machines have contiguous APIC IDs: 0..NR_CPUS-1.
+ * CONFIG_MAX_LAPIC_ID can override.
+ */
+#define MAX_LOCAL_APIC (NR_CPUS < 16 ? 16 : NR_CPUS) +#if MAX_LOCAL_APIC < CONFIG_MAX_LAPIC_ID
+# undef MAX_LOCAL_APIC
+# define MAX_LOCAL_APIC CONFIG_MAX_LAPIC_ID
#endif
For example, if you'd build with NR_CPUS=128
(for example, Fedora kernels do that),
max accepted APIC id will be NR_CPUS-1 = 127
even if CONFIX_MAX_LAPIC_ID is 8.
If Fedora would want to support APIC ids up to
255, it will need to set CONFIG_MAX_LAPIC_ID=256.
Otherwise, if it's happy with "only" supporting up to 128, it does not need to change CONFIG_MAX_LAPIC_ID from default.
With current kernels, max APIC id for any kernel is 32768, which is in most cases way bigger than necessary.
Perhaps I need to update the text.
Something like:
- This setting will be automatically rounded up, if necessary
+ This setting will be increased to NR_CPUS, if necessary
Quoted 2 lines
ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
ACPI: LAPIC (acpi_id[0x01] lapic_id[0x02] enabled)
Does it mean that on a 2-CPU machine, CPU #1 has APIC_ID=2?
My patch will work fine for this machine,
with any CONFIG_MAX_LAPIC_ID.
Quoted 52 lines · Denys Vlasenko
Quoted 15 lines · Thomas Gleixner
arch/x86/Kconfig config NR_CPUS · line 884
Quoted 11 lines · Denys Vlasenko
arch/x86/Kconfig +10
-0+config MAX_LAPIC_ID + int "Maximum APIC ID" + range 8 32768 + default "8" + ---help--- + Use this option to set maximum allowed Local APIC ID higher than + maximum number of CPUs. This may be necessary for machines + with large number of processor sockets and non-contiguous + LAPIC numbering. + This setting will be automatically rounded up, if necessary.This is wrong. If you would limit the APIC IDs then you really break stuff. You can only limit the number of APICs.
This CONFIG setting allows to _increase_ max APIC ID.
Check out this part of the patch:
arch/x86/include/asm/apicdef.h +10
-0+/* +* Allow non-contiguous APIC IDs for small machines: +* APIC ids 0..15 are valid in any config. +* Typical SMP machines have contiguous APIC IDs: 0..NR_CPUS-1. +* CONFIG_MAX_LAPIC_ID can override. +*/ +#define MAX_LOCAL_APIC (NR_CPUS < 16 ? 16 : NR_CPUS) +#if MAX_LOCAL_APIC <CONFIG_MAX_LAPIC_ID +# undef MAX_LOCAL_APIC +# define MAX_LOCAL_APIC CONFIG_MAX_LAPIC_ID#endif For example, if you'd build with NR_CPUS=128
(for example, Fedora kernels do that),
max accepted APIC id will be NR_CPUS-1 = 127
even ifCONFIX_MAX_LAPIC_IDis 8.If Fedora would want to support APIC ids up to
255, it will need to setCONFIG_MAX_LAPIC_ID=256.Otherwise, if it's happy with "only" supporting up to 128, it does not need to change
CONFIG_MAX_LAPIC_IDfrom default.With current kernels, max APIC id for any kernel is 32768, which is in most cases way bigger than necessary.
Perhaps I need to update the text.
Something like:
Quoted 8 lines
- This setting will be automatically rounded up, if necessary
+ This setting will be increased toNR_CPUS, if necessaryQuoted 2 lines
ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
ACPI: LAPIC (acpi_id[0x01] lapic_id[0x02] enabled)Does it mean that on a 2-CPU machine, CPU #1 has APIC_ID=2?
Yes, APIC IDs are assigned by BIOS and may not be continuous.
Quoted 17 lines · Denys Vlasenko
Quoted 15 lines · Thomas Gleixner
arch/x86/Kconfig config NR_CPUS · line 884
Quoted 11 lines · Denys Vlasenko
arch/x86/Kconfig +10
-0+config MAX_LAPIC_ID + int "Maximum APIC ID" + range 8 32768 + default "8" + ---help--- + Use this option to set maximum allowed Local APIC ID higher than + maximum number of CPUs. This may be necessary for machines + with large number of processor sockets and non-contiguous + LAPIC numbering. + This setting will be automatically rounded up, if necessary.This is wrong. If you would limit the APIC IDs then you really break stuff. You can only limit the number of APICs.
Quoted 1 line
This CONFIG setting allows to _increase_ max APIC ID.
NO. This is crap. I don't want to tweak a gazillion of knobs just to build a kernel with CONFIG_NR_CPUS=8. Really not.
If you really want to make that space saving, then make it a runtime allocation.
Thanks,
tglx
Quoted 99 lines · Denys Vlasenko
Before this change
MAX_LOCAL_APIChad the fixed value of 32*1024. Such a big value causes several data arrays to be quite oversized:
phys_cpu_present_mapis 4 kbytes (one bit per apic id), __apicid_to_node[] is 64 kbytes, apic_version[] is 128 kbytes.On "usual" systems, APIC ids simply go from zero to maximum logical CPU number, mirroring CPU ids.
On broken and unusual multi-socket systems
APIC ids can be non-contiguous.This patch changes
MAX_LOCAL_APICdefinition as follows:= It is guaranteed to be at least 16.
= IfNR_CPUS> 16, then it's equal toNR_CPUS. = A newCONFIG_MAX_LAPIC_IDcan be used to increase it
(but not decrease).
MAX_IO_APICSwas 128. This is a bit large too, making, for example, ioapics[] array 9216 bytes big.After this patch,
MAX_IO_APICSis at least 8, at most 128. IfNR_CPUSis in this range, thenMAX_IO_APICS=NR_CPUS.apic_version[] array is changed from int to u8 - APIC version values as of year 2015 are no larger than
0x1fon all known CPUs.A bit of code added to ensure that the statement
apic_version[apicid] = version;in
generic_processor_info()is safe wrt bad values in both 'apicid' and 'version' variables.This change reduces NR_CPUS=64 kernel's data size by 204661 bytes:
text data bss dec hex filename
91353669 13825744 19021824 124201237 7672915 vmlinux.before 91353680 13760336 18882560 123996576 76409a0 vmlinuxSigned-off-by Denys Vlasenko <[email protected]>CC Ingo Molnar <[email protected]>CC Jiang Liu <[email protected]>CC Thomas Gleixner <[email protected]>CC Len Brown <[email protected]>--- arch/x86/Kconfig | 11 +++++++++++ arch/x86/include/asm/apicdef.h | 23 +++++++++++++++++------ arch/x86/include/asm/mpspec.h | 2 +- arch/x86/kernel/apic/apic.c | 19 ++++++++++++++++++- 4 files changed, 47 insertions(+), 8 deletions(-)arch/x86/Kconfig +11
-0@@ -872,6 +872,17 @@ config NR_CPUS This is purely to save memory - each supported CPU adds approximately eight kilobytes to the kernel image. +config MAX_LAPIC_ID + int "Maximum APIC ID" + range 8 32768 + default "8" + ---help--- + Use this option to set maximum allowed Local APIC ID higher than + maximum number of CPUs. This may be necessary for machines + with large number of processor sockets and non-contiguous + LAPIC numbering. + This setting will be automatically rounded up, if necessary. + config SCHED_SMT bool "SMT (Hyperthreading) scheduler support" depends on SMParch/x86/include/asm/apicdef.h +8
-6@@ -147,15 +147,26 @@#define XAPIC_ENABLE (1UL << 11)#define X2APIC_ENABLE (1UL << 10) -#ifdef CONFIG_X86_32 -# define MAX_IO_APICS 64 -# define MAX_LOCAL_APIC 256 -#else -# define MAX_IO_APICS 128 -# define MAX_LOCAL_APIC 32768 +/* +* Allow non-contiguous APIC IDs for small machines: +* APIC ids 0..15 are valid in any config. +* Typical SMP machines have contiguous APIC IDs: 0..NR_CPUS-1. +* CONFIG_MAX_LAPIC_ID can override. +*/ +#define MAX_LOCAL_APIC (NR_CPUS < 16 ? 16 : NR_CPUS) +#if MAX_LOCAL_APIC <CONFIG_MAX_LAPIC_ID
arch/x86/include/asm/apicdef.h line 167
Quoted 10 lines
arch/x86/include/asm/apicdef.h +7
-0+# undef MAX_LOCAL_APIC +# define MAX_LOCAL_APIC CONFIG_MAX_LAPIC_ID#endif /* +* Minimum is 8. +* For largish NR_CPUS, we expect to have no more IOAPICs than CPUs. +* No matter how large NR_CPUS is, max is 128. +*/ +#define MAX_IO_APICS (NR_CPUS < 8 ? 8 : NR_CPUS < 128 ? NR_CPUS : 128)
This is a little risky. For example, a typical eight-socket Intel platform will have nine IOAPICs. IO devices may get inaccessible if some IOAPICs are ignored due to MAX_IO_APICS limitation. It's a surprising if IO devices get lost if user runs a kernel built with low NR_CPUS.
Thanks!
Gerry