x86/mm/ident_map: Add PUD level 1GB page support
Subsystems (5)
PATCH v3 1/2 x86/mm/ident_map: Add PUD level 1GB page support
The current kernel_ident_mapping_init() creates the identity mapping always using 2MB page(PMD level), this patch adds the 1GB page(PUD level) support.
The primary advantage would be better TLB coverage/performance, because we'd utilize 1GB TLBs instead of 2MB ones.
It is also useful for machines with large number of memory to save paging structure allocations(around 4MB/TB using 2MB page) when setting identity mappings for all the memory, after using 1GB page it will consume only 8KB/TB.
--- arch/x86/boot/compressed/pagetable.c | 2 +- arch/x86/include/asm/init.h | 3 ++- arch/x86/kernel/machine_kexec_64.c | 2 +- arch/x86/mm/ident_map.c | 14 +++++++++++++- arch/x86/power/hibernate_64.c | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-)
arch/x86/boot/compressed/pagetable.c +1 -1
@@ -70,7 +70,7 @@ static void *alloc_pgt_page(void *context)* Due to relocation, pointers must be assigned at run time not build time. */ static struct x86_mapping_info mapping_info = { - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, + .page_flag = __PAGE_KERNEL_LARGE_EXEC, };/* Locates and clears a region for a new top level page table. */
arch/x86/include/asm/init.h +2 -1
@@ -4,8 +4,9 @@struct x86_mapping_info {void *(*alloc_pgt_page)(void *);/* allocate buf for page table */ void *context;/* context for alloc_pgt_page */ -unsigned long pmd_flag;/* page flag for PMD entry */ +unsigned long page_flag;/* page flag for PMD or PUD entry */ unsigned long offset;/* ident mapping offset */ +bool direct_gbpages;/* PUD level 1GB page support */ }; int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
arch/x86/kernel/machine_kexec_64.c +1 -1
@@ -113,7 +113,7 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)struct x86_mapping_info info = { .alloc_pgt_page = alloc_pgt_page, .context = image, - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, + .page_flag = __PAGE_KERNEL_LARGE_EXEC, };unsigned long mstart, mend;pgd_t *level4p;
arch/x86/mm/ident_map.c +13 -1
@@ -13,7 +13,7 @@ static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,if (pmd_present (*pmd))continue ; -set_pmd (pmd,__pmd ((addr - info->offset) | info->pmd_flag)); +set_pmd (pmd,__pmd ((addr - info->offset) | info->page_flag)); } }
@@ -30,6 +30,18 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,if (next > end) next = end; +if (info->direct_gbpages) { +pud_t pudval; + +if (pud_present (*pud)) +continue ; + + addr &=PUD_MASK ; + pudval =__pud ((addr - info->offset) | info->page_flag); +set_pud (pud, pudval); +continue ; + } +if (pud_present (*pud)) { pmd =pmd_offset (pud,0 );ident_pmd_init (info, pmd, addr, next);
arch/x86/power/hibernate_64.c +1 -1
@@ -104,7 +104,7 @@ static int set_up_temporary_mappings(void) {struct x86_mapping_info info = { .alloc_pgt_page = alloc_pgt_page, - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, + .page_flag = __PAGE_KERNEL_LARGE_EXEC, .offset = __PAGE_OFFSET, };unsigned long mstart, mend;
Signature · 1 line
1.8.3.1
PATCH v3 2/2 x86_64/kexec: Use PUD level 1GB page for identity mapping if available
Kexec setups all identity mappings before booting into the new kernel, and this will cause extra memory consumption for paging structures which is quite considerable on modern machines with huge number of memory.
E.g. On one 32TB machine, in kdump case, it could waste around 128MB (around 4MB/TB) from the reserved memory after kexec set all the identity mappings using the current 2MB page, plus the loaded kdump kernel, initramfs, etc, it caused kexec syscall -NOMEM failure. As a result, we had to enlarge reserved memory via "crashkernel=X".
This causes some trouble for distributions that use policies to evaluate the proper "crashkernel=X" value for users.
Given that on machines with large number of memory, 1GB feature is very likely available, and that kernel_ident_mapping_init() supports PUD level 1GB page, to solve this problem, we use 1GB size page to create the identity mapping pgtable for kdump if 1GB feature is available.
--- arch/x86/kernel/machine_kexec_64.c | 4 ++++ 1 file changed, 4 insertions(+)
arch/x86/kernel/machine_kexec_64.c +4 -0
@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable) level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true; +for (i =0 ; i < nr_pfn_mapped; i++) { mstart = pfn_mapped[i].start <<PAGE_SHIFT ; mend = pfn_mapped[i].end <<PAGE_SHIFT ;
Signature · 1 line
1.8.3.1
Re: [PATCH v3 2/2] x86_64/kexec: Use PUD level 1GB page for identity mapping if available
arch/x86/kernel/machine_kexec_64.c static int init_pgtable(struct kimage *image, unsigned long start_pgtable) · line 127
Quoted 7 lines · Xunlei Pang
arch/x86/kernel/machine_kexec_64.c +3
-0@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable) level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true;
No, this should be keyed off the CPU feature (X86_FEATURE_GBPAGES) automatically, not set blindly! AFAICS this patch will crash kexec on any CPU that does not support gbpages.
I only noticed this problem after having fixed/enhanced all the changelogs - so please pick up the new changelog up from the log below.
Thanks,
Ingo
============================>
Author: Xunlei Pang <[email protected]>
x86/mm: Add support for gbpages to kernel_ident_mapping_init()
Kernel identity mappings on x86-64 kernels are created in two
ways: by the early x86 boot code, or by kernel_ident_mapping_init().
Native kernels (which is the dominant usecase) use the former, but the kexec and the hibernation code uses kernel_ident_mapping_init().
There's a subtle difference between these two ways of how identity mappings are created, the current kernel_ident_mapping_init() code creates identity mappings always using 2MB page(PMD level) - while the native kernel boot path also utilizes gbpages where available.
This difference is suboptimal both for performance and for memory
usage: kernel_ident_mapping_init() needs to allocate pages for the page tables when creating the new identity mappings.
This patch adds 1GB page(PUD level) support to kernel_ident_mapping_init() to address these concerns.
The primary advantage would be better TLB coverage/performance, because we'd utilize 1GB TLBs instead of 2MB ones.
It is also useful for machines with large number of memory to save paging structure allocations(around 4MB/TB using 2MB page) when setting identity mappings for all the memory, after using 1GB page it will consume only 8KB/TB.
( Note that this change alone does not activate gbpages in kexec,
we are doing that in a separate patch. )
Re: [PATCH v3 2/2] x86_64/kexec: Use PUD level 1GB page for identity mapping if available
Quoted 4 lines · Ingo Molnar
Quoted 2 lines · Xunlei Pang
(quoted diff) +0
-0@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)
Quoted 8 lines
arch/x86/kernel/machine_kexec_64.c static int init_pgtable(struct kimage *image, unsigned long start_pgtable) · line 127
Quoted 5 lines
arch/x86/kernel/machine_kexec_64.c +3
-0level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true;No, this should be keyed off the CPU feature (
X86_FEATURE_GBPAGES) automatically, not set blindly! AFAICS this patch will crash kexec on any CPU that does not support gbpages.
It should be fine, probe_page_size_mask() already takes care of this:
if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) {
printk(KERN_INFO "Using GB pages for direct mapping\n");
page_size_mask |= 1 << PG_LEVEL_1G;
} else {
direct_gbpages = 0;
}So if X86_FEATURE_GBPAGES is not supported, direct_gbpages will be set to 0.
Quoted 3 lines
I only noticed this problem after having fixed/enhanced all the changelogs - so please pick up the new changelog up from the log below.
Thanks for the rewrite, it looks better.
Regards,
Xunlei
Quoted 41 lines
Thanks,
Ingo
============================>
Author: Xunlei Pang <[email protected]>
x86/mm: Add support for gbpages to
kernel_ident_mapping_init()Kernel identity mappings on x86-64 kernels are created in two
ways: by the early x86 boot code, or bykernel_ident_mapping_init().Native kernels (which is the dominant usecase) use the former, but the kexec and the hibernation code uses
kernel_ident_mapping_init().There's a subtle difference between these two ways of how identity mappings are created, the current
kernel_ident_mapping_init()code creates identity mappings always using 2MB page(PMD level) - while the native kernel boot path also utilizes gbpages where available.This difference is suboptimal both for performance and for memory
usage:kernel_ident_mapping_init()needs to allocate pages for the page tables when creating the new identity mappings.This patch adds 1GB page(PUD level) support to
kernel_ident_mapping_init()to address these concerns.The primary advantage would be better TLB coverage/performance, because we'd utilize 1GB TLBs instead of 2MB ones.
It is also useful for machines with large number of memory to save paging structure allocations(around 4MB/TB using 2MB page) when setting identity mappings for all the memory, after using 1GB page it will consume only 8KB/TB.
( Note that this change alone does not activate gbpages in kexec,
we are doing that in a separate patch. )
Re: [PATCH v3 2/2] x86_64/kexec: Use PUD level 1GB page for identity mapping if available
Quoted 15 lines · Xunlei Pang
Quoted 4 lines · Ingo Molnar
Quoted 2 lines · Xunlei Pang
(quoted diff) +0
-0@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)Quoted 8 lines
arch/x86/kernel/machine_kexec_64.c static int init_pgtable(struct kimage *image, unsigned long start_pgtable) · line 127
Quoted 5 lines
arch/x86/kernel/machine_kexec_64.c +3
-0level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true;No, this should be keyed off the CPU feature (
X86_FEATURE_GBPAGES) automatically, not set blindly! AFAICS this patch will crash kexec on any CPU that does not support gbpages.It should be fine,
probe_page_size_mask()already takes care of this:
Quoted 8 lines
if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) { printk(KERN_INFO "Using GB pages for direct mapping\n"); page_size_mask |= 1 << PG_LEVEL_1G; } else { direct_gbpages = 0; }So if
X86_FEATURE_GBPAGESis not supported,direct_gbpageswill be set to 0.
So why is the introduction of the info.direct_gbpages flag necessary? AFAICS it just duplicates the kernel's direct_gbpages flag. One outcome is that hibernation won't use gbpages, which is silly.
Thanks,
Ingo
Re: [PATCH v3 2/2] x86_64/kexec: Use PUD level 1GB page for identity mapping if available
Quoted 23 lines · Ingo Molnar
Quoted 21 lines · Xunlei Pang
Quoted 4 lines · Ingo Molnar
Quoted 2 lines · Xunlei Pang
(quoted diff) +0
-0@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)Quoted 8 lines
arch/x86/kernel/machine_kexec_64.c static int init_pgtable(struct kimage *image, unsigned long start_pgtable) · line 127
Quoted 5 lines
arch/x86/kernel/machine_kexec_64.c +3
-0level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true;No, this should be keyed off the CPU feature (
X86_FEATURE_GBPAGES) automatically, not set blindly! AFAICS this patch will crash kexec on any CPU that does not support gbpages.It should be fine,
probe_page_size_mask()already takes care of this:if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) { printk(KERN_INFO "Using GB pages for direct mapping\n"); page_size_mask |= 1 << PG_LEVEL_1G; } else { direct_gbpages = 0; }
Quoted 4 lines
Quoted 1 line
So if
X86_FEATURE_GBPAGESis not supported,direct_gbpageswill be set to 0.So why is the introduction of the info.direct_gbpages flag necessary? AFAICS it just duplicates the kernel's
direct_gbpagesflag. One outcome is that hibernation won't use gbpages, which is silly.
boot/compressed/pagetable.c also uses kernel_ident_mapping_init() for kaslr, at the moment we don't have "direct_gbpages" definition or X86_FEATURE_GBPAGES feature detection.
I thought that we can change the other call sites when found really needed.
Regards,
Xunlei
Re: [PATCH v3 2/2] x86_64/kexec: Use PUD level 1GB page for identity mapping if available
Quoted 29 lines · Xunlei Pang
Quoted 23 lines · Ingo Molnar
Quoted 21 lines · Xunlei Pang
Quoted 4 lines · Ingo Molnar
Quoted 2 lines · Xunlei Pang
(quoted diff) +0
-0@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)Quoted 8 lines
arch/x86/kernel/machine_kexec_64.c static int init_pgtable(struct kimage *image, unsigned long start_pgtable) · line 127
Quoted 5 lines
arch/x86/kernel/machine_kexec_64.c +3
-0level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true;No, this should be keyed off the CPU feature (
X86_FEATURE_GBPAGES) automatically, not set blindly! AFAICS this patch will crash kexec on any CPU that does not support gbpages.It should be fine,
probe_page_size_mask()already takes care of this:if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) { printk(KERN_INFO "Using GB pages for direct mapping\n"); page_size_mask |= 1 << PG_LEVEL_1G; } else { direct_gbpages = 0; }Quoted 4 lines
Quoted 1 line
So if
X86_FEATURE_GBPAGESis not supported,direct_gbpageswill be set to 0.So why is the introduction of the info.direct_gbpages flag necessary? AFAICS it just duplicates the kernel's
direct_gbpagesflag. One outcome is that hibernation won't use gbpages, which is silly.
Quoted 5 lines
boot/compressed/pagetable.calso useskernel_ident_mapping_init()for kaslr, at the moment we don't have "direct_gbpages" definition orX86_FEATURE_GBPAGESfeature detection.I thought that we can change the other call sites when found really needed.
Ok, you are right - I'll use the original patches as submitted, with the updated changelogs.
Thanks,
Ingo
Re: [PATCH v3 2/2] x86_64/kexec: Use PUD level 1GB page for identity mapping if available
Quoted 30 lines · Ingo Molnar
Quoted 28 lines · Xunlei Pang
Quoted 27 lines · Ingo Molnar
Quoted 14 lines · Xunlei Pang
Quoted 4 lines · Ingo Molnar
Quoted 2 lines · Xunlei Pang
(quoted diff) +0
-0@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)Quoted 8 lines
arch/x86/kernel/machine_kexec_64.c static int init_pgtable(struct kimage *image, unsigned long start_pgtable) · line 127
Quoted 5 lines
arch/x86/kernel/machine_kexec_64.c +3
-0level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true;No, this should be keyed off the CPU feature (
X86_FEATURE_GBPAGES) automatically, not set blindly! AFAICS this patch will crash kexec on any CPU that does not support gbpages.It should be fine,
probe_page_size_mask()already takes care of this:Quoted 8 lines
if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) { printk(KERN_INFO "Using GB pages for direct mapping\n"); page_size_mask |= 1 << PG_LEVEL_1G; } else { direct_gbpages = 0; }So if
X86_FEATURE_GBPAGESis not supported,direct_gbpageswill be set to 0.So why is the introduction of the info.direct_gbpages flag necessary? AFAICS it just duplicates the kernel's
direct_gbpagesflag. One outcome is that hibernation won't use gbpages, which is silly.
Quoted 7 lines
Quoted 5 lines
boot/compressed/pagetable.calso useskernel_ident_mapping_init()for kaslr, at the moment we don't have "direct_gbpages" definition orX86_FEATURE_GBPAGESfeature detection.I thought that we can change the other call sites when found really needed.
Ok, you are right - I'll use the original patches as submitted, with the updated changelogs.
Thanks!
Regards,
Xunlei
tip-bot for Xunlei Pang
merge bot[tip:x86/urgent] x86/kexec/64: Use gbpages for identity mappings if available
In reply to Xunlei Pangx86/kexec/64: Use gbpages for identity mappings if available
Commit-ID: 8638100c52bb7782462b14aad102a4aaf0c7094c Gitweb: http://git.kernel.org/tip/8638100c52bb7782462b14aad102a4aaf0c7094c Author: Xunlei Pang <[email protected]> AuthorDate: Thu, 4 May 2017 09:42:51 +0800 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 8 May 2017 08:28:44 +0200
x86/kexec/64: Use gbpages for identity mappings if available
Kexec sets up all identity mappings before booting into the new kernel, and this will cause extra memory consumption for paging structures which is quite considerable on modern machines with huge memory sizes.
E.g. on a 32TB machine that is kdumping, it could waste around 128MB (around 4MB/TB) from the reserved memory after kexec sets all the identity mappings using the current 2MB page.
Add to that the memory needed for the loaded kdump kernel, initramfs, etc., and it causes a kexec syscall -NOMEM failure.
As a result, we had to enlarge reserved memory via "crashkernel=X" to work around this problem.
This causes some trouble for distributions that use policies to evaluate the proper "crashkernel=X" value for users.
So enable gbpages for kexec mappings.
--- arch/x86/kernel/machine_kexec_64.c | 4 ++++ 1 file changed, 4 insertions(+)
arch/x86/kernel/machine_kexec_64.c +4 -0
@@ -122,6 +122,10 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable) level4p = (pgd_t *)__va (start_pgtable);clear_page (level4p); + +if (direct_gbpages) + info.direct_gbpages = true; +for (i =0 ; i < nr_pfn_mapped; i++) { mstart = pfn_mapped[i].start <<PAGE_SHIFT ; mend = pfn_mapped[i].end <<PAGE_SHIFT ;
tip-bot for Xunlei Pang
merge bot[tip:x86/urgent] x86/mm: Add support for gbpages to kernel_ident_mapping_init()
In reply to Xunlei Pangx86/mm: Add support for gbpages to kernel_ident_mapping_init()
Commit-ID: 66aad4fdf2bf0af29c7decb4433dc5ec6c7c5451 Gitweb: http://git.kernel.org/tip/66aad4fdf2bf0af29c7decb4433dc5ec6c7c5451 Author: Xunlei Pang <[email protected]> AuthorDate: Thu, 4 May 2017 09:42:50 +0800 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 8 May 2017 08:28:40 +0200
x86/mm: Add support for gbpages to kernel_ident_mapping_init()
Kernel identity mappings on x86-64 kernels are created in two
ways: by the early x86 boot code, or by kernel_ident_mapping_init().
Native kernels (which is the dominant usecase) use the former, but the kexec and the hibernation code uses kernel_ident_mapping_init().
There's a subtle difference between these two ways of how identity mappings are created, the current kernel_ident_mapping_init() code creates identity mappings always using 2MB page(PMD level) - while the native kernel boot path also utilizes gbpages where available.
This difference is suboptimal both for performance and for memory
usage: kernel_ident_mapping_init() needs to allocate pages for the page tables when creating the new identity mappings.
This patch adds 1GB page(PUD level) support to kernel_ident_mapping_init() to address these concerns.
The primary advantage would be better TLB coverage/performance, because we'd utilize 1GB TLBs instead of 2MB ones.
It is also useful for machines with large number of memory to save paging structure allocations(around 4MB/TB using 2MB page) when setting identity mappings for all the memory, after using 1GB page it will consume only 8KB/TB.
( Note that this change alone does not activate gbpages in kexec,
we are doing that in a separate patch. )
--- arch/x86/boot/compressed/pagetable.c | 2 +- arch/x86/include/asm/init.h | 3 ++- arch/x86/kernel/machine_kexec_64.c | 2 +- arch/x86/mm/ident_map.c | 14 +++++++++++++- arch/x86/power/hibernate_64.c | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-)
arch/x86/boot/compressed/pagetable.c +1 -1
@@ -70,7 +70,7 @@ static unsigned long level4p;* Due to relocation, pointers must be assigned at run time not build time. */ static struct x86_mapping_info mapping_info = { - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, + .page_flag = __PAGE_KERNEL_LARGE_EXEC, };/* Locates and clears a region for a new top level page table. */
arch/x86/include/asm/init.h +2 -1
@@ -4,8 +4,9 @@struct x86_mapping_info {void *(*alloc_pgt_page)(void *);/* allocate buf for page table */ void *context;/* context for alloc_pgt_page */ -unsigned long pmd_flag;/* page flag for PMD entry */ +unsigned long page_flag;/* page flag for PMD or PUD entry */ unsigned long offset;/* ident mapping offset */ +bool direct_gbpages;/* PUD level 1GB page support */ }; int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
arch/x86/kernel/machine_kexec_64.c +1 -1
@@ -113,7 +113,7 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable)struct x86_mapping_info info = { .alloc_pgt_page = alloc_pgt_page, .context = image, - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, + .page_flag = __PAGE_KERNEL_LARGE_EXEC, };unsigned long mstart, mend;pgd_t *level4p;
arch/x86/mm/ident_map.c +13 -1
@@ -13,7 +13,7 @@ static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,if (pmd_present (*pmd))continue ; -set_pmd (pmd,__pmd ((addr - info->offset) | info->pmd_flag)); +set_pmd (pmd,__pmd ((addr - info->offset) | info->page_flag)); } }
@@ -30,6 +30,18 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,if (next > end) next = end; +if (info->direct_gbpages) { +pud_t pudval; + +if (pud_present (*pud)) +continue ; + + addr &=PUD_MASK ; + pudval =__pud ((addr - info->offset) | info->page_flag); +set_pud (pud, pudval); +continue ; + } +if (pud_present (*pud)) { pmd =pmd_offset (pud,0 );ident_pmd_init (info, pmd, addr, next);
arch/x86/power/hibernate_64.c +1 -1
@@ -104,7 +104,7 @@ static int set_up_temporary_mappings(void) {struct x86_mapping_info info = { .alloc_pgt_page = alloc_pgt_page, - .pmd_flag = __PAGE_KERNEL_LARGE_EXEC, + .page_flag = __PAGE_KERNEL_LARGE_EXEC, .offset = __PAGE_OFFSET, };unsigned long mstart, mend;