x86_64: preserve large page text mappings with CONFIG_DEBUG_RODATA
cover x86_64: preserve large page text mappings with CONFIG_DEBUG_RODATA
Most of the distributions enable CONFIG_DEBUG_RODATA by default, with the assumption that the debug protection checks come with no cost. Our OLTP tests showed 1% improvement (with 2% reduction in system time and 1% increase in iowait idle time) by disabling CONFIG_DEBUG_RODATA on a two socket NHM-EP system.
CONFIG_DEBUG_RODATA breaks some of the kernel text/data large page mappings to small pages, thus causing this regression. These two patches address this issue.
Experiments with EL5 kernel and mainline show that the first patch in the series (preserve large page mapping for 1st 2MB kernel txt with CONFIG_DEBUG_RODATA) should recover most of the above mentioned regression. We are doing couple of more OLTP tests for the confirmation.
thanks,
suresh
PATCH 1/2 x86_64: preserve large page mapping for 1st 2MB kernel txt with CONFIG_DEBUG_RODATA
In the first 2MB, kernel text is co-located with kernel static page tables setup by head_64.S. CONFIG_DEBUG_RODATA chops this 2MB large page mapping to small 4KB pages as we mark the kernel text as RO, leaving the static page tables as RW.
With CONFIG_DEBUG_RODATA disabled, OLTP run on NHM-EP shows 1% improvement with 2% reduction in system time and 1% improvement in iowait idle time.
To recover this, move the kernel static page tables to .data section, so that we don't have to break the first 2MB of kernel text to small pages with CONFIG_DEBUG_RODATA.
--- arch/x86/kernel/head_64.S | 3 ++- arch/x86/mm/init_64.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-)
arch/x86/kernel/head_64.S +2 -1
@@ -262,11 +262,11 @@ ENTRY(secondary_startup_64) .quad x86_64_start_kernel ENTRY(initial_gs) .quad INIT_PER_CPU_VAR(irq_stack_union) - __FINITDATA ENTRY(stack_start) .quad init_thread_union+THREAD_SIZE-8 .word 0 + __FINITDATA bad_address: jmp bad_address
@@ -340,6 +340,7 @@ ENTRY(name) i = i + 1 ; \ .endr + .data /* * This default setting generates an ident mapping at address 0x100000 * and a mapping for the kernel that precisely maps virtual address
arch/x86/mm/init_64.c +3 -3
@@ -698,7 +698,7 @@ static int kernel_set_to_readonly;void set_kernel_text_rw (void ) { -unsigned long start =PFN_ALIGN (_stext); +unsigned long start =PFN_ALIGN (_text);unsigned long end =PFN_ALIGN (__start_rodata);if (!kernel_set_to_readonly)
@@ -712,7 +712,7 @@ void set_kernel_text_rw(void)void set_kernel_text_ro (void ) { -unsigned long start =PFN_ALIGN (_stext); +unsigned long start =PFN_ALIGN (_text);unsigned long end =PFN_ALIGN (__start_rodata);if (!kernel_set_to_readonly)
@@ -726,7 +726,7 @@ void set_kernel_text_ro(void)void mark_rodata_ro (void ) { -unsigned long start =PFN_ALIGN (_stext), end =PFN_ALIGN (__end_rodata); +unsigned long start =PFN_ALIGN (_text), end =PFN_ALIGN (__end_rodata);unsigned long rodata_start = ((unsigned long )__start_rodata +PAGE_SIZE -1 ) &PAGE_MASK ;
[tip:branch?] x86-64: preserve large page mapping for 1st 2MB kernel txt with CONFIG_DEBUG_RODATA
Commit-ID: b9af7c0d44b8bb71e3af5e94688d076414aa8c87 Gitweb: http://git.kernel.org/tip/b9af7c0d44b8bb71e3af5e94688d076414aa8c87 Author: Suresh Siddha <[email protected]> AuthorDate: Wed, 14 Oct 2009 14:46:55 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Tue, 20 Oct 2009 14:46:00 +0900
x86-64: preserve large page mapping for 1st 2MB kernel txt with CONFIG_DEBUG_RODATA
In the first 2MB, kernel text is co-located with kernel static page tables setup by head_64.S. CONFIG_DEBUG_RODATA chops this 2MB large page mapping to small 4KB pages as we mark the kernel text as RO, leaving the static page tables as RW.
With CONFIG_DEBUG_RODATA disabled, OLTP run on NHM-EP shows 1% improvement with 2% reduction in system time and 1% improvement in iowait idle time.
To recover this, move the kernel static page tables to .data section, so that we don't have to break the first 2MB of kernel text to small pages with CONFIG_DEBUG_RODATA.
LKML-Reference: <[email protected]>
--- arch/x86/kernel/head_64.S | 3 ++- arch/x86/mm/init_64.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-)
arch/x86/kernel/head_64.S +2 -1
@@ -262,11 +262,11 @@ ENTRY(secondary_startup_64) .quad x86_64_start_kernel ENTRY(initial_gs) .quad INIT_PER_CPU_VAR(irq_stack_union) - __FINITDATA ENTRY(stack_start) .quad init_thread_union+THREAD_SIZE-8 .word 0 + __FINITDATA bad_address: jmp bad_address
@@ -340,6 +340,7 @@ ENTRY(name) i = i + 1 ; \ .endr + .data /* * This default setting generates an ident mapping at address 0x100000 * and a mapping for the kernel that precisely maps virtual address
arch/x86/mm/init_64.c +3 -3
@@ -699,7 +699,7 @@ static int kernel_set_to_readonly;void set_kernel_text_rw (void ) { -unsigned long start =PFN_ALIGN (_stext); +unsigned long start =PFN_ALIGN (_text);unsigned long end =PFN_ALIGN (__start_rodata);if (!kernel_set_to_readonly)
@@ -713,7 +713,7 @@ void set_kernel_text_rw(void)void set_kernel_text_ro (void ) { -unsigned long start =PFN_ALIGN (_stext); +unsigned long start =PFN_ALIGN (_text);unsigned long end =PFN_ALIGN (__start_rodata);if (!kernel_set_to_readonly)
@@ -727,7 +727,7 @@ void set_kernel_text_ro(void)void mark_rodata_ro (void ) { -unsigned long start =PFN_ALIGN (_stext), end =PFN_ALIGN (__end_rodata); +unsigned long start =PFN_ALIGN (_text), end =PFN_ALIGN (__end_rodata);unsigned long rodata_start = ((unsigned long )__start_rodata +PAGE_SIZE -1 ) &PAGE_MASK ;
PATCH 2/2 x86_64: align RODATA kernel section to 2MB with CONFIG_DEBUG_RODATA
In reply to Suresh SiddhaCONFIG_DEBUG_RODATA chops the large pages spanning boundaries of kernel text/rodata/data to small 4KB pages as they are mapped with different attributes (text as RO, RODATA as RO and NX etc).
On x86_64, preserve the large page mappings for kernel text/rodata/data boundaries when CONFIG_DEBUG_RODATA is enabled. This is done by allowing the RODATA section to be hugepage aligned and having same RWX attributes for the 2MB page boundaries
Extra Memory pages padding the sections will be freed during the end of the boot and the kernel identity mappings will have different RWX permissions compared to the kernel text mappings.
Kernel identity mappings to these physical pages will be mapped with smaller pages but large page mappings are still retained for kernel text,rodata,data mappings.
--- arch/x86/include/asm/sections.h | 6 ++++++ arch/x86/kernel/vmlinux.lds.S | 17 +++++++++++++++++ arch/x86/mm/init_64.c | 14 +++++++++++++- arch/x86/mm/pageattr.c | 14 ++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-)
arch/x86/include/asm/sections.h +6 -0
@@ -2,7 +2,13 @@#define _ASM_X86_SECTIONS_H#include <asm-generic/sections.h> +#include <asm/uaccess.h> extern char __brk_base[], __brk_limit[]; +extern struct exception_table_entry __stop___ex_table[]; + +#if defined(CONFIG_X86_64 ) && defined(CONFIG_DEBUG_RODATA ) +extern char __end_rodata_hpage_align[]; +#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/vmlinux.lds.S +17 -0
@@ -41,6 +41,21 @@ ENTRY(phys_startup_64) jiffies_64 = jiffies; #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA) + +#define X64_ALIGN_DEBUG_RODATA_BEGIN . = ALIGN(HPAGE_SIZE); + +#define X64_ALIGN_DEBUG_RODATA_END \ + . = ALIGN(HPAGE_SIZE); \ + __end_rodata_hpage_align = .; + +#else + +#define X64_ALIGN_DEBUG_RODATA_BEGIN +#define X64_ALIGN_DEBUG_RODATA_END + +#endif + PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(7); /* RWE */
@@ -90,7 +105,9 @@ SECTIONS EXCEPTION_TABLE(16) :text = 0x9090 + X64_ALIGN_DEBUG_RODATA_BEGIN RO_DATA(PAGE_SIZE) + X64_ALIGN_DEBUG_RODATA_END /* Data */ .data : AT(ADDR(.data) - LOAD_OFFSET) {
arch/x86/mm/init_64.c +13 -1
@@ -726,9 +726,13 @@ void set_kernel_text_ro(void)void mark_rodata_ro (void ) { -unsigned long start =PFN_ALIGN (_text), end =PFN_ALIGN (__end_rodata); +unsigned long start =PFN_ALIGN (_text);unsigned long rodata_start = ((unsigned long )__start_rodata +PAGE_SIZE -1 ) &PAGE_MASK ; +unsigned long end = (unsigned long ) &__end_rodata_hpage_align; +unsigned long text_end =PAGE_ALIGN ((unsigned long ) &__stop___ex_table); +unsigned long rodata_end =PAGE_ALIGN ((unsigned long ) &__end_rodata); +unsigned long data_start = (unsigned long ) &_sdata;printk (KERN_INFO "Write protecting the kernel read-only data: %luk\n" , (end - start) >>10 );
@@ -751,6 +755,14 @@ void mark_rodata_ro(void)printk (KERN_INFO "Testing CPA: again\n" );set_memory_ro (start, (end-start) >>PAGE_SHIFT );#endif + +free_init_pages ("unused kernel memory" , + (unsigned long )page_address (virt_to_page (text_end)), + (unsigned long ) +page_address (virt_to_page (rodata_start))); +free_init_pages ("unused kernel memory" , + (unsigned long )page_address (virt_to_page (rodata_end)), + (unsigned long) page_address(virt_to_page(data_start))); }#endif
arch/x86/mm/pageattr.c +14 -0
@@ -279,6 +279,20 @@ static inline pgprot_t static_protection __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))pgprot_val (forbidden) |= _PAGE_RW; +#if defined(CONFIG_X86_64 ) && defined(CONFIG_DEBUG_RODATA ) +/* +* Kernel text mappings for the large page aligned .rodata section +* will be read-only. For the kernel identity mappings covering +* the holes caused by this alignment can be anything. +* +* This will preserve the large page mappings for kernel text/data +* at no extra cost. +*/ +if (within (address, (unsigned long )_text, + (unsigned long )__end_rodata_hpage_align)) +pgprot_val (forbidden) |= _PAGE_RW; +#endif + prot =__pgprot (pgprot_val (prot) & ~pgprot_val (forbidden));return prot;
Re: [patch 2/2] x86_64: align RODATA kernel section to 2MB with CONFIG_DEBUG_RODATA
Quoted 23 lines
Index:
tree/arch/x86/include/asm/sections.h
===================================================================arch/x86/include/asm/sections.h +6
-0@@ -2,7 +2,13 @@#define _ASM_X86_SECTIONS_H#include <asm-generic/sections.h> +#include <asm/uaccess.h> extern char __brk_base[], __brk_limit[]; +extern struct exception_table_entry __stop___ex_table[]; + +#if defined(CONFIG_X86_64 ) && defined(CONFIG_DEBUG_RODATA ) +extern char __end_rodata_hpage_align[]; +#endif #endif /* _ASM_X86_SECTIONS_H */ Index:
tree/arch/x86/kernel/vmlinux.lds.S
===================================================================
arch/x86/kernel/vmlinux.lds.S ENTRY(phys_startup_64) · line 50
Quoted 10 lines
arch/x86/kernel/vmlinux.lds.S +7
-0@@ -41,6 +41,21 @@ ENTRY(phys_startup_64) jiffies_64 = jiffies; #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA) + +#define X64_ALIGN_DEBUG_RODATA_BEGIN . = ALIGN(HPAGE_SIZE); + +#define X64_ALIGN_DEBUG_RODATA_END \ + . = ALIGN(HPAGE_SIZE); \ + __end_rodata_hpage_align = .;
This symbol is only defined if CONFIG_DEBUG_RODATA is defined, but (see below).
arch/x86/kernel/vmlinux.lds.S ENTRY(phys_startup_64) · line 57
Quoted 7 lines
arch/x86/kernel/vmlinux.lds.S +7
-0+ +#else + +#define X64_ALIGN_DEBUG_RODATA_BEGIN +#define X64_ALIGN_DEBUG_RODATA_END + +#endif
A coment describing the purpose of
this would be nice.
Quoted 19 lines
arch/x86/kernel/vmlinux.lds.S +3
-0+ PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(7); /* RWE */@@ -90,7 +105,9 @@ SECTIONS EXCEPTION_TABLE(16) :text = 0x9090 + X64_ALIGN_DEBUG_RODATA_BEGIN RO_DATA(PAGE_SIZE) + X64_ALIGN_DEBUG_RODATA_END /* Data */ .data : AT(ADDR(.data) - LOAD_OFFSET) {Index:
tree/arch/x86/mm/init_64.c
===================================================================
arch/x86/mm/init_64.c void set_kernel_text_ro(void) · line 732
Quoted 8 lines
arch/x86/mm/init_64.c +2
-1@@ -726,9 +726,13 @@ void set_kernel_text_ro(void)void mark_rodata_ro (void ) { -unsigned long start =PFN_ALIGN (_text), end =PFN_ALIGN (__end_rodata); +unsigned long start =PFN_ALIGN (_text);unsigned long rodata_start = ((unsigned long )__start_rodata +PAGE_SIZE -1 ) &PAGE_MASK ; +unsigned long end = (unsigned long ) &__end_rodata_hpage_align;
Here the symbol seems to be referenced unconditionally.
Sam
Re: [patch 2/2] x86_64: align RODATA kernel section to 2MB with CONFIG_DEBUG_RODATA
Quoted 9 lines · Sam Ravnborg
arch/x86/kernel/vmlinux.lds.S ENTRY(phys_startup_64) · line 57
Quoted 6 lines
arch/x86/kernel/vmlinux.lds.S +6
-0+#else + +#define X64_ALIGN_DEBUG_RODATA_BEGIN +#define X64_ALIGN_DEBUG_RODATA_END + +#endifA coment describing the purpose of
this would be nice.
I will update this in the next version of the patch.
Quoted 10 lines
arch/x86/mm/init_64.c void set_kernel_text_ro(void) · line 732
Quoted 9 lines
arch/x86/mm/init_64.c +2
-1@@ -726,9 +726,13 @@ void set_kernel_text_ro(void)void mark_rodata_ro (void ) { -unsigned long start =PFN_ALIGN (_text), end =PFN_ALIGN (__end_rodata); +unsigned long start =PFN_ALIGN (_text);unsigned long rodata_start = ((unsigned long )__start_rodata +PAGE_SIZE -1 ) &PAGE_MASK ; +unsigned long end = (unsigned long ) &__end_rodata_hpage_align;Here the symbol seems to be referenced unconditionally.
This function is defined only for CONFIG_DEBUG_RODATA.
thanks,
suresh
[tip:branch?] x86-64: align RODATA kernel section to 2MB with CONFIG_DEBUG_RODATA
In reply to Suresh SiddhaCommit-ID: 74e081797bd9d2a7d8005fe519e719df343a2ba8 Gitweb: http://git.kernel.org/tip/74e081797bd9d2a7d8005fe519e719df343a2ba8 Author: Suresh Siddha <[email protected]> AuthorDate: Wed, 14 Oct 2009 14:46:56 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Tue, 20 Oct 2009 14:46:00 +0900
x86-64: align RODATA kernel section to 2MB with CONFIG_DEBUG_RODATA
CONFIG_DEBUG_RODATA chops the large pages spanning boundaries of kernel text/rodata/data to small 4KB pages as they are mapped with different attributes (text as RO, RODATA as RO and NX etc).
On x86_64, preserve the large page mappings for kernel text/rodata/data boundaries when CONFIG_DEBUG_RODATA is enabled. This is done by allowing the RODATA section to be hugepage aligned and having same RWX attributes for the 2MB page boundaries
Extra Memory pages padding the sections will be freed during the end of the boot and the kernel identity mappings will have different RWX permissions compared to the kernel text mappings.
Kernel identity mappings to these physical pages will be mapped with smaller pages but large page mappings are still retained for kernel text,rodata,data mappings.
LKML-Reference: <[email protected]>
--- arch/x86/include/asm/sections.h | 6 ++++++ arch/x86/kernel/vmlinux.lds.S | 17 +++++++++++++++++ arch/x86/mm/init_64.c | 14 +++++++++++++- arch/x86/mm/pageattr.c | 14 ++++++++++++++ 4 files changed, 50 insertions(+), 1 deletions(-)
arch/x86/include/asm/sections.h +6 -0
@@ -2,7 +2,13 @@#define _ASM_X86_SECTIONS_H#include <asm-generic/sections.h> +#include <asm/uaccess.h> extern char __brk_base[], __brk_limit[]; +extern struct exception_table_entry __stop___ex_table[]; + +#if defined(CONFIG_X86_64 ) && defined(CONFIG_DEBUG_RODATA ) +extern char __end_rodata_hpage_align[]; +#endif #endif /* _ASM_X86_SECTIONS_H */
arch/x86/kernel/vmlinux.lds.S +17 -0
@@ -41,6 +41,21 @@ ENTRY(phys_startup_64) jiffies_64 = jiffies; #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA) + +#define X64_ALIGN_DEBUG_RODATA_BEGIN . = ALIGN(HPAGE_SIZE); + +#define X64_ALIGN_DEBUG_RODATA_END \ + . = ALIGN(HPAGE_SIZE); \ + __end_rodata_hpage_align = .; + +#else + +#define X64_ALIGN_DEBUG_RODATA_BEGIN +#define X64_ALIGN_DEBUG_RODATA_END + +#endif + PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(7); /* RWE */
@@ -90,7 +105,9 @@ SECTIONS EXCEPTION_TABLE(16) :text = 0x9090 + X64_ALIGN_DEBUG_RODATA_BEGIN RO_DATA(PAGE_SIZE) + X64_ALIGN_DEBUG_RODATA_END /* Data */ .data : AT(ADDR(.data) - LOAD_OFFSET) {
arch/x86/mm/init_64.c +13 -1
@@ -727,9 +727,13 @@ void set_kernel_text_ro(void)void mark_rodata_ro (void ) { -unsigned long start =PFN_ALIGN (_text), end =PFN_ALIGN (__end_rodata); +unsigned long start =PFN_ALIGN (_text);unsigned long rodata_start = ((unsigned long )__start_rodata +PAGE_SIZE -1 ) &PAGE_MASK ; +unsigned long end = (unsigned long ) &__end_rodata_hpage_align; +unsigned long text_end =PAGE_ALIGN ((unsigned long ) &__stop___ex_table); +unsigned long rodata_end =PAGE_ALIGN ((unsigned long ) &__end_rodata); +unsigned long data_start = (unsigned long ) &_sdata;printk (KERN_INFO "Write protecting the kernel read-only data: %luk\n" , (end - start) >>10 );
@@ -752,6 +756,14 @@ void mark_rodata_ro(void)printk (KERN_INFO "Testing CPA: again\n" );set_memory_ro (start, (end-start) >>PAGE_SHIFT );#endif + +free_init_pages ("unused kernel memory" , + (unsigned long )page_address (virt_to_page (text_end)), + (unsigned long ) +page_address (virt_to_page (rodata_start))); +free_init_pages ("unused kernel memory" , + (unsigned long )page_address (virt_to_page (rodata_end)), + (unsigned long) page_address(virt_to_page(data_start))); }#endif
arch/x86/mm/pageattr.c +14 -0
@@ -279,6 +279,20 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address, __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))pgprot_val (forbidden) |= _PAGE_RW; +#if defined(CONFIG_X86_64 ) && defined(CONFIG_DEBUG_RODATA ) +/* +* Kernel text mappings for the large page aligned .rodata section +* will be read-only. For the kernel identity mappings covering +* the holes caused by this alignment can be anything. +* +* This will preserve the large page mappings for kernel text/data +* at no extra cost. +*/ +if (within (address, (unsigned long )_text, + (unsigned long )__end_rodata_hpage_align)) +pgprot_val (forbidden) |= _PAGE_RW; +#endif + prot =__pgprot (pgprot_val (prot) & ~pgprot_val (forbidden));return prot;