Use global pages with PTI
Dave Hansen
maintainercover Use global pages with PTI
The later verions of the KAISER pathces (pre-PTI) allowed the user/kernel shared areas to be GLOBAL. The thought was that this would reduce the TLB overhead of keeping two copies of these mappings.
During the switch over to PTI, we seem to have lost our ability to have GLOBAL mappings. This adds them back.
Did we have some reason for leaving out global pages entirely? We _should_ have all the TLB flushing mechanics in place to handle these already since all of our kernel mapping changes have to know how to flush global pages regardless.
Dave Hansen
maintainerPATCH 1/3 x86/mm: factor out conditional pageattr PTE bit setting code
From: Dave Hansen <[email protected]>
The pageattr code has a pattern repeated where it sets a PTE bit for present PTEs but clears it for non-present PTEs. This helps to keep pte_none() from getting messed up. _PAGE_GLOBAL is the most frequent target of this pattern.
This pattern also includes a nice, copy-and-pasted comment.
I want to do some special stuff with _PAGE_GLOBAL in a moment, so refactor this a _bit_ to centralize the comment and the bit operations.
---
b/arch/x86/mm/pageattr.c | 65 ++++++++++++++--------------------------------- 1 file changed, 20 insertions(+), 45 deletions(-)
arch/x86/mm/pageattr.c renamed from arch/x86/mm/pageattr.c~kpti-centralize-global-setting +20 -45
@@ -512,6 +512,22 @@ static void __set_pmd_pte(pte_t *kpte, u#endif } +static pgprot_t pgprot_set_on_present (pgprot_t prot,pteval_t flags) +{ +/* +* Set 'flags' only if PRESENT. Ensures that we do not +* set flags in an otherwise empty PTE breaking pte_none(). +* A later function (such as canon_pgprot()) must clear +* possibly unsupported flags (like _PAGE_GLOBAL). +*/ +if (pgprot_val (prot) & _PAGE_PRESENT) +pgprot_val (prot) |= flags; +else +pgprot_val (prot) &= ~flags; + +return prot; +} +static int try_preserve_large_page (pte_t *kpte,unsigned long address,struct cpa_data *cpa)
@@ -577,18 +593,7 @@ try_preserve_large_page(pte_t *kpte, uns* different bit positions in the two formats. */ req_prot =pgprot_4k_2_large (req_prot); - -/* -* Set the PSE and GLOBAL flags only if the PRESENT flag is -* set otherwise pmd_present/pmd_huge will return true even on -* a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL -* for the ancient hardware that doesn't support it. -*/ -if (pgprot_val (req_prot) & _PAGE_PRESENT) -pgprot_val (req_prot) |= _PAGE_PSE | _PAGE_GLOBAL; -else -pgprot_val (req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL); - + req_prot =pgprot_set_on_present (req_prot, _PAGE_GLOBAL | _PAGE_PSE); req_prot =canon_pgprot (req_prot);/*
@@ -698,16 +703,7 @@ __split_large_page(struct cpa_data *cpa,return 1 ; } -/* -* Set the GLOBAL flags only if the PRESENT flag is set -* otherwise pmd/pte_present will return true even on a non -* present pmd/pte. The canon_pgprot will clear _PAGE_GLOBAL -* for the ancient hardware that doesn't support it. -*/ -if (pgprot_val (ref_prot) & _PAGE_PRESENT) -pgprot_val (ref_prot) |= _PAGE_GLOBAL; -else -pgprot_val (ref_prot) &= ~_PAGE_GLOBAL; + ref_prot =pgprot_set_on_present (ref_prot, _PAGE_GLOBAL);/* * Get the target pfn from the original entry:
@@ -930,18 +926,7 @@ static void populate_pte(struct cpa_data pte =pte_offset_kernel (pmd, start); -/* -* Set the GLOBAL flags only if the PRESENT flag is -* set otherwise pte_present will return true even on -* a non present pte. The canon_pgprot will clear -* _PAGE_GLOBAL for the ancient hardware that doesn't -* support it. -*/ -if (pgprot_val (pgprot) & _PAGE_PRESENT) -pgprot_val (pgprot) |= _PAGE_GLOBAL; -else -pgprot_val (pgprot) &= ~_PAGE_GLOBAL; - + pgprot =pgprot_set_on_present (pgprot, _PAGE_GLOBAL); pgprot =canon_pgprot (pgprot);while (num_pages-- && start < end) {
@@ -1234,17 +1219,7 @@ repeat: new_prot =static_protections (new_prot, address, pfn); -/* -* Set the GLOBAL flags only if the PRESENT flag is -* set otherwise pte_present will return true even on -* a non present pte. The canon_pgprot will clear -* _PAGE_GLOBAL for the ancient hardware that doesn't -* support it. -*/ -if (pgprot_val (new_prot) & _PAGE_PRESENT) -pgprot_val (new_prot) |= _PAGE_GLOBAL; -else -pgprot_val (new_prot) &= ~_PAGE_GLOBAL; + new_prot =pgprot_set_on_present (new_prot, _PAGE_GLOBAL);/* * We need to keep the pfn from the existing PTE,
_
Dave Hansen
maintainerPATCH 2/3 x86/mm: introduce __PAGE_KERNEL_GLOBAL
In reply to Dave HansenFrom: Dave Hansen <[email protected]>
Kernel mappings are historically _PAGE_GLOBAL. But, with PTI, we do not want them to be _PAGE_GLOBAL. We currently accomplish this by simply clearing _PAGE_GLOBAL from the suppotred mask which ensures it is cleansed from many of our PTE construction sites:
if (!static_cpu_has(X86_FEATURE_PTI)) __supported_pte_mask |= _PAGE_GLOBAL;
But, this also means that we now get *no* opportunity to use global pages with PTI, even for data which is shared such as the cpu_entry_area and entry/exit text.
This patch introduces a new mask: __PAGE_KERNEL_GLOBAL. This mask can be thought of as the default global bit value when creating kernel mappings. We make it _PAGE_GLOBAL when PTI=n, but 0 when PTI=y. This ensures that on PTI kernels, all of the __PAGE_KERNEL_* users will not get _PAGE_GLOBAL.
This also restores _PAGE_GLOBAL to __supported_pte_mask, allowing it to be set in the first place.
---
b/arch/x86/include/asm/pgtable_types.h | 9 ++++++++- b/arch/x86/mm/init.c | 8 +------- b/arch/x86/mm/pageattr.c | 9 +++++---- 3 files changed, 14 insertions(+), 12 deletions(-)
arch/x86/include/asm/pgtable_types.h renamed from arch/x86/include/asm/pgtable_types.h~kpti-no-global-for-kernel-mappings +8 -1
@@ -180,8 +180,15 @@ enum page_cache_mode {#define PAGE_READONLY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ _PAGE_ACCESSED) +#ifdef CONFIG_PAGE_TABLE_ISOLATION +#define __PAGE_KERNEL_GLOBAL 0 +#else +#define __PAGE_KERNEL_GLOBAL _PAGE_GLOBAL +#endif +#define __PAGE_KERNEL_EXEC \ - (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_GLOBAL) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED | \ + __PAGE_KERNEL_GLOBAL)#define __PAGE_KERNEL (__PAGE_KERNEL_EXEC | _PAGE_NX)#define __PAGE_KERNEL_RO (__PAGE_KERNEL & ~_PAGE_RW)
arch/x86/mm/init.c renamed from arch/x86/mm/init.c~kpti-no-global-for-kernel-mappings +1 -7
@@ -162,12 +162,6 @@ struct map_range {static int page_size_mask; -static void enable_global_pages (void ) -{ -if (!static_cpu_has (X86_FEATURE_PTI )) - __supported_pte_mask |= _PAGE_GLOBAL; -} -static void __init probe_page_size_mask (void ) {/*
@@ -189,7 +183,7 @@ static void __init probe_page_size_mask( __supported_pte_mask &= ~_PAGE_GLOBAL;if (boot_cpu_has (X86_FEATURE_PGE )) {cr4_set_bits_and_update_boot (X86_CR4_PGE ); -enable_global_pages (); + __supported_pte_mask |= _PAGE_GLOBAL; }/* Enable 1 GB linear kernel mappings if available: */
arch/x86/mm/pageattr.c renamed from arch/x86/mm/pageattr.c~kpti-no-global-for-kernel-mappings +5 -4
@@ -593,7 +593,8 @@ try_preserve_large_page(pte_t *kpte, uns* different bit positions in the two formats. */ req_prot =pgprot_4k_2_large (req_prot); - req_prot =pgprot_set_on_present (req_prot, _PAGE_GLOBAL | _PAGE_PSE); + req_prot =pgprot_set_on_present (req_prot, + __PAGE_KERNEL_GLOBAL | _PAGE_PSE); req_prot =canon_pgprot (req_prot);/*
@@ -703,7 +704,7 @@ __split_large_page(struct cpa_data *cpa,return 1 ; } - ref_prot =pgprot_set_on_present (ref_prot, _PAGE_GLOBAL); + ref_prot =pgprot_set_on_present (ref_prot, __PAGE_KERNEL_GLOBAL);/* * Get the target pfn from the original entry:
@@ -926,7 +927,7 @@ static void populate_pte(struct cpa_data pte =pte_offset_kernel (pmd, start); - pgprot =pgprot_set_on_present (pgprot, _PAGE_GLOBAL); + pgprot =pgprot_set_on_present (pgprot, __PAGE_KERNEL_GLOBAL); pgprot =canon_pgprot (pgprot);while (num_pages-- && start < end) {
@@ -1219,7 +1220,7 @@ repeat: new_prot =static_protections (new_prot, address, pfn); - new_prot =pgprot_set_on_present (new_prot, _PAGE_GLOBAL); + new_prot =pgprot_set_on_present (new_prot, __PAGE_KERNEL_GLOBAL);/* * We need to keep the pfn from the existing PTE,
_
Dave Hansen
maintainerPATCH 3/3 x86/pti: enable global pages for shared areas
In reply to Dave HansenFrom: Dave Hansen <[email protected]>
The entry/exit text and cpu_entry_area are mapped into userspace and the kernel. But, they are not _PAGE_GLOBAL. This creates unnecessary TLB misses.
Add the _PAGE_GLOBAL flag for these areas.
After this patch, we can see that "GLB" shows up in each copy of the page tables, that we have the same number of global entries in each and that they are the *same* entries.
# grep -c GLB /sys/kernel/debug/page_tables/* /sys/kernel/debug/page_tables/current_kernel:11 /sys/kernel/debug/page_tables/current_user:11 /sys/kernel/debug/page_tables/kernel:11
# for f in `ls /sys/kernel/debug/page_tables/`; do grep GLB /sys/kernel/debug/page_tables/$f > $f.GLB; done # md5sum *.GLB 9caae8ad6a1fb53aca2407ec037f612d current_kernel.GLB 9caae8ad6a1fb53aca2407ec037f612d current_user.GLB 9caae8ad6a1fb53aca2407ec037f612d kernel.GLB
A quick visual audit also shows that all the entries make sense. 0xfffffe0000000000 is the cpu_entry_area and 0xffffffff81c00000 is the entry/exit text:
# grep -c GLB /sys/kernel/debug/page_tables/current_user 0xfffffe0000000000-0xfffffe0000002000 8K ro GLB NX pte 0xfffffe0000002000-0xfffffe0000003000 4K RW GLB NX pte 0xfffffe0000003000-0xfffffe0000006000 12K ro GLB NX pte 0xfffffe0000006000-0xfffffe0000007000 4K ro GLB x pte 0xfffffe0000007000-0xfffffe000000d000 24K RW GLB NX pte 0xfffffe000002d000-0xfffffe000002e000 4K ro GLB NX pte 0xfffffe000002e000-0xfffffe000002f000 4K RW GLB NX pte 0xfffffe000002f000-0xfffffe0000032000 12K ro GLB NX pte 0xfffffe0000032000-0xfffffe0000033000 4K ro GLB x pte 0xfffffe0000033000-0xfffffe0000039000 24K RW GLB NX pte 0xffffffff81c00000-0xffffffff81e00000 2M ro PSE GLB x pmd
---
b/arch/x86/mm/cpu_entry_area.c | 7 +++++++ b/arch/x86/mm/pti.c | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-)
arch/x86/mm/cpu_entry_area.c renamed from arch/x86/mm/cpu_entry_area.c~kpti-why-no-global +7 -0
@@ -28,6 +28,13 @@ void cea_set_pte(void *cea_vaddr, phys_a {unsigned long va = (unsigned long ) cea_vaddr; +/* +* The cpu_entry_area is shared between the user and kernel +* page tables. All of its ptes can safely be global. +*/ +if (boot_cpu_has (X86_FEATURE_PGE )) +pgprot_val (flags) |= _PAGE_GLOBAL; +set_pte_vaddr (va,pfn_pte (pa >>PAGE_SHIFT , flags)); }
arch/x86/mm/pti.c renamed from arch/x86/mm/pti.c~kpti-why-no-global +8 -1
@@ -300,6 +300,13 @@ pti_clone_pmds(unsigned long start, unsireturn ;/* +* Setting 'target_pmd' below creates a mapping in both +* the user and kernel page tables. It is effectively +* global, so set it as global in both copies. +*/ + *pmd =pmd_set_flags (*pmd, _PAGE_GLOBAL); + +/* * Copy the PMD. That is, the kernelmode and usermode * tables will share the last-level page tables of this * address range
@@ -348,7 +355,7 @@ static void __init pti_clone_entry_text( {pti_clone_pmds ((unsigned long ) __entry_text_start, (unsigned long ) __irqentry_text_end, - _PAGE_RW | _PAGE_GLOBAL); + _PAGE_RW); }/*
_