Add pagetable allocation notifiers
PATCH 1/5 Add pagetable allocation notifiers
Hooks are provided for the mach-XXX subarchitecture at the time prior to a page being used as a page table at all levels, for PAE and non-PAE kernels. Note that in PAE mode, multiple PDP roots may exist on the same page with other data, so the root must be shadowed instead. This is not a performance issue, since PAE only uses 4 top level PDPEs.
The hooks are:
SetPagePTE(ppn) - indicates that a given physical page is going to be used as a page table.
SetPagePDE(ppn) - indicates that a given physical page is going to be used as a page directory.
ClearPageXXX(ppn) - indicates that the physical page is now done being used as a certain type of page.
These hooks can be used in two ways; for shadow mode, they serve as requests to pre-allocate and deallocate shadow page tables, and for direct page table mode, they serve as write protect/unprotect requests.
Patch-subject: Add pagetable allocation notifiers
Index: linux-2.6.13/arch/i386/mm/init.c
===================================================================
arch/i386/mm/init.c +1 -0
@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini {if (pmd_none (*pmd)) {pte_t *page_table = (pte_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +SetPagePTE (virt_to_page (page_table));set_pmd (pmd,__pmd (__pa (page_table) | _PAGE_TABLE));if (page_table !=pte_offset_kernel (pmd,0 ))BUG ();
arch/i386/mm/pageattr.c +3 -1
@@ -52,8 +52,9 @@ static struct page *split_large_page(uns address =__pa (address); addr = address &LARGE_PAGE_MASK ; pbase = (pte_t *)page_address (base); +SetPagePTE (virt_to_page (pbase));for (i =0 ; i <PTRS_PER_PTE ; i++, addr +=PAGE_SIZE ) { -set_pte (&pbase[i],pfn_pte (addr >>PAGE_SHIFT , +set_pte (&pbase[i],pfn_pte (addr >>PAGE_SHIFT , addr == address ? prot :PAGE_KERNEL )); }return base;
@@ -146,6 +147,7 @@ __change_page_attr(struct page *page, pgBUG_ON (!page_count (kpte_page));if (cpu_has_pse && (page_count (kpte_page) ==1 )) { +ClearPagePTE (virt_to_page (kpte));list_add (&kpte_page->lru, &df_list);revert_page (kpte_page, address); }
arch/i386/mm/pgtable.c +10 -3
@@ -209,6 +209,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *cif (PTRS_PER_PMD ==1 ) {memset (pgd,0 ,USER_PTRS_PER_PGD *sizeof (pgd_t)); +SetPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags); }
@@ -227,6 +228,7 @@ void pgd_dtor(void *pgd, kmem_cache_t *c {unsigned long flags;/* can be called from interrupt context */ +ClearPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags);pgd_list_del (pgd);spin_unlock_irqrestore (&pgd_lock, flags);
@@ -244,13 +246,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)pmd_t *pmd =kmem_cache_alloc (pmd_cache,GFP_KERNEL );if (!pmd) goto out_oom; +SetPagePDE (virt_to_page (pmd));set_pgd (&pgd[i],__pgd (1 +__pa (pmd))); }return pgd; out_oom: -for (i--; i >=0 ; i--) +for (i--; i >=0 ; i--) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT ));kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); + }kmem_cache_free (pgd_cache, pgd);return NULL ; }
@@ -261,8 +266,10 @@ void pgd_free(pgd_t *pgd)/* in the PAE case user pgd entries are overwritten before usage */ if (PTRS_PER_PMD >1 ) -for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) -kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); +for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT )); +kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i]) &PAGE_MASK )); + }/* in the non-PAE case, free_pgtables() clears user pgd entries */ kmem_cache_free (pgd_cache, pgd); }
include/asm-i386/pgalloc.h +16 -4
@@ -5,14 +5,22 @@#include <asm/fixmap.h> #include <linux/threads.h> #include <linux/mm.h> /* for struct page */ +#include <mach_pgalloc.h> -#define pmd_populate_kernel (mm, pmd, pte) \ - set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) +#define pmd_populate_kernel (mm, pmd, pte) \ +do { \ + SetPagePTE(virt_to_page(pte)); \ + set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))); \ +} while (0)#define pmd_populate (mm, pmd, pte) \ +do { \ + SetPagePTE(pte); \ set_pmd(pmd, __pmd(_PAGE_TABLE + \ ((unsigned long long)page_to_pfn(pte) << \ - (unsigned long long) PAGE_SHIFT))) + (unsigned long long) PAGE_SHIFT))); \ +} while (0) +/* * Allocate and free page tables. */
@@ -33,7 +41,11 @@ static inline void pte_free(struct page } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + tlb_remove_page((tlb),(pte)); \ + ClearPagePTE(pte); \ +} while (0)#ifdef CONFIG_X86_PAE /*
include/asm-i386/mach-default/mach_pgalloc.h +34 -1
@@ -1,7 +1,40 @@ +/* +* Copyright (C) 2005, VMware, Inc. +* +* All rights reserved. +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, but +* WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or +* NON INFRINGEMENT. See the GNU General Public License for more +* details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +* +* Send feedback to [email protected] +* +*/ #ifndef __ASM_MACH_PGALLOC_H#define __ASM_MACH_PGALLOC_H +/* +* These hooks allow the hypervisor to be notified about page type +* transition events. +*/ + +#define SetPagePTE (_page) +#define ClearPagePTE (_page) +#define SetPagePDE (_page) +#define ClearPagePDE (_page) +#define SetPagesLDT (_va, _pages)#define ClearPagesLDT (_va, _pages) -#endif +#endif /* _ASM_MACH_PGALLOC_H */
arch/i386/mm/init.c static pte_t * __init one_page_table_ini · line 82
Quoted 7 lines · * Zachary Amsden ([email protected]) wrote:
arch/i386/mm/init.c +1
-0@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini {if (pmd_none (*pmd)) {pte_t *page_table = (pte_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +SetPagePTE (virt_to_page (page_table));
Xen has this on one_md_table_init() as well for pmd.
Quoted 8 lines
arch/i386/mm/init.c +0
-0set_pmd (pmd,__pmd (__pa (page_table) | _PAGE_TABLE));if (page_table !=pte_offset_kernel (pmd,0 ))BUG ();Index:
linux-2.6.13/arch/i386/mm/pageattr.c
===================================================================
arch/i386/mm/pageattr.c static struct page *split_large_page(uns · line 57
Quoted 7 lines
arch/i386/mm/pageattr.c +2
-1@@ -52,8 +52,9 @@ static struct page *split_large_page(uns address =__pa (address); addr = address &LARGE_PAGE_MASK ; pbase = (pte_t *)page_address (base); +SetPagePTE (virt_to_page (pbase));for (i =0 ; i <PTRS_PER_PTE ; i++, addr +=PAGE_SIZE ) { - set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, + set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT,
No need to intersperse whitespace cleanups.
Quoted 24 lines
arch/i386/mm/pageattr.c +1
-0addr == address ? prot : PAGE_KERNEL)); }return base;@@ -146,6 +147,7 @@ __change_page_attr(struct page *page, pgBUG_ON (!page_count (kpte_page));if (cpu_has_pse && (page_count (kpte_page) ==1 )) { +ClearPagePTE (virt_to_page (kpte));list_add (&kpte_page->lru, &df_list);revert_page (kpte_page, address); }Index:
linux-2.6.13/arch/i386/mm/pgtable.c
===================================================================arch/i386/mm/pgtable.c +1
-0@@ -209,6 +209,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *cif (PTRS_PER_PMD ==1 ) {memset (pgd,0 ,USER_PTRS_PER_PGD *sizeof (pgd_t)); +SetPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags); }
arch/i386/mm/pgtable.c void pgd_dtor(void *pgd, kmem_cache_t *c · line 231
Quoted 4 lines
arch/i386/mm/pgtable.c +1
-0@@ -227,6 +228,7 @@ void pgd_dtor(void *pgd, kmem_cache_t *c {unsigned long flags;/* can be called from interrupt context */ +ClearPagePDE (virt_to_page (pgd));
This reminds me, Xen had some unconditional use of pgd_dtor I need to go back and look at.
Quoted 6 lines
arch/i386/mm/pgtable.c +0
-0spin_lock_irqsave (&pgd_lock, flags);pgd_list_del (pgd);spin_unlock_irqrestore (&pgd_lock, flags);@@ -244,13 +246,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)pmd_t *pmd =kmem_cache_alloc (pmd_cache,GFP_KERNEL );if (!pmd)
arch/i386/mm/pgtable.c pgd_t *pgd_alloc(struct mm_struct *mm) · line 256
Quoted 10 lines
arch/i386/mm/pgtable.c +3
-1goto out_oom; +SetPagePDE (virt_to_page (pmd));set_pgd (&pgd[i],__pgd (1 +__pa (pmd))); }return pgd; out_oom: -for (i--; i >=0 ; i--) +for (i--; i >=0 ; i--) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT ));
Is that the right pfn? That -1 throws me off.
Quoted 6 lines
arch/i386/mm/pgtable.c +1
-0kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); + }kmem_cache_free (pgd_cache, pgd);return NULL ; }
arch/i386/mm/pgtable.c void pgd_free(pgd_t *pgd) · line 271
Quoted 8 lines
arch/i386/mm/pgtable.c +3
-2@@ -261,8 +266,10 @@ void pgd_free(pgd_t *pgd)/* in the PAE case user pgd entries are overwritten before usage */ if (PTRS_PER_PMD >1 ) -for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) -kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); +for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT )); +kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i]) &PAGE_MASK ));
Why the switch of kmem_cache_free call?
Quoted 93 lines
arch/i386/mm/pgtable.c +1
-0+ }/* in the non-PAE case, free_pgtables() clears user pgd entries */ kmem_cache_free (pgd_cache, pgd); }Index:
linux-2.6.13/include/asm-i386/pgalloc.h
===================================================================include/asm-i386/pgalloc.h +16
-4@@ -5,14 +5,22 @@#include <asm/fixmap.h> #include <linux/threads.h> #include <linux/mm.h> /* for struct page */ +#include <mach_pgalloc.h> -#define pmd_populate_kernel (mm, pmd, pte) \ - set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) +#define pmd_populate_kernel (mm, pmd, pte) \ +do { \ + SetPagePTE(virt_to_page(pte)); \ + set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))); \ +} while (0)#define pmd_populate (mm, pmd, pte) \ +do { \ + SetPagePTE(pte); \ set_pmd(pmd, __pmd(_PAGE_TABLE + \ ((unsigned long long)page_to_pfn(pte) << \ - (unsigned long long) PAGE_SHIFT))) + (unsigned long long) PAGE_SHIFT))); \ +} while (0) +/* * Allocate and free page tables. */ @@ -33,7 +41,11 @@ static inline void pte_free(struct page } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + tlb_remove_page((tlb),(pte)); \ + ClearPagePTE(pte); \ +} while (0)#ifdef CONFIG_X86_PAE /* Index:
linux-2.6.13/include/asm-i386/mach-default/mach_pgalloc.h
===================================================================include/asm-i386/mach-default/mach_pgalloc.h +34
-1@@ -1,7 +1,40 @@ +/* +* Copyright (C) 2005, VMware, Inc. +* +* All rights reserved. +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, but +* WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or +* NON INFRINGEMENT. See the GNU General Public License for more +* details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +* +* Send feedback to [email protected] +* +*/ #ifndef __ASM_MACH_PGALLOC_H#define __ASM_MACH_PGALLOC_H +/* +* These hooks allow the hypervisor to be notified about page type +* transition events. +*/ + +#define SetPagePTE (_page) +#define ClearPagePTE (_page) +#define SetPagePDE (_page) +#define ClearPagePDE (_page) +#define SetPagesLDT (_va, _pages)#define ClearPagesLDT (_va, _pages) -#endif +#endif /* _ASM_MACH_PGALLOC_H */
Quoted 16 lines · Chris Wright
arch/i386/mm/init.c static pte_t * __init one_page_table_ini · line 84
Quoted 9 lines · * Zachary Amsden ([email protected]) wrote:
arch/i386/mm/init.c +1
-0@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini {if (pmd_none (*pmd)) {pte_t *page_table = (pte_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +SetPagePTE (virt_to_page (page_table));Xen has this on
one_md_table_init()as well for pmd.
I'll add that in another patch. It's easy to miss some of the init time call sites (we don't actually depend on them for correctness).
Quoted 30 lines
Quoted 15 lines
arch/i386/mm/init.c +0
-0spin_lock_irqsave (&pgd_lock, flags);pgd_list_del(pgd); spin_unlock_irqrestore(&pgd_lock, flags);arch/i386/mm/init.c +1
-0@@ -244,13 +246,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)pmd_t *pmd =kmem_cache_alloc (pmd_cache,GFP_KERNEL );if (!pmd) goto out_oom; +SetPagePDE (virt_to_page (pmd));set_pgd (&pgd[i],__pgd (1 +__pa (pmd))); }return pgd;
out_oom:arch/i386/mm/init.c
Quoted 3 lines
arch/i386/mm/init.c +2
-1-for (i--; i >=0 ; i--) +for (i--; i >=0 ; i--) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT ));Is that the right pfn? That -1 throws me off.
Quoted 7 lines
kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i])-1));arch/i386/mm/init.c +1
-0+ }kmem_cache_free(pgd_cache, pgd); return NULL; }arch/i386/mm/init.c +0
-0@@ -261,8 +266,10 @@ void pgd_free(pgd_t *pgd)
Quoted 13 lines
arch/i386/mm/init.c void pgd_free(pgd_t *pgd) · line 273
Quoted 9 lines
arch/i386/mm/init.c +3
-2/* in the PAE case user pgd entries are overwritten before usage */ if (PTRS_PER_PMD >1 ) -for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) -kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); +for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT )); +kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i]) &PAGE_MASK ));Why the switch of
kmem_cache_freecall?
Because pgd_val(pgd[i])-1 is confusing. Using (pgd_val(pgd[i]) - _PAGE_PRESENT) would be better, but the +/- 1s all over the place here could use some general cleanup as well. I smell a cleanup fit coming on. Using (pgd_val(pgd[i]) & PAGE_MASK) is a less error prone way to get the physical frame bits, since it is not wrong if you turn on PCD or PWD.
Zach
Quoted 19 lines · * Zachary Amsden ([email protected]) wrote:
Quoted 16 lines · Chris Wright
Review static pte_t * __init one_page_table_ini · line 85
Quoted 11 lines · * Zachary Amsden ([email protected]) wrote:
---
linux-2.6.13.orig/arch/i386/mm/init.c2005-08-24
09:31:05.000000000 -0700
+++linux-2.6.13/arch/i386/mm/init.c2005-08-24 09:31:31.000000000 -0700(quoted diff) +1
-0@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini { if (pmd_none(*pmd)) { pte_t *page_table = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE); + SetPagePTE(virt_to_page(page_table));Xen has this on
one_md_table_init()as well for pmd.
Quoted 2 lines
I'll add that in another patch. It's easy to miss some of the init time call sites (we don't actually depend on them for correctness).
You could just respin this patch.
Quoted 45 lines
Quoted 30 lines
Quoted 15 lines
spin_lock_irqsave(&pgd_lock, flags); pgd_list_del(pgd); spin_unlock_irqrestore(&pgd_lock, flags);arch/i386/mm/pgtable.c +1
-0@@ -244,13 +246,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)pmd_t *pmd =kmem_cache_alloc (pmd_cache,GFP_KERNEL );if (!pmd) goto out_oom; +SetPagePDE (virt_to_page (pmd));set_pgd (&pgd[i],__pgd (1 +__pa (pmd))); }return pgd;
out_oom:arch/i386/mm/pgtable.c pgd_t *pgd_alloc(struct mm_struct *mm) · line 256
Quoted 3 lines
arch/i386/mm/pgtable.c +2
-1-for (i--; i >=0 ; i--) +for (i--; i >=0 ; i--) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT ));Is that the right pfn? That -1 throws me off.
Quoted 7 lines
kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i])-1));(quoted diff) +1
-0+ }kmem_cache_free(pgd_cache, pgd); return NULL; }(quoted diff) +0
-0@@ -261,8 +266,10 @@ void pgd_free(pgd_t *pgd)Quoted 14 lines
Review void pgd_free(pgd_t *pgd) · line 274
Quoted 12 lines
(quoted diff) +3
-2/* in the PAE case user pgd entries are overwritten before usage */ if (PTRS_PER_PMD > 1) - for (i = 0; i < USER_PTRS_PER_PGD; ++i) - kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i])-1)); + for (i = 0; i < USER_PTRS_PER_PGD; ++i) { + ClearPagePDE(pfn_to_page(pgd_val(pgd[i]) >> PAGE_SHIFT)); + kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i]) & PAGE_MASK));Why the switch of
kmem_cache_freecall?
Quoted 1 line
Because pgd_val(pgd[i])-1 is confusing.
Yes, I agree it is. That's why I was wondering about the ClearPagePDE calls which don't use them.
Quoted 6 lines
Using (
pgd_val(pgd[i])-_PAGE_PRESENT) would be better, but the +/- 1s all over the place here could use some general cleanup as well. I smell a cleanup fit coming on. Using (pgd_val(pgd[i])&PAGE_MASK) is a less error prone way to get the physical frame bits, since it is not wrong if you turn on PCD or PWD.
Please save all that for later cleanup. As it stands it's just a conversion of one random call site, which should be dropped from the patch.
thanks,
-chris
Quoted 28 lines · Chris Wright
Quoted 25 lines · * Zachary Amsden ([email protected]) wrote:
Quoted 21 lines · Chris Wright
Review static pte_t * __init one_page_table_ini · line 85
Quoted 13 lines · * Zachary Amsden ([email protected]) wrote:
---
linux-2.6.13.orig/arch/i386/mm/init.c2005-08-24
09:31:05.000000000 -0700
+++linux-2.6.13/arch/i386/mm/init.c2005-08-24 09:31:31.000000000 -0700(quoted diff) +1
-0@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini { if (pmd_none(*pmd)) { pte_t *page_table = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE); + SetPagePTE(virt_to_page(page_table));Xen has this on
one_md_table_init()as well for pmd.
Quoted 7 lines
Quoted 3 lines
I'll add that in another patch. It's easy to miss some of the init time call sites (we don't actually depend on them for correctness).
You could just respin this patch.
Done. Looks like you want empty_zero_page write protected too. That seems like a fine idea to me unless something really wants to do atomic 64-bit reads on it.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Why the switch of
kmem_cache_freecall?
Quoted 8 lines
Quoted 3 lines
Because pgd_val(pgd[i])-1 is confusing.
Yes, I agree it is. That's why I was wondering about the ClearPagePDE calls which don't use them.
The -1 is quite useless when you're going to shift >> 12 anyways to get a frame number to index into mem_map, which is why they are not there. Plus, it just seems scary if you got it wrong - let's say you had a not present page - not that you could, but now you are freeing a misaligned address in the _previous_ page. I really don't like that -1 at all. I will clean it up, but it does certainly deserve another patch.
Quoted 10 lines
Quoted 5 lines
Using (
pgd_val(pgd[i])-_PAGE_PRESENT) would be better, but the +/- 1s all over the place here could use some general cleanup as well. I smell a cleanup fit coming on. Using (pgd_val(pgd[i])&PAGE_MASK) is a less error prone way to get the physical frame bits, since it is not wrong if you turn on PCD or PWD.Please save all that for later cleanup. As it stands it's just a conversion of one random call site, which should be dropped from the patch.
Working on it now.
Hooks are provided for the mach-XXX subarchitecture at the time prior to a page being used as a page table at all levels, for PAE and non-PAE kernels. Note that in PAE mode, multiple PDP roots may exist on the same page with other data, so the root must be shadowed instead. This is not a performance issue, since PAE only uses 4 top level PDPEs.
The hooks are:
SetPagePTE(ppn) - indicates that a given physical page is going to be used as a page table.
SetPagePDE(ppn) - indicates that a given physical page is going to be used as a page directory.
ClearPageXXX(ppn) - indicates that the physical page is now done being used as a certain type of page.
These hooks can be used in two ways; for shadow mode, they serve as requests to pre-allocate and deallocate shadow page tables, and for direct page table mode, they serve as write protect/unprotect requests.
Patch-subject: Add pagetable allocation notifiers
Index: linux-2.6.13/arch/i386/mm/init.c
===================================================================
arch/i386/mm/init.c +3 -0
@@ -59,6 +59,7 @@ static pmd_t * __init one_md_table_init(#ifdef CONFIG_X86_PAE pmd_table = (pmd_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +SetPagePDE (virt_to_page (pmd_table));set_pgd (pgd,__pgd (__pa (pmd_table) | _PAGE_PRESENT)); pud =pud_offset (pgd,0 );if (pmd_table !=pmd_offset (pud,0 ))
@@ -79,6 +80,7 @@ static pte_t * __init one_page_table_ini {if (pmd_none (*pmd)) {pte_t *page_table = (pte_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +SetPagePTE (virt_to_page (page_table));set_pmd (pmd,__pmd (__pa (page_table) | _PAGE_TABLE));if (page_table !=pte_offset_kernel (pmd,0 ))BUG ();
@@ -313,6 +315,7 @@ static void __init pagetable_init (void)#ifdef CONFIG_X86_PAE int i;/* Init entries of the first-level page table to the zero page */ +SetPagePDE (virt_to_page (empty_zero_page));for (i =0 ; i <PTRS_PER_PGD ; i++)set_pgd (pgd_base + i,__pgd (__pa (empty_zero_page) | _PAGE_PRESENT));#endif
arch/i386/mm/pageattr.c +2 -0
@@ -52,6 +52,7 @@ static struct page *split_large_page(uns address =__pa (address); addr = address &LARGE_PAGE_MASK ; pbase = (pte_t *)page_address (base); +SetPagePTE (virt_to_page (pbase));for (i =0 ; i <PTRS_PER_PTE ; i++, addr +=PAGE_SIZE ) {set_pte (&pbase[i],pfn_pte (addr >>PAGE_SHIFT , addr == address ? prot :PAGE_KERNEL ));
@@ -146,6 +147,7 @@ __change_page_attr(struct page *page, pgBUG_ON (!page_count (kpte_page));if (cpu_has_pse && (page_count (kpte_page) ==1 )) { +ClearPagePTE (virt_to_page (kpte));list_add (&kpte_page->lru, &df_list);revert_page (kpte_page, address); }
arch/i386/mm/pgtable.c +9 -2
@@ -209,6 +209,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *cif (PTRS_PER_PMD ==1 ) {memset (pgd,0 ,USER_PTRS_PER_PGD *sizeof (pgd_t)); +SetPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags); }
@@ -227,6 +228,7 @@ void pgd_dtor(void *pgd, kmem_cache_t *c {unsigned long flags;/* can be called from interrupt context */ +ClearPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags);pgd_list_del (pgd);spin_unlock_irqrestore (&pgd_lock, flags);
@@ -244,13 +246,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)pmd_t *pmd =kmem_cache_alloc (pmd_cache,GFP_KERNEL );if (!pmd) goto out_oom; +SetPagePDE (virt_to_page (pmd));set_pgd (&pgd[i],__pgd (1 +__pa (pmd))); }return pgd; out_oom: -for (i--; i >=0 ; i--) +for (i--; i >=0 ; i--) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT ));kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); + }kmem_cache_free (pgd_cache, pgd);return NULL ; }
@@ -261,8 +266,10 @@ void pgd_free(pgd_t *pgd)/* in the PAE case user pgd entries are overwritten before usage */ if (PTRS_PER_PMD >1 ) -for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) +for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) { +ClearPagePDE (pfn_to_page (pgd_val (pgd[i]) >>PAGE_SHIFT ));kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); + }/* in the non-PAE case, free_pgtables() clears user pgd entries */ kmem_cache_free (pgd_cache, pgd); }
include/asm-i386/pgalloc.h +16 -4
@@ -5,14 +5,22 @@#include <asm/fixmap.h> #include <linux/threads.h> #include <linux/mm.h> /* for struct page */ +#include <mach_pgalloc.h> -#define pmd_populate_kernel (mm, pmd, pte) \ - set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) +#define pmd_populate_kernel (mm, pmd, pte) \ +do { \ + SetPagePTE(virt_to_page(pte)); \ + set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))); \ +} while (0)#define pmd_populate (mm, pmd, pte) \ +do { \ + SetPagePTE(pte); \ set_pmd(pmd, __pmd(_PAGE_TABLE + \ ((unsigned long long)page_to_pfn(pte) << \ - (unsigned long long) PAGE_SHIFT))) + (unsigned long long) PAGE_SHIFT))); \ +} while (0) +/* * Allocate and free page tables. */
@@ -33,7 +41,11 @@ static inline void pte_free(struct page } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + tlb_remove_page((tlb),(pte)); \ + ClearPagePTE(pte); \ +} while (0)#ifdef CONFIG_X86_PAE /*
include/asm-i386/mach-default/mach_pgalloc.h +34 -1
@@ -1,7 +1,40 @@ +/* +* Copyright (C) 2005, VMware, Inc. +* +* All rights reserved. +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, but +* WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or +* NON INFRINGEMENT. See the GNU General Public License for more +* details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +* +* Send feedback to [email protected] +* +*/ #ifndef __ASM_MACH_PGALLOC_H#define __ASM_MACH_PGALLOC_H +/* +* These hooks allow the hypervisor to be notified about page type +* transition events. +*/ + +#define SetPagePTE (_page) +#define ClearPagePTE (_page) +#define SetPagePDE (_page) +#define ClearPagePDE (_page) +#define SetPagesLDT (_va, _pages)#define ClearPagesLDT (_va, _pages) -#endif +#endif /* _ASM_MACH_PGALLOC_H */
Quoted 3 lines · * Zachary Amsden ([email protected]) wrote:
Done. Looks like you want
empty_zero_pagewrite protected too. That seems like a fine idea to me unless something really wants to do atomic 64-bit reads on it.
Thanks, I added this set (minus the 3/5, which I already have) to the virt-2.6 tree.
Quoted 6 lines
The -1 is quite useless when you're going to shift >> 12 anyways to get a frame number to index into
mem_map, which is why they are not there. Plus, it just seems scary if you got it wrong - let's say you had a not present page - not that you could, but now you are freeing a misaligned address in the _previous_ page. I really don't like that -1 at all. I will clean it up, but it does certainly deserve another patch.
Heh, that's exactly what I was concerned with from the patch (being on the wrong page). Thanks for clearing up my confusion.
Quoted 66 lines · Chris Wright
arch/i386/mm/init.c static pte_t * __init one_page_table_ini · line 84
Quoted 9 lines · * Zachary Amsden ([email protected]) wrote:
arch/i386/mm/init.c +1
-0@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini {if (pmd_none (*pmd)) {pte_t *page_table = (pte_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +SetPagePTE (virt_to_page (page_table));Xen has this on
one_md_table_init()as well for pmd.Quoted 8 lines
arch/i386/mm/init.c +0
-0set_pmd (pmd,__pmd (__pa (page_table) | _PAGE_TABLE));if (page_table != pte_offset_kernel(pmd, 0)) BUG();Index:
linux-2.6.13/arch/i386/mm/pageattr.c
===================================================================arch/i386/mm/pageattr.c static struct page *split_large_page(uns · line 59
Quoted 9 lines
arch/i386/mm/pageattr.c +2
-1@@ -52,8 +52,9 @@ static struct page *split_large_page(uns address =__pa (address); addr = address &LARGE_PAGE_MASK ; pbase = (pte_t *)page_address (base); +SetPagePTE (virt_to_page (pbase));for (i =0 ; i <PTRS_PER_PTE ; i++, addr +=PAGE_SIZE ) { - set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, + set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT,No need to intersperse whitespace cleanups.
Quoted 27 lines
arch/i386/mm/pageattr.c +0
-0addr == address ? prot : PAGE_KERNEL));} return base;arch/i386/mm/pageattr.c +1
-0@@ -146,6 +147,7 @@ __change_page_attr(struct page *page, pgBUG_ON (!page_count (kpte_page));if (cpu_has_pse && (page_count (kpte_page) ==1 )) { +ClearPagePTE (virt_to_page (kpte));list_add (&kpte_page->lru, &df_list);revert_page (kpte_page, address); }Index:
linux-2.6.13/arch/i386/mm/pgtable.c
===================================================================arch/i386/mm/pgtable.c +1
-0@@ -209,6 +209,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *cif (PTRS_PER_PMD ==1 ) {memset (pgd,0 ,USER_PTRS_PER_PGD *sizeof (pgd_t)); +SetPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags); }@@ -227,6 +228,7 @@ void pgd_dtor(void *pgd, kmem_cache_t *c {unsigned long flags;/* can be called from interrupt context */
Quoted 8 lines
arch/i386/mm/pgtable.c void pgd_dtor(void *pgd, kmem_cache_t *c · line 233
Quoted 3 lines
arch/i386/mm/pgtable.c +1
-0+ClearPagePDE (virt_to_page (pgd));This reminds me, Xen had some unconditional use of
pgd_dtorI need to go back and look at.
It looks quite unneeded:
void pgd_dtor(void *pgd, kmem_cache_t *cache, unsigned long unused)
{
unsigned long flags; /* can be called from interrupt context */
if (HAVE_SHARED_KERNEL_PMD) <--- /* PAE */
return;
spin_lock_irqsave(&pgd_lock, flags);
pgd_list_del(pgd);
spin_unlock_irqrestore(&pgd_lock, flags);
}Leaving the dtor conditional saves an indirect function call ;)
Zach
Quoted 84 lines · * Zachary Amsden ([email protected]) wrote:
Quoted 73 lines · Chris Wright
Review static pte_t * __init one_page_table_ini · line 85
Quoted 11 lines · * Zachary Amsden ([email protected]) wrote:
---
linux-2.6.13.orig/arch/i386/mm/init.c2005-08-24
09:31:05.000000000 -0700
+++linux-2.6.13/arch/i386/mm/init.c2005-08-24 09:31:31.000000000 -0700(quoted diff) +1
-0@@ -79,6 +79,7 @@ static pte_t * __init one_page_table_ini { if (pmd_none(*pmd)) { pte_t *page_table = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE); + SetPagePTE(virt_to_page(page_table));Xen has this on
one_md_table_init()as well for pmd.Quoted 10 lines
set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); if (page_table != pte_offset_kernel(pmd, 0)) BUG();Index:
linux-2.6.13/arch/i386/mm/pageattr.c
===================================================================
---linux-2.6.13.orig/arch/i386/mm/pageattr.c2005-08-24
09:31:05.000000000 -0700
+++linux-2.6.13/arch/i386/mm/pageattr.c2005-08-24
09:31:31.000000000 -0700arch/i386/mm/pageattr.c static struct page *split_large_page(uns · line 59
Quoted 9 lines
arch/i386/mm/pageattr.c +2
-1@@ -52,8 +52,9 @@ static struct page *split_large_page(uns address =__pa (address); addr = address &LARGE_PAGE_MASK ; pbase = (pte_t *)page_address (base); +SetPagePTE (virt_to_page (pbase));for (i =0 ; i <PTRS_PER_PTE ; i++, addr +=PAGE_SIZE ) { - set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, + set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT,No need to intersperse whitespace cleanups.
Quoted 30 lines
(quoted diff) +0
-0addr == address ? prot :PAGE_KERNEL)); } return base;arch/i386/mm/pageattr.c +1
-0@@ -146,6 +147,7 @@ __change_page_attr(struct page *page, pgBUG_ON (!page_count (kpte_page));if (cpu_has_pse && (page_count (kpte_page) ==1 )) { +ClearPagePTE (virt_to_page (kpte));list_add (&kpte_page->lru, &df_list);revert_page (kpte_page, address); }Index:
linux-2.6.13/arch/i386/mm/pgtable.c
===================================================================
---linux-2.6.13.orig/arch/i386/mm/pgtable.c2005-08-24
09:31:05.000000000 -0700
+++linux-2.6.13/arch/i386/mm/pgtable.c2005-08-24
09:40:22.000000000 -0700arch/i386/mm/pgtable.c +1
-0@@ -209,6 +209,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *cif (PTRS_PER_PMD ==1 ) {memset (pgd,0 ,USER_PTRS_PER_PGD *sizeof (pgd_t)); +SetPagePDE (virt_to_page (pgd));spin_lock_irqsave (&pgd_lock, flags); }@@ -227,6 +228,7 @@ void pgd_dtor(void *pgd, kmem_cache_t *c {unsigned long flags;/* can be called from interrupt context */ Quoted 8 lines
Review void pgd_dtor(void *pgd, kmem_cache_t *c · line 233
Quoted 3 lines
(quoted diff) +1
-0+ ClearPagePDE(virt_to_page(pgd));This reminds me, Xen had some unconditional use of
pgd_dtorI need to go back and look at.
Quoted 8 lines
It looks quite unneeded:
void
pgd_dtor(void *pgd, kmem_cache_t *cache, unsigned long unused){ unsigned long flags; /* can be called from interrupt context */ if (HAVE_SHARED_KERNEL_PMD) <--- /* PAE */ return;
Yeah, that's not quite the case:
include/asm-xen/asm-i386/pgtable-3level-defs.h:#define HAVE_SHARED_KERNEL_PMD 0 include/asm-xen/asm-i386/pgtable-2level-defs.h:#define HAVE_SHARED_KERNEL_PMD 0