linux/x86: Adjust page table handling
PATCH 1/1 linux/x86: Adjust page table handling
Ensure that all and only those page table entries that have their present bit set undergo p2m/m2p translation in all relevant places. This should fix migration issues with _PAGE_PROTNONE pages which previously could retain MFNs in PTEs while having the present bit clear (and thus were not getting (un)canonicalized during save/restore).
Many thanks to Keir Fraser, who set me strait on a number of aspects in this area while working out the way things should work.
Signed-off-by Jan Beulich <[email protected]>
include/asm-i386/mach-xen/asm/maddr.h +20 -5
@@ -21,6 +21,7 @@ typedef unsigned long maddr_t;#ifdef CONFIG_XEN extern unsigned long *phys_to_machine_mapping; +extern unsigned long max_mapnr;#undef machine_to_phys_mappingextern unsigned long *machine_to_phys_mapping;
@@ -30,20 +31,20 @@ static inline unsigned long pfn_to_mfn(u {if (xen_feature (XENFEAT_auto_translated_physmap))return pfn; -return phys_to_machine_mapping[(unsigned int )(pfn)] & - ~FOREIGN_FRAME_BIT ; +BUG_ON (max_mapnr && pfn >= max_mapnr); +return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT ; }static inline int phys_to_machine_mapping_valid (unsigned long pfn) {if (xen_feature (XENFEAT_auto_translated_physmap))return 1 ; +BUG_ON (max_mapnr && pfn >= max_mapnr);return (phys_to_machine_mapping[pfn] !=INVALID_P2M_ENTRY ); }static inline unsigned long mfn_to_pfn (unsigned long mfn) { -extern unsigned long max_mapnr;unsigned long pfn;if (xen_feature (XENFEAT_auto_translated_physmap))
@@ -92,7 +93,6 @@ static inline unsigned long mfn_to_pfn(u*/ static inline unsigned long mfn_to_local_pfn (unsigned long mfn) { -extern unsigned long max_mapnr;unsigned long pfn =mfn_to_pfn (mfn);if ((pfn < max_mapnr) && !xen_feature(XENFEAT_auto_translated_physmap)
@@ -103,6 +103,7 @@ static inline unsigned long mfn_to_localstatic inline void set_phys_to_machine (unsigned long pfn,unsigned long mfn) { +BUG_ON (pfn >= max_mapnr);if (xen_feature (XENFEAT_auto_translated_physmap)) {BUG_ON (pfn != mfn && mfn !=INVALID_P2M_ENTRY );return ;
@@ -124,6 +125,20 @@ static inline paddr_t machine_to_phys(mareturn phys; } +#ifdef CONFIG_X86_PAE +static inline paddr_t pte_phys_to_machine (paddr_t phys) +{ +/* +* In PAE mode, the NX bit needs to be dealt with in the value +* passed to pfn_to_mfn(). On x86_64, we need to mask it off, +* but for i386 the conversion to ulong for the argument will +* clip it off. +*/ +maddr_t machine =pfn_to_mfn (phys >>PAGE_SHIFT ); + machine = (machine <<PAGE_SHIFT ) | (phys & ~PHYSICAL_PAGE_MASK ); +return machine; +} +static inline paddr_t pte_machine_to_phys (maddr_t machine) {/*
@@ -136,6 +151,7 @@ static inline paddr_t pte_machine_to_phy phys = (phys <<PAGE_SHIFT ) | (machine & ~PHYSICAL_PAGE_MASK ); return phys; } +#endif #else /* !CONFIG_XEN */
@@ -146,7 +162,6 @@ static inline paddr_t pte_machine_to_phy#define phys_to_machine_mapping_valid (pfn) (1)#define phys_to_machine (phys) ((maddr_t)(phys))#define machine_to_phys (mach) ((paddr_t)(mach)) -#define pte_machine_to_phys (mach) ((paddr_t)(mach))#endif /* !CONFIG_XEN */
include/asm-i386/mach-xen/asm/page.h +28 -23
97 lines not on this page · raw message
include/asm-i386/mach-xen/asm/pgtable-2level.h +1 -1
@@ -39,7 +39,7 @@#define ptep_get_and_clear (mm,addr,xp) __pte_ma(xchg(&(xp)->pte_low, 0))#define pte_same (a, b) ((a).pte_low == (b).pte_low)#define pte_mfn (_pte) ((_pte).pte_low >> PAGE_SHIFT) -#define pte_pfn (_pte) mfn_to_local_pfn(pte_mfn(_pte)) +#define pte_pfn (_pte) (pte_val(_pte) >> PAGE_SHIFT)#define pte_page (_pte) pfn_to_page(pte_pfn(_pte))
include/asm-i386/mach-xen/asm/pgtable-3level.h +5 -5
@@ -146,20 +146,20 @@ static inline int pte_none(pte_t pte) }#define pte_mfn (_pte) (((_pte).pte_low >> PAGE_SHIFT) |\ - (((_pte).pte_high & 0xfff) << (32-PAGE_SHIFT))) -#define pte_pfn (_pte) mfn_to_local_pfn(pte_mfn(_pte)) + ((_pte).pte_high << (32-PAGE_SHIFT))) +#define pte_pfn (_pte) ((pte_val(_pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)extern unsigned long long __supported_pte_mask;static inline pte_t pfn_pte (unsigned long page_nr,pgprot_t pgprot) { -return pfn_pte_ma (pfn_to_mfn (page_nr), pgprot); +return __pte ((((unsigned long long )page_nr <<PAGE_SHIFT ) | +pgprot_val (pgprot)) & __supported_pte_mask); }static inline pmd_t pfn_pmd (unsigned long page_nr,pgprot_t pgprot) { -BUG ();panic ("needs review" ); -return __pmd ((((unsigned long long )page_nr <<PAGE_SHIFT ) | \ +return __pmd ((((unsigned long long )page_nr <<PAGE_SHIFT ) |pgprot_val (pgprot)) & __supported_pte_mask); }
include/asm-i386/mach-xen/asm/pgtable.h +10 -9
@@ -315,18 +315,19 @@ static inline void clone_pgd_range(pgd_tstatic inline pte_t pte_modify (pte_t pte,pgprot_t newprot) { - pte.pte_low &= _PAGE_CHG_MASK; - pte.pte_low |=pgprot_val (newprot); -#ifdef CONFIG_X86_PAE /* -* Chop off the NX bit (if present), and add the NX portion of -* the newprot (if present): +* Since this might change the present bit (which controls whether +* a pte_t object has undergone p2m translation), we must use +* pte_val() on the input pte and __pte() for the return value. */ - pte.pte_high &= ~(1 << (_PAGE_BIT_NX -32 )); - pte.pte_high |= (pgprot_val (newprot) >>32 ) & \ - (__supported_pte_mask >>32 ); +paddr_t pteval =pte_val (pte); + + pteval &= _PAGE_CHG_MASK; + pteval |=pgprot_val (newprot); +#ifdef CONFIG_X86_PAE + pteval &= __supported_pte_mask;#endif -return pte; +return __pte (pteval); }#define pmd_large (pmd) \
include/asm-x86_64/mach-xen/asm/maddr.h +12 -3
@@ -25,14 +25,15 @@ static inline unsigned long pfn_to_mfn(u {if (xen_feature (XENFEAT_auto_translated_physmap))return pfn; -return phys_to_machine_mapping[(unsigned int )(pfn)] & - ~FOREIGN_FRAME_BIT ; +BUG_ON (end_pfn && pfn >= end_pfn); +return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT ; }static inline int phys_to_machine_mapping_valid (unsigned long pfn) {if (xen_feature (XENFEAT_auto_translated_physmap))return 1 ; +BUG_ON (end_pfn && pfn >= end_pfn);return (phys_to_machine_mapping[pfn] !=INVALID_P2M_ENTRY ); }
@@ -96,6 +97,7 @@ static inline unsigned long mfn_to_localstatic inline void set_phys_to_machine (unsigned long pfn,unsigned long mfn) { +BUG_ON (pfn >= end_pfn);if (xen_feature (XENFEAT_auto_translated_physmap)) {BUG_ON (pfn != mfn && mfn !=INVALID_P2M_ENTRY );return ;
@@ -117,6 +119,14 @@ static inline paddr_t machine_to_phys(mareturn phys; } +static inline paddr_t pte_phys_to_machine (paddr_t phys) +{ +maddr_t machine; + machine =pfn_to_mfn ((phys &PHYSICAL_PAGE_MASK ) >>PAGE_SHIFT ); + machine = (machine <<PAGE_SHIFT ) | (phys & ~PHYSICAL_PAGE_MASK ); +return machine; +} +static inline paddr_t pte_machine_to_phys (maddr_t machine) {paddr_t phys;
@@ -134,7 +144,6 @@ static inline paddr_t pte_machine_to_phy#define phys_to_machine_mapping_valid (pfn) (1)#define phys_to_machine (phys) ((maddr_t)(phys))#define machine_to_phys (mach) ((paddr_t)(mach)) -#define pte_machine_to_phys (mach) ((paddr_t)(mach))#endif /* !CONFIG_XEN */
include/asm-x86_64/mach-xen/asm/page.h +20 -8
@@ -9,6 +9,13 @@#endif #include <xen/interface/xen.h> +/* +* Need to repeat this here in order to not include pgtable.h (which in turn +* depends on definitions made here), but to be able to use the symbolic +* below. The preprocessor will warn if the two definitions aren't identical. +*/ +#define _PAGE_PRESENT 0x001 +#define arch_free_page (_page,_order) \ ({ int foreign = PageForeign(_page); \ if (foreign) \
@@ -95,28 +102,33 @@ typedef struct { unsigned long pgd; } pgtypedef struct {unsigned long pgprot; }pgprot_t ; -#define pte_val (x) (((x).pte & 1) ? pte_machine_to_phys((x).pte) : \ +#define pte_val (x) (((x).pte & _PAGE_PRESENT) ? \ + pte_machine_to_phys((x).pte) : \ (x).pte)#define pte_val_ma (x) ((x).pte)static inline unsigned long pmd_val (pmd_t x) {unsigned long ret = x.pmd; -if (ret) ret =pte_machine_to_phys (ret); +#ifdef CONFIG_XEN_COMPAT_030002 +if (ret) ret =pte_machine_to_phys (ret) | _PAGE_PRESENT; +#else +if (ret & _PAGE_PRESENT) ret =pte_machine_to_phys (ret); +#endif return ret; }static inline unsigned long pud_val (pud_t x) {unsigned long ret = x.pud; -if (ret) ret =pte_machine_to_phys (ret); +if (ret & _PAGE_PRESENT) ret =pte_machine_to_phys (ret);return ret; }static inline unsigned long pgd_val (pgd_t x) {unsigned long ret = x.pgd; -if (ret) ret =pte_machine_to_phys (ret); +if (ret & _PAGE_PRESENT) ret =pte_machine_to_phys (ret);return ret; }
@@ -124,25 +136,25 @@ static inline unsigned long pgd_val(pgd_static inline pte_t __pte (unsigned long x) { -if (x &1 ) x =phys_to_machine (x); +if (x & _PAGE_PRESENT) x =pte_phys_to_machine (x);return ((pte_t ) { (x) }); }static inline pmd_t __pmd (unsigned long x) { -if ((x &1 )) x =phys_to_machine (x); +if (x & _PAGE_PRESENT) x =pte_phys_to_machine (x);return ((pmd_t ) { (x) }); }static inline pud_t __pud (unsigned long x) { -if ((x &1 )) x =phys_to_machine (x); +if (x & _PAGE_PRESENT) x =pte_phys_to_machine (x);return ((pud_t ) { (x) }); }static inline pgd_t __pgd (unsigned long x) { -if ((x &1 )) x =phys_to_machine (x); +if (x & _PAGE_PRESENT) x =pte_phys_to_machine (x);return ((pgd_t ) { (x) }); }
include/asm-x86_64/mach-xen/asm/pgtable.h +19 -14
@@ -303,18 +303,16 @@ static inline unsigned long pud_bad(pud_#define pages_to_mb (x) ((x) >> (20-PAGE_SHIFT))#define pte_mfn (_pte) (((_pte).pte & PTE_MASK) >> PAGE_SHIFT) -#define pte_pfn (_pte) mfn_to_local_pfn(pte_mfn(_pte)) +#define pte_pfn (_pte) ((pte_val(_pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)#define pte_page (x) pfn_to_page(pte_pfn(x))static inline pte_t pfn_pte (unsigned long page_nr,pgprot_t pgprot) { -pte_t pte; - - (pte).pte = (pfn_to_mfn (page_nr) <<PAGE_SHIFT ); - (pte).pte |=pgprot_val (pgprot); - (pte).pte &= __supported_pte_mask; -return pte; +unsigned long pte = page_nr <<PAGE_SHIFT ; + pte |=pgprot_val (pgprot); + pte &= __supported_pte_mask; +return __pte (pte); }/*
@@ -446,18 +444,25 @@ static inline pud_t *pud_offset_k(pgd_t/* physical address -> PTE */ static inline pte_t mk_pte_phys (unsigned long physpage,pgprot_t pgprot) { -pte_t pte; - (pte).pte = physpage |pgprot_val (pgprot); -return pte; +unsigned long pteval; + pteval = physpage |pgprot_val (pgprot); +return __pte (pteval); }/* Change flags of a PTE */ static inline pte_t pte_modify (pte_t pte,pgprot_t newprot) { - (pte).pte &= _PAGE_CHG_MASK; - (pte).pte |=pgprot_val (newprot); - (pte).pte &= __supported_pte_mask; -return pte; +/* +* Since this might change the present bit (which controls whether +* a pte_t object has undergone p2m translation), we must use +* pte_val() on the input pte and __pte() for the return value. +*/ +unsigned long pteval =pte_val (pte); + + pteval &= _PAGE_CHG_MASK; + pteval |=pgprot_val (newprot); + pteval &= __supported_pte_mask; +return __pte (pteval); }#define pte_index (address) \