x86: fix change_page_attr() TLB and cache flushing
PATCH 1/1 x86: fix change_page_attr() TLB and cache flushing
change_page_attr() failed to flush the caches for the affected pages for quite a while, and on x86-64 only recently got a brute-force fix to flush the entire TLB (which now gets reduced to just the affected pages when possible).
Signed-off-by Jan Beulich <[email protected]>
arch/i386/mm/ioremap.c | 4 arch/i386/mm/pageattr.c | 181 +++++++++++++++++++++++++++----------- arch/x86_64/mm/ioremap.c | 6 - arch/x86_64/mm/pageattr.c | 202 ++++++++++++++++++++++++++++++------------- include/asm-i386/page.h | 10 ++ include/asm-i386/pgtable.h | 3 include/asm-x86_64/page.h | 2 include/asm-x86_64/pgtable.h | 4 include/asm-x86_64/system.h | 2 9 files changed, 292 insertions(+), 122 deletions(-)
arch/i386/mm/ioremap.c +2 -2
@@ -193,10 +193,10 @@ void iounmap(volatile void __iomem *addrreturn ; } -/* Reset the direct mapping. Can block */ +/* Reset the direct mapping. Can block. Heed the trailing guard page. */ if ((p->flags >>20 ) && p->phys_addr <virt_to_phys (high_memory) -1 ) {change_page_attr (virt_to_page (__va (p->phys_addr)), - p->size >>PAGE_SHIFT , + (p->size >>PAGE_SHIFT ) -1 ,PAGE_KERNEL );global_flush_tlb (); }
arch/i386/mm/pageattr.c +129 -52
271 lines not on this page · raw message
arch/x86_64/mm/ioremap.c +3 -3
@@ -48,7 +48,7 @@ ioremap_change_attr(unsigned long phys_a* Must use a address here and not struct page because the phys addr * can be a in hole between nodes and not have an memmap entry. */ - err =change_page_attr_addr (vaddr,npages,__pgprot (__PAGE_KERNEL|flags)); + err =change_page_attr_addr (vaddr,npages,MAKE_GLOBAL (__PAGE_KERNEL|flags));if (!err)global_flush_tlb (); }
@@ -197,9 +197,9 @@ void iounmap(volatile void __iomem *addrreturn ; } -/* Reset the direct mapping. Can block */ +/* Reset the direct mapping. Can block. Heed the trailing guard page. */ if (p->flags >>20 ) -ioremap_change_attr (p->phys_addr, p->size,0 ); +ioremap_change_attr (p->phys_addr, p->size -PAGE_SIZE ,0 );/* Finally remove it */ o =remove_vm_area ((void *)addr);
arch/x86_64/mm/pageattr.c +141 -61
285 lines not on this page · raw message
include/asm-i386/page.h +10 -0
@@ -6,6 +6,16 @@#define PAGE_SIZE (1UL << PAGE_SHIFT)#define PAGE_MASK (~(PAGE_SIZE-1)) +#ifdef CONFIG_X86_PAE +#define __PHYSICAL_MASK_SHIFT 52 +#define __PHYSICAL_MASK ((1ULL << __PHYSICAL_MASK_SHIFT) - 1) +#define PHYSICAL_PAGE_MASK (~(PAGE_SIZE - 1ULL) & __PHYSICAL_MASK) +#else +#define __PHYSICAL_MASK_SHIFT 32 +#define __PHYSICAL_MASK (~0UL) +#define PHYSICAL_PAGE_MASK (PAGE_MASK & __PHYSICAL_MASK) +#endif +#define LARGE_PAGE_MASK (~(LARGE_PAGE_SIZE-1))#define LARGE_PAGE_SIZE (1UL << PMD_SHIFT)
include/asm-i386/pgtable.h +3 -0
@@ -235,6 +235,7 @@ static inline pte_t pte_exprotect(pte_tstatic inline pte_t pte_mkclean (pte_t pte) { (pte).pte_low &= ~_PAGE_DIRTY;return pte; }static inline pte_t pte_mkold (pte_t pte) { (pte).pte_low &= ~_PAGE_ACCESSED;return pte; }static inline pte_t pte_wrprotect (pte_t pte) { (pte).pte_low &= ~_PAGE_RW;return pte; } +static inline pte_t pte_clrhuge (pte_t pte) { (pte).pte_low &= ~_PAGE_PSE;return pte; }static inline pte_t pte_mkread (pte_t pte) { (pte).pte_low |= _PAGE_USER;return pte; }static inline pte_t pte_mkexec (pte_t pte) { (pte).pte_low |= _PAGE_USER;return pte; }static inline pte_t pte_mkdirty (pte_t pte) { (pte).pte_low |= _PAGE_DIRTY;return pte; }
@@ -242,6 +243,8 @@ static inline pte_t pte_mkyoung(pte_t ptstatic inline pte_t pte_mkwrite (pte_t pte) { (pte).pte_low |= _PAGE_RW;return pte; }static inline pte_t pte_mkhuge (pte_t pte) { (pte).pte_low |= _PAGE_PSE;return pte; } +#define pte_pgprot (pte) (__pgprot(pte_val(pte) & ~PHYSICAL_PAGE_MASK)) +#ifdef CONFIG_X86_PAE # include <asm/pgtable-3level.h> #else
include/asm-x86_64/page.h +1 -1
@@ -97,7 +97,7 @@ extern unsigned long phys_base;#define PAGE_ALIGN (addr) (((addr)+PAGE_SIZE-1)&PAGE_MASK)/* See Documentation/x86_64/mm.txt for a description of the memory map. */ -#define __PHYSICAL_MASK_SHIFT 46 +#define __PHYSICAL_MASK_SHIFT 52#define __PHYSICAL_MASK ((_AC(1,UL) << __PHYSICAL_MASK_SHIFT) - 1)#define __VIRTUAL_MASK_SHIFT 48#define __VIRTUAL_MASK ((_AC(1,UL) << __VIRTUAL_MASK_SHIFT) - 1)
include/asm-x86_64/pgtable.h +2 -2
@@ -356,9 +356,9 @@ static inline int pmd_large(pmd_t pte) {#define pfn_pmd (nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot)))#define pmd_pfn (x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) -#define pte_to_pgoff (pte) ((pte_val(pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) +#define pte_to_pgoff (pte) (pte_val(pte) >> PAGE_SHIFT)#define pgoff_to_pte (off) ((pte_t) { ((off) << PAGE_SHIFT) | _PAGE_FILE }) -#define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT +#define PTE_FILE_MAX_BITS (64 - PAGE_SHIFT)/* PTE - Level 1 access. */
include/asm-x86_64/system.h +1 -1
@@ -109,7 +109,7 @@ static inline void write_cr4(unsigned lo#define stts () write_cr0(8 | read_cr0())#define wbinvd () \ - __asm__ __volatile__ ("wbinvd": : :"memory"); + __asm__ __volatile__ ("wbinvd": : :"memory")/* * On SMP systems, when the scheduler does migration-cost autodetection,