Skip to contentSkip to conversation

x86: fix change_page_attr() TLB and cache flushing

PATCH New Jan Beulich 1 participant 1 message Jun 12, 2007

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86: fix change_page_attr() TLB and cache flushing

change_page_attr() failed to flush the caches for the affected pages for quite a while, and on x86-64 only recently got a brute-force fix to flush the entire TLB (which now gets reduced to just the affected pages when possible).

 arch/i386/mm/ioremap.c       |    4 
 arch/i386/mm/pageattr.c      |  181 +++++++++++++++++++++++++++-----------
 arch/x86_64/mm/ioremap.c     |    6 -
 arch/x86_64/mm/pageattr.c    |  202 ++++++++++++++++++++++++++++++-------------
 include/asm-i386/page.h      |   10 ++
 include/asm-i386/pgtable.h   |    3 
 include/asm-x86_64/page.h    |    2 
 include/asm-x86_64/pgtable.h |    4 
 include/asm-x86_64/system.h  |    2 
 9 files changed, 292 insertions(+), 122 deletions(-)
arch/i386/mm/ioremap.c +2 -2
@@ -193,10 +193,10 @@ void iounmap(volatile void __iomem *addr
 		return;
 	}
 
-	/* Reset the direct mapping. Can block */
+	/* Reset the direct mapping. Can block. Heed the trailing guard page. */
 	if ((p->flags >> 20) && p->phys_addr < virt_to_phys(high_memory) - 1) {
 		change_page_attr(virt_to_page(__va(p->phys_addr)),
-				 p->size >> PAGE_SHIFT,
+				 (p->size >> PAGE_SHIFT) - 1,
 				 PAGE_KERNEL);
 		global_flush_tlb();
 	} 
arch/i386/mm/pageattr.c +129 -52

271 lines not on this page · raw message

arch/x86_64/mm/ioremap.c +3 -3
@@ -48,7 +48,7 @@ ioremap_change_attr(unsigned long phys_a
  		 * Must use a address here and not struct page because the phys addr
 		 * can be a in hole between nodes and not have an memmap entry.
 		 */
-		err = change_page_attr_addr(vaddr,npages,__pgprot(__PAGE_KERNEL|flags));
+		err = change_page_attr_addr(vaddr,npages,MAKE_GLOBAL(__PAGE_KERNEL|flags));
 		if (!err)
 			global_flush_tlb();
 	}
@@ -197,9 +197,9 @@ void iounmap(volatile void __iomem *addr
 		return;
 	}
 
-	/* Reset the direct mapping. Can block */
+	/* Reset the direct mapping. Can block. Heed the trailing guard page. */
 	if (p->flags >> 20)
-		ioremap_change_attr(p->phys_addr, p->size, 0);
+		ioremap_change_attr(p->phys_addr, p->size - PAGE_SIZE, 0);
 
 	/* Finally remove it */
 	o = remove_vm_area((void *)addr);
arch/x86_64/mm/pageattr.c +141 -61

285 lines not on this page · raw message

include/asm-i386/page.h +10 -0
@@ -6,6 +6,16 @@
 #define PAGE_SIZE	(1UL << PAGE_SHIFT)
 #define PAGE_MASK	(~(PAGE_SIZE-1))
 
+#ifdef CONFIG_X86_PAE
+#define __PHYSICAL_MASK_SHIFT	52
+#define __PHYSICAL_MASK		((1ULL << __PHYSICAL_MASK_SHIFT) - 1)
+#define PHYSICAL_PAGE_MASK	(~(PAGE_SIZE - 1ULL) & __PHYSICAL_MASK)
+#else
+#define __PHYSICAL_MASK_SHIFT	32
+#define __PHYSICAL_MASK		(~0UL)
+#define PHYSICAL_PAGE_MASK	(PAGE_MASK & __PHYSICAL_MASK)
+#endif
+
 #define LARGE_PAGE_MASK (~(LARGE_PAGE_SIZE-1))
 #define LARGE_PAGE_SIZE (1UL << PMD_SHIFT)
 
include/asm-i386/pgtable.h +3 -0
@@ -235,6 +235,7 @@ static inline pte_t pte_exprotect(pte_t 
 static inline pte_t pte_mkclean(pte_t pte)	{ (pte).pte_low &= ~_PAGE_DIRTY; return pte; }
 static inline pte_t pte_mkold(pte_t pte)	{ (pte).pte_low &= ~_PAGE_ACCESSED; return pte; }
 static inline pte_t pte_wrprotect(pte_t pte)	{ (pte).pte_low &= ~_PAGE_RW; return pte; }
+static inline pte_t pte_clrhuge(pte_t pte)	{ (pte).pte_low &= ~_PAGE_PSE; return pte; }
 static inline pte_t pte_mkread(pte_t pte)	{ (pte).pte_low |= _PAGE_USER; return pte; }
 static inline pte_t pte_mkexec(pte_t pte)	{ (pte).pte_low |= _PAGE_USER; return pte; }
 static inline pte_t pte_mkdirty(pte_t pte)	{ (pte).pte_low |= _PAGE_DIRTY; return pte; }
@@ -242,6 +243,8 @@ static inline pte_t pte_mkyoung(pte_t pt
 static inline pte_t pte_mkwrite(pte_t pte)	{ (pte).pte_low |= _PAGE_RW; return pte; }
 static inline pte_t pte_mkhuge(pte_t pte)	{ (pte).pte_low |= _PAGE_PSE; return pte; }
 
+#define pte_pgprot(pte) (__pgprot(pte_val(pte) & ~PHYSICAL_PAGE_MASK))
+
 #ifdef CONFIG_X86_PAE
 # include <asm/pgtable-3level.h>
 #else
include/asm-x86_64/page.h +1 -1
@@ -97,7 +97,7 @@ extern unsigned long phys_base;
 #define PAGE_ALIGN(addr)	(((addr)+PAGE_SIZE-1)&PAGE_MASK)
 
 /* See Documentation/x86_64/mm.txt for a description of the memory map. */
-#define __PHYSICAL_MASK_SHIFT	46
+#define __PHYSICAL_MASK_SHIFT	52
 #define __PHYSICAL_MASK		((_AC(1,UL) << __PHYSICAL_MASK_SHIFT) - 1)
 #define __VIRTUAL_MASK_SHIFT	48
 #define __VIRTUAL_MASK		((_AC(1,UL) << __VIRTUAL_MASK_SHIFT) - 1)
include/asm-x86_64/pgtable.h +2 -2
@@ -356,9 +356,9 @@ static inline int pmd_large(pmd_t pte) {
 #define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot)))
 #define pmd_pfn(x)  ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT)
 
-#define pte_to_pgoff(pte) ((pte_val(pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
+#define pte_to_pgoff(pte) (pte_val(pte) >> PAGE_SHIFT)
 #define pgoff_to_pte(off) ((pte_t) { ((off) << PAGE_SHIFT) | _PAGE_FILE })
-#define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT
+#define PTE_FILE_MAX_BITS (64 - PAGE_SHIFT)
 
 /* PTE - Level 1 access. */
 
include/asm-x86_64/system.h +1 -1
@@ -109,7 +109,7 @@ static inline void write_cr4(unsigned lo
 #define stts() write_cr0(8 | read_cr0())
 
 #define wbinvd() \
-	__asm__ __volatile__ ("wbinvd": : :"memory");
+	__asm__ __volatile__ ("wbinvd": : :"memory")
 
 /*
  * On SMP systems, when the scheduler does migration-cost autodetection,