Skip to contentSkip to conversation

improve x86 page table handling performance

PATCH In review Jan Beulich 2 participants 3 messages Mar 26, 2007 – Mar 27, 2007

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 improve x86 page table handling performance

Where possible,
- use hypercalls instead of writing to read-only pages
- fold TLB flushes into page table update hypercalls
- on PAE, use single-access updates instead of two-access ones

The single change to pte_clear() yields a 25-30% boost for kernel builds on a 4x2x2 CPUs, 8Gb box; the other changes together yield improvements of 2-5%.

Also, adjust backward compatibility handling in a few more places.

include/asm-i386/mach-xen/asm/pgtable.h +4 -0
Index: head-2007-03-19/include/asm-i386/mach-xen/asm/pgtable.h
===================================================================
@@ -210,9 +210,13 @@ extern unsigned long pg0[];
 
 /* To avoid harmful races, pmd_none(x) should check only the lower when PAE */
 #define pmd_none(x)	(!(unsigned long)pmd_val(x))
+#ifdef CONFIG_XEN_COMPAT_030002
 /* pmd_present doesn't just test the _PAGE_PRESENT bit since wr.p.t.
    can temporarily clear it. */
 #define pmd_present(x)	(pmd_val(x))
+#else
+#define pmd_present(x)	(pmd_val(x) & _PAGE_PRESENT)
+#endif
 #define pmd_bad(x)	((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER & ~_PAGE_PRESENT)) != (_KERNPG_TABLE &

~_PAGE_PRESENT))

@@ -252,36 +256,47 @@ static inline pte_t pte_mkhuge(pte_t pte

 # include <asm/pgtable-2level.h>
 #endif

-static inline int ptep_test_and_clear_dirty(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) -{
- if (!pte_dirty(*ptep))
- return 0;
- return test_and_clear_bit(_PAGE_BIT_DIRTY, &ptep->pte_low);
-}
+#define ptep_test_and_clear_dirty(vma, addr, ptep) \
+({ \
+ pte_t __pte = *(ptep); \
+ int __ret = pte_dirty(__pte); \
+ if (__ret) { \
+ __pte = pte_mkclean(__pte); \
+ if ((vma)->vm_mm != current->mm || \
+ HYPERVISOR_update_va_mapping(addr, __pte, 0)) \
+ (ptep)->pte_low = __pte.pte_low; \
+ } \
+ __ret; \
+})

-static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) -{
- if (!pte_young(*ptep))
- return 0;
- return test_and_clear_bit(_PAGE_BIT_ACCESSED, &ptep->pte_low);
-}
+#define ptep_test_and_clear_young(vma, addr, ptep) \
+({ \
+ pte_t __pte = *(ptep); \
+ int __ret = pte_young(__pte); \
+ if (__ret) \
+ __pte = pte_mkold(__pte); \
+ if ((vma)->vm_mm != current->mm || \
+ HYPERVISOR_update_va_mapping(addr, __pte, 0)) \
+ (ptep)->pte_low = __pte.pte_low; \
+ __ret; \
+})

-static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) -{
- pte_t pte;
- if (full) {
- pte = *ptep;
- pte_clear(mm, addr, ptep);
- } else {
- pte = ptep_get_and_clear(mm, addr, ptep);
- }
- return pte;
-}
+#define ptep_get_and_clear_full(mm, addr, ptep, full) \
+ ((full) ? ({ \
+ pte_t __res = *(ptep); \
+ if (test_bit(PG_pinned, &virt_to_page((mm)->pgd)->flags)) \
+ xen_l1_entry_update(ptep, __pte(0)); \
+ else \
+ *(ptep) = __pte(0); \
+ __res; \
+ }) : \
+ ptep_get_and_clear(mm, addr, ptep))

static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) {
- if (pte_write(*ptep))
- clear_bit(_PAGE_BIT_RW, &ptep->pte_low);
+ pte_t pte = *ptep;
+ if (pte_write(pte))
+ set_pte_at(mm, addr, ptep, pte_wrprotect(pte));
}

/*
@@ -418,6 +433,20 @@ extern void noexec_setup(const char *str
#define pte_unmap_nested(pte) do { } while (0) #endif

+#define __HAVE_ARCH_PTEP_ESTABLISH
+#define ptep_establish(vma, address, ptep, pteval) \
+ do { \
+ if ( likely((vma)->vm_mm == current->mm) ) { \
+ BUG_ON(HYPERVISOR_update_va_mapping(address, \
+ pteval, \
+ (unsigned long)(vma)->vm_mm->cpu_vm_mask.bits| \
+ UVMF_INVLPG|UVMF_MULTI)); \
+ } else { \
+ xen_l1_entry_update(ptep, pteval); \
+ flush_tlb_page(vma, address); \
+ } \
+ } while (0)
+
/*
* The i386 doesn't have any external MMU info: the kernel page
* tables contain all the necessary information.
@@ -430,27 +459,12 @@ extern void noexec_setup(const char *str
*/
#define update_mmu_cache(vma,address,pte) do { } while (0) #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
-#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
- do { \
- if (__dirty) { \
- if ( likely((__vma)->vm_mm == current->mm) ) { \
- BUG_ON(HYPERVISOR_update_va_mapping(__address, \
- __entry, \
- (unsigned long)(__vma)->vm_mm->cpu_vm_mask.bits| \
- UVMF_INVLPG|UVMF_MULTI)); \
- } else { \
- xen_l1_entry_update(__ptep, __entry); \
- flush_tlb_page(__vma, __address); \
- } \
- } \
+#define ptep_set_access_flags(vma, address, ptep, entry, dirty) \
+ do { \
+ if (dirty) \
+ ptep_establish(vma, address, ptep, entry); \
} while (0)

-#define __HAVE_ARCH_PTEP_ESTABLISH
-#define ptep_establish(__vma, __address, __ptep, __entry) \
-do { \
- ptep_set_access_flags(__vma, __address, __ptep, __entry, 1); \
-} while (0)
-
#include <xen/features.h>
void make_lowmem_page_readonly(void *va, unsigned int feature); void make_lowmem_page_writable(void *va, unsigned int feature);
@@ -508,6 +522,7 @@ direct_remap_pfn_range(vma,from,pfn,size
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
+#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
#define __HAVE_ARCH_PTE_SAME
#include <asm-generic/pgtable.h>

include/asm-i386/mach-xen/asm/pgtable-2level.h +30 -2
Index: head-2007-03-19/include/asm-i386/mach-xen/asm/pgtable-2level.h
===================================================================
@@ -36,8 +36,37 @@
 #define pte_clear(mm,addr,xp)	do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)
 #define pmd_clear(xp)	do { set_pmd(xp, __pmd(0)); } while (0)
 
-#define ptep_get_and_clear(mm,addr,xp)	__pte_ma(xchg(&(xp)->pte_low, 0))
+#define pte_none(x) (!(x).pte_low)
+
+static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+{
+	pte_t pte = *ptep;
+	if (!pte_none(pte)) {
+		if (mm != &init_mm)
+			pte = __pte_ma(xchg(&ptep->pte_low, 0));
+		else
+			HYPERVISOR_update_va_mapping(addr, __pte(0), 0);
+	}
+	return pte;
+}
+
+#define ptep_clear_flush(vma, addr, ptep)			\
+({								\
+	pte_t *__ptep = (ptep);					\
+	pte_t __res = *__ptep;					\
+	if (!pte_none(__res) &&					\
+	    ((vma)->vm_mm != current->mm ||			\
+	     HYPERVISOR_update_va_mapping(addr, __pte(0),	\
+			(unsigned long)(vma)->vm_mm->cpu_vm_mask.bits| \
+				UVMF_INVLPG|UVMF_MULTI))) {	\
+		__ptep->pte_low = 0;				\
+		flush_tlb_page(vma, addr);			\
+	}							\
+	__res;							\
+})
+
 #define pte_same(a, b)		((a).pte_low == (b).pte_low)
+
 #define __pte_mfn(_pte) ((_pte).pte_low >> PAGE_SHIFT)
 #define pte_mfn(_pte) ((_pte).pte_low & _PAGE_PRESENT ? \
 	__pte_mfn(_pte) : pfn_to_mfn(__pte_mfn(_pte)))
@@ -46,7 +75,6 @@
 
 #define pte_page(_pte) pfn_to_page(pte_pfn(_pte))
 
-#define pte_none(x)		(!(x).pte_low)
 #define pfn_pte(pfn, prot)	__pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot))
 #define pfn_pmd(pfn, prot)	__pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot))
 
include/asm-i386/mach-xen/asm/pgtable-3level.h +43 -17

84 lines not on this page · raw message

include/asm-x86_64/mach-xen/asm/pgtable.h +88 -73

229 lines not on this page · raw message

#1 lore raw
Quoted 9 lines
Quoted 1 line
Quoted 1 line

"Jan Beulich" <[email protected]> 26.03.07 16:50 >>>

Where possible,
- use hypercalls instead of writing to read-only pages
- fold TLB flushes into page table update hypercalls
- on PAE, use single-access updates instead of two-access ones

The single change to pte_clear() yields a 25-30% boost for kernel builds on a 4x2x2 CPUs, 8Gb box; the other changes together yield improvements of 2-5%.

I shall clarify that this is for the PAE case only (i.e. resulting from the folding of two page faults into a single hypercall, which is being used in the kmap path).

Jan

#2 lore raw
Quoted 7 lines · Jan Beulich
Quoted 3 lines

The single change to pte_clear() yields a 25-30% boost for kernel builds on a 4x2x2 CPUs, 8Gb box; the other changes together yield improvements of 2-5%.

I shall clarify that this is for the PAE case only (i.e. resulting from the folding of two page faults into a single hypercall, which is being used in the kmap path).

Spotted that. The patch is pretty big but I think probably all good. I've been away for a week so catching up on patches...

-- Keir