Skip to contentSkip to conversation

Implement faster superpage mapping

PATCH In review Dave McCracken 2 participants 2 messages May 13, 2010 – May 14, 2010

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Implement faster superpage mapping

Here's my first cut of a faster superpage mapping scheme. It uses a separate superpage table to track mappings of superpages and mappings that conflict with using a superpage.

One new feature of this code is that it requires that every superpage be allocated to a domain with a single call. This ensures that every page in the superpage is allocated to the same domain.

Dave McCracken
Oracle Corp.

/xen/common/page_alloc.c +50 -0
@@ -1083,6 +1083,50 @@ void init_domheap_pages(paddr_t ps, padd
     init_heap_pages(mfn_to_page(smfn), emfn - smfn);
 }
 
+static void enable_superpage(
+    struct page_info *pg,
+    unsigned int order)
+{
+    struct spage_info *spage;
+    int i;
+
+    spage = page_to_spage(pg);
+    if (order < SUPERPAGE_ORDER)
+    {
+        test_and_clear_bit(_SGT_enabled, &spage->type_info);
+        return;
+    }        
+    if (order == SUPERPAGE_ORDER)
+    {
+        test_and_set_bit(_SGT_enabled, &spage->type_info);
+        return;
+    }
+    order -= SUPERPAGE_ORDER;
+    for(i = 0; i < (1 << order); i++)
+        test_and_set_bit(_SGT_enabled, &spage[i].type_info);
+}
+
+static void disable_superpage(
+    struct page_info *pg,
+    unsigned int order)
+{
+    struct spage_info *spage;
+    int i;
+
+    spage = page_to_spage(pg);
+    test_and_clear_bit(_SGT_enabled, &spage->type_info);
+
+    if (order > SUPERPAGE_ORDER)
+    {
+        order -= SUPERPAGE_ORDER;
+        for(i = 1; i < (1 << order); i++)
+        {
+            BUG_ON((spage[i].type_info & SGT_count_mask) != 0);
+            test_and_clear_bit(_SGT_enabled, &spage[i].type_info);
+        }
+    }
+    
+}
 
 int assign_pages(
     struct domain *d,
@@ -1128,6 +1172,9 @@ int assign_pages(
         page_list_add_tail(&pg[i], &d->page_list);
     }
 
+    if (opt_allow_superpage)
+        enable_superpage(pg, order);
+
     spin_unlock(&d->page_alloc_lock);
     return 0;
 
@@ -1201,6 +1248,9 @@ void free_domheap_pages(struct page_info
             page_list_del2(&pg[i], &d->page_list, &d->arch.relmem_list);
         }
 
+        if (opt_allow_superpage)
+            disable_superpage(pg, order);
+
         d->tot_pages -= 1 << order;
         drop_dom_ref = (d->tot_pages == 0);
 
/xen/include/asm-x86/mm.h +31 -1
@@ -214,6 +214,33 @@ struct page_info
 #define PGC_count_width   PG_shift(9)
 #define PGC_count_mask    ((1UL<<PGC_count_width)-1)
 
+struct spage_info
+{
+	unsigned long count_info;
+	unsigned long type_info;
+};
+
+ /* The following page types are MUTUALLY EXCLUSIVE. */
+#define SGT_none          PG_mask(0, 2)  /* superpage not in use */
+#define SGT_superpage     PG_mask(1, 2)  /* mapped as a superpage */
+#define SGT_nosuper       PG_mask(2, 2)  /* has mappings that conflict with superpage */
+#define SGT_type_mask     PG_mask(3, 2)  /* Bits 30-31 or 62-63. */
+
+/* Enabled is set when the entire superpage is allocated as a block to a domain */
+#define	_SGT_enabled      PG_shift(3)
+#define	SGT_enabled       PG_mask(1, 3)
+
+ /* Count of uses of this superpage as its current type. */
+#define SGT_count_width   PG_shift(3)
+#define SGT_count_mask    ((1UL<<SGT_count_width)-1)
+
+static inline int spage_conflicts(unsigned long type)
+{
+	if (type && (type != PGT_writable_page))
+		return 1;
+	return 0;
+}
+
 #if defined(__i386__)
 #define is_xen_heap_page(page) is_xen_heap_mfn(page_to_mfn(page))
 #define is_xen_heap_mfn(mfn) ({                         \
@@ -262,12 +289,15 @@ extern void share_xen_page_with_privileg
     struct page_info *page, int readonly);
 
 #define frame_table ((struct page_info *)FRAMETABLE_VIRT_START)
+#define spage_table ((struct spage_info *)SPAGETABLE_VIRT_START)
 extern unsigned long max_page;
 extern unsigned long total_pages;
 void init_frametable(void);
 
 #define PDX_GROUP_COUNT ((1 << L2_PAGETABLE_SHIFT) / \
                          (sizeof(*frame_table) & -sizeof(*frame_table)))
+#define SDX_GROUP_COUNT ((1 << L2_PAGETABLE_SHIFT) / \
+                         (sizeof(*spage_table) & -sizeof(*spage_table)))
 extern unsigned long pdx_group_valid[];
 
 /* Convert between Xen-heap virtual addresses and page-info structures. */
@@ -370,7 +400,7 @@ pae_copy_root(struct vcpu *v, l3_pgentry
 
 int check_descriptor(const struct domain *, struct desc_struct *d);
 
-extern int opt_allow_hugepage;
+extern int opt_allow_superpage;
 extern int mem_hotplug;
 
 /******************************************************************************
/xen/include/asm-x86/guest_pt.h +3 -2
@@ -186,10 +186,11 @@ guest_supports_superpages(struct vcpu *v
     /* The _PAGE_PSE bit must be honoured in HVM guests, whenever
      * CR4.PSE is set or the guest is in PAE or long mode. 
      * It's also used in the dummy PT for vcpus with CR4.PG cleared. */
-    return (is_hvm_vcpu(v) && 
+    return (opt_allow_superpage ||
+	    (is_hvm_vcpu(v) && 
             (GUEST_PAGING_LEVELS != 2 
              || !hvm_paging_enabled(v)
-             || (v->arch.hvm_vcpu.guest_cr[4] & X86_CR4_PSE)));
+             || (v->arch.hvm_vcpu.guest_cr[4] & X86_CR4_PSE))));
 }
 
 static inline int
/xen/include/asm-x86/x86_32/page.h +5 -0
@@ -6,6 +6,7 @@
 #define L2_PAGETABLE_SHIFT      21
 #define L3_PAGETABLE_SHIFT      30
 #define PAGE_SHIFT              L1_PAGETABLE_SHIFT
+#define SUPERPAGE_SHIFT         L2_PAGETABLE_SHIFT
 #define ROOT_PAGETABLE_SHIFT    L3_PAGETABLE_SHIFT
 
 #define PAGETABLE_ORDER         9
@@ -13,6 +14,7 @@
 #define L2_PAGETABLE_ENTRIES    (1<<PAGETABLE_ORDER)
 #define L3_PAGETABLE_ENTRIES    4
 #define ROOT_PAGETABLE_ENTRIES  L3_PAGETABLE_ENTRIES
+#define SUPERPAGE_ORDER         PAGETABLE_ORDER
 
 /*
  * Architecturally, physical addresses may be up to 52 bits. However, the
@@ -53,6 +55,9 @@
 #define virt_to_pdx(va)         virt_to_mfn(va)
 #define pdx_to_virt(pdx)        mfn_to_virt(pdx)
 
+#define pfn_to_sdx(pfn)         ((pfn)>>(SUPERPAGE_SHIFT-PAGE_SHIFT))
+#define sdx_to_pfn(sdx)         ((sdx)<<(SUPERPAGE_SHIFT-PAGE_SHIFT))
+
 static inline unsigned long __virt_to_maddr(unsigned long va)
 {
     ASSERT(va >= DIRECTMAP_VIRT_START && va < DIRECTMAP_VIRT_END);
/xen/include/asm-x86/config.h +5 -0
@@ -225,6 +225,11 @@ extern unsigned int video_mode, video_fl
 /* Slot 261: xen text, static data and bss (1GB). */
 #define XEN_VIRT_START          (HIRO_COMPAT_MPT_VIRT_END)
 #define XEN_VIRT_END            (XEN_VIRT_START + GB(1))
+/* Slot 261: superpage information array (40MB). */
+#define SPAGETABLE_VIRT_END     FRAMETABLE_VIRT_START
+#define SPAGETABLE_SIZE         ((DIRECTMAP_SIZE >> SUPERPAGE_SHIFT) * \
+                                 sizeof(struct spage_info))
+#define SPAGETABLE_VIRT_START   (SPAGETABLE_VIRT_END - SPAGETABLE_SIZE)
 /* Slot 261: page-frame information array (40GB). */
 #define FRAMETABLE_VIRT_END     DIRECTMAP_VIRT_START
 #define FRAMETABLE_SIZE         ((DIRECTMAP_SIZE >> PAGE_SHIFT) * \
/xen/include/asm-x86/page.h +9 -0
@@ -240,6 +240,13 @@ void copy_page_sse2(void *, const void *
 #define __pfn_to_paddr(pfn) ((paddr_t)(pfn) << PAGE_SHIFT)
 #define __paddr_to_pfn(pa)  ((unsigned long)((pa) >> PAGE_SHIFT))
 
+/* Convert between machine frame numbers and spage-info structures. */
+#define __mfn_to_spage(mfn)  (spage_table + pfn_to_sdx(mfn))
+#define __spage_to_mfn(pg)   sdx_to_pfn((unsigned long)((pg) - spage_table))
+
+/* Convert between page-info structures and spage-info structures. */
+#define page_to_spage(page)  (spage_table+(((page)-frame_table)>>(SUPERPAGE_SHIFT-PAGE_SHIFT)))
+
 /*
  * We define non-underscored wrappers for above conversion functions. These are
  * overridden in various source files while underscored versions remain intact.
@@ -251,6 +258,8 @@ void copy_page_sse2(void *, const void *
 #define maddr_to_virt(ma)   __maddr_to_virt((unsigned long)(ma))
 #define mfn_to_page(mfn)    __mfn_to_page(mfn)
 #define page_to_mfn(pg)     __page_to_mfn(pg)
+#define mfn_to_spage(mfn)    __mfn_to_spage(mfn)
+#define spage_to_mfn(pg)     __spage_to_mfn(pg)
 #define maddr_to_page(ma)   __maddr_to_page(ma)
 #define page_to_maddr(pg)   __page_to_maddr(pg)
 #define virt_to_page(va)    __virt_to_page(va)
/xen/include/asm-x86/x86_64/page.h +14 -0
@@ -7,6 +7,7 @@
 #define L3_PAGETABLE_SHIFT      30
 #define L4_PAGETABLE_SHIFT      39
 #define PAGE_SHIFT              L1_PAGETABLE_SHIFT
+#define SUPERPAGE_SHIFT         L2_PAGETABLE_SHIFT
 #define ROOT_PAGETABLE_SHIFT    L4_PAGETABLE_SHIFT
 
 #define PAGETABLE_ORDER         9
@@ -15,6 +16,7 @@
 #define L3_PAGETABLE_ENTRIES    (1<<PAGETABLE_ORDER)
 #define L4_PAGETABLE_ENTRIES    (1<<PAGETABLE_ORDER)
 #define ROOT_PAGETABLE_ENTRIES  L4_PAGETABLE_ENTRIES
+#define SUPERPAGE_ORDER         PAGETABLE_ORDER
 
 #define __PAGE_OFFSET           DIRECTMAP_VIRT_START
 #define __XEN_VIRT_START        XEN_VIRT_START
@@ -41,6 +43,8 @@ extern void pfn_pdx_hole_setup(unsigned
 
 #define page_to_pdx(pg)  ((pg) - frame_table)
 #define pdx_to_page(pdx) (frame_table + (pdx))
+#define spage_to_pdx(spg) ((spg>>(SUPERPAGE_SHIFT-PAGE_SHIFT)) - spage_table)
+#define pdx_to_spage(pdx) (spage_table + ((pdx)<<(SUPERPAGE_SHIFT-PAGE_SHIFT)))
 /*
  * Note: These are solely for the use by page_{get,set}_owner(), and
  *       therefore don't need to handle the XEN_VIRT_{START,END} range.
@@ -64,6 +68,16 @@ static inline unsigned long pdx_to_pfn(u
            ((pdx << pfn_pdx_hole_shift) & pfn_top_mask);
 }
 
+static inline unsigned long pfn_to_sdx(unsigned long pfn)
+{
+    return pfn_to_pdx(pfn) >> (SUPERPAGE_SHIFT-PAGE_SHIFT);
+}
+
+static inline unsigned long sdx_to_pfn(unsigned long sdx)
+{
+    return pdx_to_pfn(sdx << (SUPERPAGE_SHIFT-PAGE_SHIFT));
+}
+
 static inline unsigned long __virt_to_maddr(unsigned long va)
 {
     ASSERT(va >= XEN_VIRT_START);
/xen/arch/x86/mm.c +150 -38

263 lines not on this page · raw message

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#1 lore raw
Quoted 7 lines · Dave McCracken

Here's my first cut of a faster superpage mapping scheme. It uses a separate superpage table to track mappings of superpages and mappings that conflict with using a superpage.

One new feature of this code is that it requires that every superpage be allocated to a domain with a single call. This ensures that every page in the superpage is allocated to the same domain.

Hm, not sure about that restriction but in any case it's not the weakest aspect of the patch by far anyway...

/xen/common/page_alloc.c void init_domheap_pages(paddr_t ps, padd · line 1098

Quoted 13 lines
/xen/common/page_alloc.c +13 -0
+static void enable_superpage(
+    struct page_info *pg,
+    unsigned int order)
+{
+    struct spage_info *spage;
+    int i;
+
+    spage = page_to_spage(pg);
+    if (order < SUPERPAGE_ORDER)
+    {
+        test_and_clear_bit(_SGT_enabled, &spage->type_info);
+        return;
+    }        

Why do you need this case? If this superpage is freshly allocated then it was not previously SGT_enabled, and the flag should be already clear?

/xen/common/page_alloc.c void init_domheap_pages(paddr_t ps, padd · line 1103

Quoted 5 lines
/xen/common/page_alloc.c +5 -0
+    if (order == SUPERPAGE_ORDER)
+    {
+        test_and_set_bit(_SGT_enabled, &spage->type_info);
+        return;
+    }

This case is completely redundant. The for loop would handle it fine.

/xen/common/page_alloc.c void init_domheap_pages(paddr_t ps, padd · line 1107

Quoted 4 lines
/xen/common/page_alloc.c +4 -0
+    order -= SUPERPAGE_ORDER;
+    for(i = 0; i < (1 << order); i++)
+        test_and_set_bit(_SGT_enabled, &spage[i].type_info);
+}

And why test_and_*() everywhere? You don't use the result of the test.

/xen/common/page_alloc.c void init_domheap_pages(paddr_t ps, padd · line 1117

Quoted 9 lines
/xen/common/page_alloc.c +9 -0
+static void disable_superpage(
+    struct page_info *pg,
+    unsigned int order)
+{
+    struct spage_info *spage;
+    int i;
+
+    spage = page_to_spage(pg);
+    test_and_clear_bit(_SGT_enabled, &spage->type_info);

Here we are at the crux of the matter. If a guest frees a page you just clear the SGT_enabled bit. So the superpage count_info is a completely pointless creation which is checked absolutely nowhere. The pages can get reused while stale superpage mappings hang around. Safety off.

/xen/common/page_alloc.c void init_domheap_pages(paddr_t ps, padd · line 1124

Quoted 6 lines
/xen/common/page_alloc.c +6 -0
+    if (order > SUPERPAGE_ORDER)
+    {
+        order -= SUPERPAGE_ORDER;
+        for(i = 1; i < (1 << order); i++)
+        {
+            BUG_ON((spage[i].type_info & SGT_count_mask) != 0);

Well, here you check the type count. The general count (count_info) would be more appropriate. Either way, crashing Xen is very obviously unacceptable.

AFAICS, your attempt to be clever on the handling of type conflicts and avoid ever needing to loop over all the pages in a superpage is so far a failure. I don't see how you'll successfully avoid it for count_info (page lifetime) handling, and in that case it may be simpler just to hold both count_info *and* type_info counts on every page when super_page's associated counts become non-zero.

I also noted you added another boot parameter. Just pick one name and stick with it.

-- Keir

Quoted 9 lines
/xen/common/page_alloc.c +4 -0
+            test_and_clear_bit(_SGT_enabled, &spage[i].type_info);
+        }
+    }   
+}

Dave McCracken
Oracle Corp.