Skip to contentSkip to conversation

xenpaging changes for xen-unstable

PATCH In review Olaf Hering 3 participants 30 messages Nov 2, 2010 – Nov 10, 2010

lore
Share & tools
lore
#0 lore raw

cover xenpaging changes for xen-unstable

Here are some changes for xenpaging in xen-unstable.

Its just for review and comments.

In a previous series for 4.0 I had a retry loop for the XENMEM_* commands. This could be done in a different way to prevent changes to existing kernel drivers in the guest. The changes to guest_physmap_add_entry are not needed with the update for machine_to_phys_mapping handling.
But the patch to handle guest_remove_page does not work as expected, the guest will hang. Any help to get this fixed without retry loops is appreciated.

Olaf

#1 lore raw

PATCH 1/16 xenpaging: whitespace fixes after addition of __get_paged_frame

Cleanup whitespace which was introduced in patch that added __get_paged_frame. Convert tabs to spaces to follow the coding style in this file.

---
 xen/common/grant_table.c |   26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)
xen/common/grant_table.c +13 -13
@@ -147,13 +147,13 @@ static int __get_paged_frame(unsigned lo
     struct p2m_domain *p2m;
     p2m_type_t p2mt;
     mfn_t mfn;
-    
+
     p2m = p2m_get_hostp2m(rd);
     if ( readonly )
         mfn = gfn_to_mfn(p2m, gfn, &p2mt);
     else
         mfn = gfn_to_mfn_unshare(p2m, gfn, &p2mt, 1);
-    
+
     if ( p2m_is_valid(p2mt) ) {
         *frame = mfn_x(mfn);
         if ( p2m_is_paged(p2mt) )
@@ -167,7 +167,7 @@ static int __get_paged_frame(unsigned lo
 #else
     *frame = readonly ? gmfn_to_mfn(rd, gfn) : gfn_to_mfn_private(rd, gfn);
 #endif
-    
+
     return rc;
 }
 
@@ -565,7 +565,7 @@ __gnttab_map_grant_ref(
             gfn = sha1 ? sha1->frame : sha2->full_page.frame;
             rc = __get_paged_frame(gfn, &frame, !!(op->flags & GNTMAP_readonly), rd);
             if ( rc != GNTST_okay )
-	         goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             act->domid = ld->domain_id;
             act->frame = frame;
@@ -1852,7 +1852,7 @@ __acquire_grant_for_copy(
             gfn = sha1->frame;
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
-		goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             is_sub_page = 0;
             trans_page_off = 0;
@@ -1864,7 +1864,7 @@ __acquire_grant_for_copy(
             gfn = sha2->full_page.frame;
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
-		    goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             is_sub_page = 0;
             trans_page_off = 0;
@@ -1876,7 +1876,7 @@ __acquire_grant_for_copy(
             gfn = sha2->sub_page.frame;
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
-		    goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             is_sub_page = 1;
             trans_page_off = sha2->sub_page.page_off;
@@ -1973,9 +1973,9 @@ __gnttab_copy(
     else
     {
 #ifdef CONFIG_X86
-	rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
-	if ( rc != GNTST_okay )
-		goto error_out;
+        rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
+        if ( rc != GNTST_okay )
+            goto error_out;
 #else
         s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);        
 #endif
@@ -2012,9 +2012,9 @@ __gnttab_copy(
     else
     {
 #ifdef CONFIG_X86
-	rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
-	if ( rc != GNTST_okay )
-		goto error_out;
+        rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
+        if ( rc != GNTST_okay )
+            goto error_out;
 #else
         d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
 #endif
#3 lore raw

PATCH 2/16 xenpaging: break endless loop during inital page-out with large pagefiles

In reply to Olaf Hering

To allow the starting for xenpaging right after 'xm start XYZ', I specified a pagefile size equal to the guest memory size in the hope to catch more errors where the paged-out state of a p2mt is not checked.

While doing that, xenpaging got into an endless loop because some pages cant be paged out right away. Now the policy reports an error if the gfn number wraps.

---
 tools/xenpaging/policy_default.c |   35 ++++++++++++++++++++++++++++-------
 tools/xenpaging/xenpaging.c      |    7 +++++--
 2 files changed, 33 insertions(+), 9 deletions(-)
tools/xenpaging/policy_default.c +28 -7
@@ -30,8 +30,12 @@
 
 
 static unsigned long mru[MRU_SIZE];
-static unsigned int i_mru = 0;
+static unsigned int i_mru;
 static unsigned long *bitmap;
+static unsigned long *unconsumed;
+static unsigned long current_gfn;
+static unsigned long bitmap_size;
+static unsigned long max_pages;
 
 
 int policy_init(xenpaging_t *paging)
@@ -43,6 +47,14 @@ int policy_init(xenpaging_t *paging)
     rc = alloc_bitmap(&bitmap, paging->bitmap_size);
     if ( rc != 0 )
         goto out;
+    /* Allocate bitmap to track unusable pages */
+    rc = alloc_bitmap(&unconsumed, paging->bitmap_size);
+    if ( rc != 0 )
+        goto out;
+
+    /* record bitmap_size */
+    bitmap_size = paging->bitmap_size;
+    max_pages = paging->domain_info->max_pages;
 
     /* Initialise MRU list of paged in pages */
     for ( i = 0; i < MRU_SIZE; i++ )
@@ -51,8 +63,6 @@ int policy_init(xenpaging_t *paging)
     /* Don't page out page 0 */
     set_bit(0, bitmap);
 
-    rc = 0;
-
  out:
     return rc;
 }
@@ -61,17 +71,27 @@ int policy_choose_victim(xc_interface *x
                          xenpaging_t *paging, domid_t domain_id,
                          xenpaging_victim_t *victim)
 {
+    unsigned long wrap = current_gfn;
     ASSERT(victim != NULL);
 
     /* Domain to pick on */
     victim->domain_id = domain_id;
-    
+
     do
     {
-        /* Randomly choose a gfn to evict */
-        victim->gfn = rand() % paging->domain_info->max_pages;
+        current_gfn++;
+        if ( current_gfn >= max_pages )
+            current_gfn = 0;
+        if ( wrap == current_gfn )
+        {
+            victim->gfn = INVALID_MFN;
+            return -ENOSPC;
+        }
     }
-    while ( test_bit(victim->gfn, bitmap) );
+    while ( test_bit(current_gfn, bitmap) || test_bit(current_gfn, unconsumed) );
+
+    set_bit(current_gfn, unconsumed);
+    victim->gfn = current_gfn;
 
     return 0;
 }
@@ -79,6 +99,7 @@ int policy_choose_victim(xc_interface *x
 void policy_notify_paged_out(domid_t domain_id, unsigned long gfn)
 {
     set_bit(gfn, bitmap);
+    clear_bit(gfn, unconsumed);
 }
 
 void policy_notify_paged_in(domid_t domain_id, unsigned long gfn)
tools/xenpaging/xenpaging.c +5 -2
@@ -446,7 +446,8 @@ static int evict_victim(xc_interface *xc
         ret = policy_choose_victim(xch, paging, domain_id, victim);
         if ( ret != 0 )
         {
-            ERROR("Error choosing victim");
+            if ( ret != -ENOSPC )
+                ERROR("Error choosing victim");
             goto out;
         }
 
@@ -525,7 +526,9 @@ int main(int argc, char *argv[])
     memset(victims, 0, sizeof(xenpaging_victim_t) * num_pages);
     for ( i = 0; i < num_pages; i++ )
     {
-        evict_victim(xch, paging, domain_id, &victims[i], fd, i);
+        rc = evict_victim(xch, paging, domain_id, &victims[i], fd, i);
+        if ( rc == -ENOSPC )
+            break;
         if ( i % 100 == 0 )
             DPRINTF("%d pages evicted\n", i);
     }
#4 lore raw

PATCH 3/16 xenpaging: Open paging file only if xenpaging_init() succeeds

In reply to Olaf Hering

Open paging file only if xenpaging_init() succeeds. It can fail if the host does not support the required virtualization features such as EPT or if xenpaging was already started for this domain_id.

---
 tools/xenpaging/xenpaging.c |   18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)
tools/xenpaging/xenpaging.c +9 -9
@@ -502,15 +502,6 @@ int main(int argc, char *argv[])
 
     victims = calloc(num_pages, sizeof(xenpaging_victim_t));
 
-    /* Open file */
-    sprintf(filename, "page_cache_%d", domain_id);
-    fd = open(filename, open_flags, open_mode);
-    if ( fd < 0 )
-    {
-        perror("failed to open file");
-        return -1;
-    }
-
     /* Seed random-number generator */
     srand(time(NULL));
 
@@ -522,6 +513,15 @@ int main(int argc, char *argv[])
         goto out;
     }
 
+    /* Open file */
+    sprintf(filename, "page_cache_%d", domain_id);
+    fd = open(filename, open_flags, open_mode);
+    if ( fd < 0 )
+    {
+        perror("failed to open file");
+        return -1;
+    }
+
     /* Evict pages */
     memset(victims, 0, sizeof(xenpaging_victim_t) * num_pages);
     for ( i = 0; i < num_pages; i++ )
#5 lore raw

PATCH 4/16 xenpaging: allow only one xenpaging binary per guest

In reply to Olaf Hering

Make sure only one xenpaging binary is active per domain. Print info when the host lacks the required features for xenpaging.

---
v2: use perror for default case

 tools/xenpaging/xenpaging.c |   12 +++++++++++-
 xen/arch/x86/mm/mem_event.c |    7 +++++++
 2 files changed, 18 insertions(+), 1 deletion(-)
tools/xenpaging/xenpaging.c +11 -1
@@ -123,7 +123,17 @@ xenpaging_t *xenpaging_init(xc_interface
                              paging->mem_event.ring_page);
     if ( rc != 0 )
     {
-        ERROR("Error initialising shared page");
+        switch ( errno ) {
+            case EBUSY:
+                ERROR("xenpaging is (or was) active on this domain");
+                break;
+            case ENODEV:
+                ERROR("EPT not supported for this guest");
+                break;
+            default:
+                perror("Error initialising shared page");
+                break;
+        }
         goto err;
     }
 
xen/arch/x86/mm/mem_event.c +7 -0
@@ -214,6 +214,13 @@ int mem_event_domctl(struct domain *d, x
             mfn_t ring_mfn;
             mfn_t shared_mfn;
 
+            /* Only one xenpaging at a time. If xenpaging crashed,
+             * the cache is in an undefined state and so is the guest
+             */
+            rc = -EBUSY;
+            if ( d->mem_event.ring_page )
+                break;
+
             /* Currently only EPT is supported */
             rc = -ENODEV;
             if ( !(hap_enabled(d) &&
#6 lore raw

PATCH 5/16 xenpaging: update machine_to_phys_mapping during page-in and page-out

In reply to Olaf Hering

The machine_to_phys_mapping array needs updating during page-out. If a page is gone, a call to get_gpfn_from_mfn will still return the old gfn for an already paged-out page. This happens when the entire guest ram is paged-out before xen_vga_populate_vram() runs. Then XENMEM_populate_physmap is called with gfn 0xff000. A new page is allocated with alloc_domheap_pages. This new page does not have a gfn yet. However, in guest_physmap_add_entry() the passed mfn maps still to an old gfn. This old gfn is paged-out and has no mfn anymore. As a result, the ASSERT() triggers because p2m_is_ram() is true for p2m_ram_paging* types.

If the machine_to_phys_mapping array is updated properly, both loops in guest_physmap_add_entry() turn into no-ops for the new page and the mfn/gfn mapping will be done at the end of the function.

The same thing needs to happen dring a page-in.
If XENMEM_add_to_physmap is used with XENMAPSPACE_gmfn, get_gpfn_from_mfn() will return an appearently valid gfn. As a result, guest_physmap_remove_page() is called. The ASSERT in p2m_remove_page triggers because the passed mfn does not match the old mfn for the passed gfn.

---
 xen/arch/x86/mm/p2m.c |    2 ++
 1 file changed, 2 insertions(+)
xen/arch/x86/mm/p2m.c +2 -0
@@ -2742,6 +2742,7 @@ int p2m_mem_paging_evict(struct p2m_doma
     /* Remove mapping from p2m table */
     p2m_lock(p2m);
     set_p2m_entry(p2m, gfn, _mfn(PAGING_MFN), 0, p2m_ram_paged);
+    set_gpfn_from_mfn(mfn_x(mfn), INVALID_M2P_ENTRY);
     p2m_unlock(p2m);
 
     /* Put the page back so it gets freed */
@@ -2820,6 +2821,7 @@ void p2m_mem_paging_resume(struct p2m_do
     mfn = gfn_to_mfn(p2m, rsp.gfn, &p2mt);
     p2m_lock(p2m);
     set_p2m_entry(p2m, rsp.gfn, mfn, 0, p2m_ram_rw);
+    set_gpfn_from_mfn(mfn_x(mfn), rsp.gfn);
     p2m_unlock(p2m);
 
     /* Unpause domain */
#8 lore raw

PATCH 6/16 xenpaging: drop paged pages in guest_remove_page

In reply to Olaf Hering

Simply drop paged-pages in guest_remove_page(), and notify xenpaging to drop reference to the gfn.

This version does not work for some reasons.
The guest hangs after pages got dropped.

---
 tools/xenpaging/xenpaging.c    |   10 ++++++++
 xen/arch/x86/mm/p2m.c          |   51 +++++++++++++++++++++++++++++++++--------
 xen/common/memory.c            |    8 ++++++
 xen/include/asm-x86/p2m.h      |    4 +++
 xen/include/public/mem_event.h |    1 
 5 files changed, 65 insertions(+), 9 deletions(-)
tools/xenpaging/xenpaging.c +10 -0
@@ -586,6 +586,15 @@ int main(int argc, char *argv[])
                     goto out;
                 }
                 
+                if ( req.flags & MEM_EVENT_FLAG_DROP_PAGE )
+                {
+                    DPRINTF("Dropping page %"PRIx64" p2m %x\n", req.gfn, req.p2mt);
+
+                    /* Notify policy of page being dropped */
+                    policy_notify_paged_in(paging->mem_event.domain_id, req.gfn);
+                }
+                else
+                {
                 /* Populate the page */
                 rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
                 if ( rc != 0 )
@@ -606,6 +615,7 @@ int main(int argc, char *argv[])
                     ERROR("Error resuming page");
                     goto out;
                 }
+                }
 
                 /* Evict a new page to replace the one we just paged in */
                 evict_victim(xch, paging, domain_id, &victims[i], fd, i);
xen/arch/x86/mm/p2m.c +42 -9

101 lines not on this page · raw message

xen/common/memory.c +8 -0
@@ -162,6 +162,14 @@ int guest_remove_page(struct domain *d,
 
 #ifdef CONFIG_X86
     mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt)); 
+    if ( unlikely(p2m_is_paging(p2mt)) )
+    {
+        gdprintk(XENLOG_INFO, "Domain %u invalidate paged gfn %lx/%x\n",
+                d->domain_id, gmfn, p2mt);
+        guest_physmap_remove_page(d, gmfn, mfn, 0);
+        p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
+        return 1;
+    }
 #else
     mfn = gmfn_to_mfn(d, gmfn);
 #endif
xen/include/asm-x86/p2m.h +4 -0
@@ -480,6 +480,8 @@ int set_shared_p2m_entry(struct p2m_doma
 int p2m_mem_paging_nominate(struct p2m_domain *p2m, unsigned long gfn);
 /* Evict a frame */
 int p2m_mem_paging_evict(struct p2m_domain *p2m, unsigned long gfn);
+/* Tell xenpaging to drop a paged out frame */
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn);
 /* Start populating a paged out frame */
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn);
 /* Prepare the p2m for paging a frame in */
@@ -487,6 +489,8 @@ int p2m_mem_paging_prep(struct p2m_domai
 /* Resume normal operation (in case a domain was paused) */
 void p2m_mem_paging_resume(struct p2m_domain *p2m);
 #else
+static inline void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{ }
 static inline void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 { }
 #endif
xen/include/public/mem_event.h +1 -0
@@ -37,6 +37,7 @@
 #define MEM_EVENT_FLAG_VCPU_PAUSED  (1 << 0)
 #define MEM_EVENT_FLAG_DOM_PAUSED   (1 << 1)
 #define MEM_EVENT_FLAG_OUT_OF_MEM   (1 << 2)
+#define MEM_EVENT_FLAG_DROP_PAGE    (1 << 3)
 
 
 typedef struct mem_event_shared_page {
#10 lore raw

PATCH 7/16 xenpaging: populate only paged-out pages

In reply to Olaf Hering

populdate a paged-out page only once to reduce pressure in the ringbuffer. Several cpus may still request a page at once. xenpaging can handle this.

But: maybe this will miss pages in paging-out state?

---
 xen/arch/x86/hvm/emulate.c       |    3 ++-
 xen/arch/x86/hvm/hvm.c           |   15 ++++++++++-----
 xen/arch/x86/mm/guest_walk.c     |    3 ++-
 xen/arch/x86/mm/hap/guest_walk.c |    6 ++++--
 4 files changed, 18 insertions(+), 9 deletions(-)
xen/arch/x86/hvm/emulate.c +2 -1
@@ -66,7 +66,8 @@ static int hvmemul_do_io(
     ram_mfn = gfn_to_mfn_unshare(p2m, ram_gfn, &p2mt, 0);
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, ram_gfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, ram_gfn);
         return X86EMUL_RETRY;
     }
     if ( p2m_is_shared(p2mt) )
xen/arch/x86/hvm/hvm.c +10 -5
@@ -345,7 +345,8 @@ static int hvm_set_ioreq_page(
         return -EINVAL;
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, gmfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, gmfn);
         return -ENOENT;
     }
     if ( p2m_is_shared(p2mt) )
@@ -1369,7 +1370,8 @@ static void *__hvm_map_guest_frame(unsig
         return NULL;
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, gfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, gfn);
         return NULL;
     }
 
@@ -1811,7 +1813,8 @@ static enum hvm_copy_result __hvm_copy(
 
         if ( p2m_is_paging(p2mt) )
         {
-            p2m_mem_paging_populate(p2m, gfn);
+            if ( p2m_is_paged(p2mt) )
+                p2m_mem_paging_populate(p2m, gfn);
             return HVMCOPY_gfn_paged_out;
         }
         if ( p2m_is_shared(p2mt) )
@@ -3118,7 +3121,8 @@ long do_hvm_op(unsigned long op, XEN_GUE
             mfn_t mfn = gfn_to_mfn(p2m, pfn, &t);
             if ( p2m_is_paging(t) )
             {
-                p2m_mem_paging_populate(p2m, pfn);
+                if ( p2m_is_paged(t) )
+                    p2m_mem_paging_populate(p2m, pfn);
 
                 rc = -EINVAL;
                 goto param_fail3;
@@ -3184,7 +3188,8 @@ long do_hvm_op(unsigned long op, XEN_GUE
             mfn = gfn_to_mfn_unshare(p2m, pfn, &t, 0);
             if ( p2m_is_paging(t) )
             {
-                p2m_mem_paging_populate(p2m, pfn);
+                if ( p2m_is_paged(t) )
+                    p2m_mem_paging_populate(p2m, pfn);
 
                 rc = -EINVAL;
                 goto param_fail4;
xen/arch/x86/mm/guest_walk.c +2 -1
@@ -96,7 +96,8 @@ static inline void *map_domain_gfn(struc
     *mfn = gfn_to_mfn_unshare(p2m, gfn_x(gfn), p2mt, 0);
     if ( p2m_is_paging(*p2mt) )
     {
-        p2m_mem_paging_populate(p2m, gfn_x(gfn));
+        if ( p2m_is_paged(*p2mt) )
+            p2m_mem_paging_populate(p2m, gfn_x(gfn));
 
         *rc = _PAGE_PAGED;
         return NULL;
xen/arch/x86/mm/hap/guest_walk.c +4 -2
@@ -50,7 +50,8 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
     top_mfn = gfn_to_mfn_unshare(p2m, cr3 >> PAGE_SHIFT, &p2mt, 0);
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, cr3 >> PAGE_SHIFT);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, cr3 >> PAGE_SHIFT);
 
         pfec[0] = PFEC_page_paged;
         return INVALID_GFN;
@@ -82,7 +83,8 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
         gfn_to_mfn_unshare(p2m, gfn_x(gfn), &p2mt, 0);
         if ( p2m_is_paging(p2mt) )
         {
-            p2m_mem_paging_populate(p2m, gfn_x(gfn));
+            if ( p2m_is_paged(p2mt) )
+                p2m_mem_paging_populate(p2m, gfn_x(gfn));
 
             pfec[0] = PFEC_page_paged;
             return INVALID_GFN;
#14 lore raw

PATCH 8/16 xenpaging: reduce MINIMUM_RESTART_TIME

In reply to Olaf Hering

Allow runlevel 6 as default runlevel.

---
 tools/python/xen/xend/XendConstants.py |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
tools/python/xen/xend/XendConstants.py +1 -1
@@ -92,7 +92,7 @@ DOM_STATES_OLD = [
 SHUTDOWN_TIMEOUT = (60.0 * 5)
 
 """Minimum time between domain restarts in seconds."""
-MINIMUM_RESTART_TIME = 60
+MINIMUM_RESTART_TIME = 5
 
 RESTART_IN_PROGRESS = 'xend/restart_in_progress'
 DUMPCORE_IN_PROGRESS = 'xend/dumpcore_in_progress'
#20 lore raw

PATCH 9/16 xenpaging: start xenpaging via config option

In reply to Olaf Hering

Start xenpaging via config option.

TODO: add config option for pagefile directory
TODO: add libxl support
TODO: parse config values like 42K, 42M, 42G, 42%

---
 tools/examples/xmexample.hvm            |    3 +
 tools/python/README.XendConfig          |    1 
 tools/python/README.sxpcfg              |    1 
 tools/python/xen/xend/XendConfig.py     |    3 +
 tools/python/xen/xend/XendDomainInfo.py |    6 ++
 tools/python/xen/xend/image.py          |   87 ++++++++++++++++++++++++++++++++
 tools/python/xen/xm/create.py           |    5 +
 tools/python/xen/xm/xenapi_create.py    |    1 
 8 files changed, 107 insertions(+)
tools/examples/xmexample.hvm +3 -0
@@ -127,6 +127,9 @@ disk = [ 'file:/var/images/min-el3-i386.
 # Device Model to be used
 device_model = 'qemu-dm'
 
+# xenpaging, number of pages
+xenpaging = 42
+
 #-----------------------------------------------------------------------------
 # boot on floppy (a), hard disk (c), Network (n) or CD-ROM (d) 
 # default: hard disk, cd-rom, floppy
tools/python/README.XendConfig +1 -0
@@ -120,6 +120,7 @@ otherConfig
                                 image.vncdisplay
                                 image.vncunused
                                 image.hvm.device_model
+                                image.hvm.xenpaging
                                 image.hvm.display
                                 image.hvm.xauthority
                                 image.hvm.vncconsole
tools/python/README.sxpcfg +1 -0
@@ -51,6 +51,7 @@ image
   - vncunused
   (HVM)
   - device_model
+  - xenpaging
   - display
   - xauthority
   - vncconsole
tools/python/xen/xend/XendConfig.py +3 -0
@@ -147,6 +147,7 @@ XENAPI_PLATFORM_CFG_TYPES = {
     'apic': int,
     'boot': str,
     'device_model': str,
+    'xenpaging': int,
     'loader': str,
     'display' : str,
     'fda': str,
@@ -508,6 +509,8 @@ class XendConfig(dict):
             self['platform']['nomigrate'] = 0
 
         if self.is_hvm():
+            if 'xenpaging' not in self['platform']:
+                self['platform']['xenpaging'] = None
             if 'timer_mode' not in self['platform']:
                 self['platform']['timer_mode'] = 1
             if 'viridian' not in self['platform']:
tools/python/xen/xend/XendDomainInfo.py +6 -0
@@ -2390,6 +2390,7 @@ class XendDomainInfo:
 
         if self.image:
             self.image.createDeviceModel()
+            self.image.createXenPaging()
 
         #if have pass-through devs, need the virtual pci slots info from qemu
         self.pci_device_configure_boot()
@@ -2402,6 +2403,11 @@ class XendDomainInfo:
                 self.image.destroyDeviceModel()
             except Exception, e:
                 log.exception("Device model destroy failed %s" % str(e))
+            try:
+                log.debug("stopping xenpaging")
+                self.image.destroyXenPaging()
+            except Exception, e:
+                log.exception("stopping xenpaging failed %s" % str(e))
         else:
             log.debug("No device model")
 
tools/python/xen/xend/image.py +87 -0

105 lines not on this page · raw message

tools/python/xen/xm/create.py +5 -0
@@ -491,6 +491,10 @@ gopts.var('nfs_root', val="PATH",
           fn=set_value, default=None,
           use="Set the path of the root NFS directory.")
 
+gopts.var('xenpaging', val='NUM',
+          fn=set_int, default=None,
+          use="Number of pages to swap.")
+
 gopts.var('device_model', val='FILE',
           fn=set_value, default=None,
           use="Path to device model program.")
@@ -1076,6 +1080,7 @@ def configure_hvm(config_image, vals):
     args = [ 'acpi', 'apic',
              'boot',
              'cpuid', 'cpuid_check',
+             'xenpaging',
              'device_model', 'display',
              'fda', 'fdb',
              'gfx_passthru', 'guest_os_type',
tools/python/xen/xm/xenapi_create.py +1 -0
@@ -1085,6 +1085,7 @@ class sxp2xml:
             'acpi',
             'apic',
             'boot',
+            'xenpaging',
             'device_model',
             'loader',
             'fda',
#21 lore raw

PATCH 10/16 xenpaging: add signal handling

In reply to Olaf Hering

Leave paging loop if xenpaging gets a signal.
Remove paging file on exit.

---
 tools/xenpaging/xenpaging.c |   39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)
tools/xenpaging/xenpaging.c +31 -8
@@ -25,6 +25,7 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <time.h>
+#include <signal.h>
 #include <xc_private.h>
 
 #include <xen/mem_event.h>
@@ -43,6 +44,11 @@
 #define DPRINTF(...) ((void)0)
 #endif
 
+static int interrupted;
+static void close_handler(int sig)
+{
+    interrupted = sig;
+}
 
 static void *init_page(void)
 {
@@ -248,7 +254,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error tearing down domain paging in xen");
-        goto err;
     }
 
     /* Unbind VIRQ */
@@ -256,7 +261,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error unbinding event port");
-        goto err;
     }
     paging->mem_event.port = -1;
 
@@ -265,7 +269,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error closing event channel");
-        goto err;
     }
     paging->mem_event.xce_handle = -1;
     
@@ -274,7 +277,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error closing connection to xen");
-        goto err;
     }
     paging->xc_handle = NULL;
 
@@ -380,7 +382,7 @@ int xenpaging_evict_page(xc_interface *x
     return ret;
 }
 
-int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp)
+static int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp)
 {
     int ret;
 
@@ -461,6 +463,11 @@ static int evict_victim(xc_interface *xc
             goto out;
         }
 
+        if ( interrupted )
+        {
+            ret = -EINTR;
+            goto out;
+        }
         ret = xc_mem_paging_nominate(paging->xc_handle,
                                      paging->mem_event.domain_id, victim->gfn);
         if ( ret == 0 )
@@ -485,6 +492,7 @@ static int evict_victim(xc_interface *xc
 
 int main(int argc, char *argv[])
 {
+    struct sigaction act;
     domid_t domain_id;
     int num_pages;
     xenpaging_t *paging;
@@ -520,7 +528,7 @@ int main(int argc, char *argv[])
     if ( paging == NULL )
     {
         ERROR("Error initialising paging");
-        goto out;
+        return 1;
     }
 
     /* Open file */
@@ -529,9 +537,18 @@ int main(int argc, char *argv[])
     if ( fd < 0 )
     {
         perror("failed to open file");
-        return -1;
+        return 2;
     }
 
+    /* ensure that if we get a signal, we'll do cleanup, then exit */
+    act.sa_handler = close_handler;
+    act.sa_flags = 0;
+    sigemptyset(&act.sa_mask);
+    sigaction(SIGHUP,  &act, NULL);
+    sigaction(SIGTERM, &act, NULL);
+    sigaction(SIGINT,  &act, NULL);
+    sigaction(SIGALRM, &act, NULL);
+
     /* Evict pages */
     memset(victims, 0, sizeof(xenpaging_victim_t) * num_pages);
     for ( i = 0; i < num_pages; i++ )
@@ -539,6 +556,8 @@ int main(int argc, char *argv[])
         rc = evict_victim(xch, paging, domain_id, &victims[i], fd, i);
         if ( rc == -ENOSPC )
             break;
+        if ( rc == -EINTR )
+            break;
         if ( i % 100 == 0 )
             DPRINTF("%d pages evicted\n", i);
     }
@@ -546,7 +565,7 @@ int main(int argc, char *argv[])
     DPRINTF("pages evicted\n");
 
     /* Swap pages in and out */
-    while ( 1 )
+    while ( !interrupted )
     {
         /* Wait for Xen to signal that a page needs paged in */
         rc = xc_wait_for_event_or_timeout(xch, paging->mem_event.xce_handle, 100);
@@ -647,8 +666,11 @@ int main(int argc, char *argv[])
             }
         }
     }
+    DPRINTF("xenpaging got signal %d\n", interrupted);
 
  out:
+    unlink(filename);
+    close(fd);
     free(victims);
 
     /* Tear down domain paging */
@@ -661,6 +683,7 @@ int main(int argc, char *argv[])
 
     xc_interface_close(xch);
 
+    DPRINTF("xenpaging exit code %d\n", rc);
     return rc;
 }
#22 lore raw

PATCH 11/16 xenpaging: increase recently used pages from 4MB to 64MB

In reply to Olaf Hering

Increase recently used pages from 4MB to 64MB.
Keeping more pages in memory allows the guest to make more progress if the paging file spans the entire guest memory.

---
 tools/xenpaging/policy_default.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
tools/xenpaging/policy_default.c +1 -1
@@ -26,7 +26,7 @@
 #include "policy.h"
 
 
-#define MRU_SIZE 1024
+#define MRU_SIZE (1024 * 16)
 
 
 static unsigned long mru[MRU_SIZE];
#23 lore raw

PATCH 12/16 xenpaging: print info when free request slots drop below 3

In reply to Olaf Hering

Add debugging aid to free request slots in the ring buffer.
It should not happen that the ring gets full, print info anyway if it happens.

---
 xen/arch/x86/mm/mem_event.c |    5 +++++
 1 file changed, 5 insertions(+)
xen/arch/x86/mm/mem_event.c +5 -0
@@ -152,6 +152,11 @@ int mem_event_check_ring(struct domain *
     mem_event_ring_lock(d);
 
     free_requests = RING_FREE_REQUESTS(&d->mem_event.front_ring);
+    if ( unlikely(free_requests < 3) )
+    {
+        gdprintk(XENLOG_INFO, "free request slots: %d\n", free_requests);
+        WARN_ON(free_requests == 0);
+    }
     ring_full = free_requests < MEM_EVENT_RING_THRESHOLD;
 
     if ( (curr->domain->domain_id == d->domain_id) && ring_full )
#24 lore raw

PATCH 13/16 xenpaging: prevent page-out of first 16MB

In reply to Olaf Hering

This is more a workaround than a bugfix:
Don't page out first 16MB of memory.
When the BIOS does its initialization process and xenpaging removes pages, crashes will occour due to lack of support of xenpaging.

---
 tools/xenpaging/policy_default.c |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
tools/xenpaging/policy_default.c +3 -2
@@ -60,8 +60,9 @@ int policy_init(xenpaging_t *paging)
     for ( i = 0; i < MRU_SIZE; i++ )
         mru[i] = INVALID_MFN;
 
-    /* Don't page out page 0 */
-    set_bit(0, bitmap);
+    /* Don't page out first 16MB */
+    for ( i = 0; i < ((16*1024*1024)/4096); i++ )
+        set_bit(i, bitmap);
 
  out:
     return rc;
#25 lore raw

PATCH 14/16 xenpaging: add dynamic startup delay for xenpaging

In reply to Olaf Hering

This is a debug helper. Since the xenpaging support is still fragile, run xenpaging at different stages in the bootprocess. Different delays will trigger more bugs. This implementation starts without delay for 5 reboots, then increments the delay by 0.1 seconds It uses xenstore for presistant storage of delay values

TODO: find the correct place to remove the xenstore directory when the guest is shutdown or crashed

---
 tools/python/xen/xend/image.py |   32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
tools/python/xen/xend/image.py +32 -0
@@ -123,6 +123,18 @@ class ImageHandler:
 
         self.device_model = vmConfig['platform'].get('device_model')
         self.xenpaging = vmConfig['platform'].get('xenpaging')
+        self.xenpaging_delay = xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay" % self.vm.info['name_label'])
+        if self.xenpaging_delay == None:
+            log.warn("XXX creating /local/domain/0/xenpaging/%s" % self.vm.info['name_label'])
+            xstransact.Mkdir("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'])
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay', '7.0'))
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_inc', '0.1'))
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_use', '5'))
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_used', '0'))
+        self.xenpaging_delay = float(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay" % self.vm.info['name_label']))
+        self.xenpaging_delay_inc = float(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay_inc" % self.vm.info['name_label']))
+        self.xenpaging_delay_use = int(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay_use" % self.vm.info['name_label']))
+        self.xenpaging_delay_used = int(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay_used" % self.vm.info['name_label']))
 
         self.display = vmConfig['platform'].get('display')
         self.xauthority = vmConfig['platform'].get('xauthority')
@@ -401,6 +413,17 @@ class ImageHandler:
             return
         if self.xenpaging_pid:
             return
+        if self.xenpaging_delay_used < self.xenpaging_delay_use:
+            self.xenpaging_delay_used += 1
+        else:
+            self.xenpaging_delay_used = 0
+            self.xenpaging_delay += self.xenpaging_delay_inc
+        log.info("delay_used %s" % self.xenpaging_delay_used)
+        log.info("delay_use %s" % self.xenpaging_delay_use)
+        log.info("delay %s" % self.xenpaging_delay)
+        log.info("delay_inc %s" % self.xenpaging_delay_inc)
+        xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay', self.xenpaging_delay))
+        xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_used', self.xenpaging_delay_used))
         xenpaging_bin = auxbin.pathTo("xenpaging")
         args = [xenpaging_bin]
         args = args + ([ "%d" % self.vm.getDomid()])
@@ -430,6 +453,9 @@ class ImageHandler:
                 except:
                     log.warn("chdir %s failed" % xenpaging_dir)
                 try:
+                    if self.xenpaging_delay != 0.0:
+                        log.info("delaying xenpaging startup %s seconds ..." % self.xenpaging_delay)
+                        time.sleep(self.xenpaging_delay)
                     log.info("starting %s" % args)
                     os.execve(xenpaging_bin, args, env)
                 except Exception, e:
@@ -445,10 +471,16 @@ class ImageHandler:
             self.xenpaging_pid = xenpaging_pid
             os.close(null)
             os.close(logfd)
+            if self.xenpaging_delay == 0.0:
+                log.warn("waiting for xenpaging ...")
+                time.sleep(22)
+                log.warn("waiting for xenpaging done.")
 
     def destroyXenPaging(self):
         if self.xenpaging is None:
             return
+        # FIXME find correct place for guest shutdown or crash
+        #xstransact.Remove("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'])
         if self.xenpaging_pid:
             try:
                 os.kill(self.xenpaging_pid, signal.SIGHUP)
#26 lore raw

PATCH 15/16 xenpaging: print p2mt for already paged-in pages

In reply to Olaf Hering

Add more debug output, print p2mt for pages which were requested more than once.

---
 tools/xenpaging/xenpaging.c |    2 ++
 1 file changed, 2 insertions(+)
tools/xenpaging/xenpaging.c +2 -0
@@ -642,8 +642,10 @@ int main(int argc, char *argv[])
             else
             {
                 DPRINTF("page already populated (domain = %d; vcpu = %d;"
+                        " p2mt = %x;"
                         " gfn = %"PRIx64"; paused = %"PRId64")\n",
                         paging->mem_event.domain_id, req.vcpu_id,
+                        req.p2mt,
                         req.gfn, req.flags & MEM_EVENT_FLAG_VCPU_PAUSED);
 
                 /* Tell Xen to resume the vcpu */
#27 lore raw

PATCH 16/16 xenpaging: (sparse) documenation

In reply to Olaf Hering

Write up some sparse documentation about xenpaging usage.

---
 docs/misc/xenpaging.txt |   48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
docs/misc/xenpaging.txt new file +48 -0
@@ -0,0 +1,48 @@
+Warning:
+
+The xenpaging code is new and not fully debugged.
+Usage of xenpaging can crash Xen or cause severe data corruption in the
+guest memory and its filesystems!
+
+Description:
+
+xenpaging writes memory pages of a given guest to a file and moves the
+pages back to the pool of available memory.  Once the guests wants to
+access the paged-out memory, the page is read from disk and placed into
+memory.  This allows the sum of all running guests to use more memory
+than physically available on the host.
+
+Usage:
+
+Once the guest is running, run xenpaging with the guest_id and the
+number of pages to page-out:
+
+  chdir /var/lib/xen/xenpaging
+  xenpaging <guest_id>  <number_of_pages>
+
+To obtain the guest_id, run 'xm list'.
+xenpaging will write the pagefile to the current directory.
+Example with 128MB pagefile on guest 1:
+
+  xenpaging 1 32768
+
+Caution: stopping xenpaging manually will cause the guest to stall or
+crash because the paged-out memory is not written back into the guest!
+
+After a reboot of a guest, its guest_id changes, the current xenpaging
+binary has no target anymore. To automate restarting of xenpaging after
+guest reboot, specify the number if pages in the guest configuration
+file /etc/xen/vm/<guest_name>:
+
+xenpaging=32768
+
+Redo the guest with 'xm create /etc/xen/vm/<guest_name>' to activate the
+changes.
+
+
+Todo:
+- implement stopping of xenpaging
+- implement/test live migration
+
+
+# vim: tw=72
#28 lore raw
In reply to Olaf Hering
Quoted 2 lines · Olaf Hering

But the patch to handle guest_remove_page does not work as expected, the guest will hang. Any help to get this fixed without retry loops is appreciated.

If the vcpu is stopped and later resumed, similar to a p2m_mem_paging_populate -> p2m_mem_paging_prep -> p2m_mem_paging_resume cycle, the guest will not stall. I will send an updated patch.

Olaf

Acked-by on v1 1/16
#2 lore raw

Re: [PATCH 01/16] xenpaging: whitespace fixes after addition of __get_paged_frame

In reply to Olaf Hering

Olaf Hering writes ("[Xen-devel] [PATCH 01/16] xenpaging: whitespace fixes after addition of __get_paged_frame"):

Quoted 2 lines

Cleanup whitespace which was introduced in patch that added __get_paged_frame. Convert tabs to spaces to follow the coding style in this file.

If we approve of patches to fix whitespace and intending infelicities, then I think this patch should be applied right away.

So on that condition:

Ian.

#11 lore raw

Re: [PATCH 07/16] xenpaging: populate only paged-out pages

In reply to Olaf Hering

Olaf Hering writes ("[Xen-devel] [PATCH 07/16] xenpaging: populate only paged-out pages"):

Quoted 4 lines

populdate a paged-out page only once to reduce pressure in the ringbuffer. Several cpus may still request a page at once. xenpaging can handle this.

But: maybe this will miss pages in paging-out state?

I can't answer your second question, but surely this pattern:

xen/arch/x86/hvm/emulate.c static int hvmemul_do_io( · line 70

Quoted 3 lines
xen/arch/x86/hvm/emulate.c +2 -1
-        p2m_mem_paging_populate(p2m, ram_gfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, ram_gfn);

would better be done inside p2m_mem_paging_populate ? Unless there are some calls to _populate which should skip the check ?

Ian.

#15 lore raw

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

In reply to Olaf Hering

Olaf Hering writes ("[Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):

Quoted 1 line

Allow runlevel 6 as default runlevel.

...

tools/python/xen/xend/XendConstants.py DOM_STATES_OLD = [ · line 95

Quoted 2 lines
tools/python/xen/xend/XendConstants.py +1 -1
-MINIMUM_RESTART_TIME = 60
+MINIMUM_RESTART_TIME = 5

I don't understand what this change is doing in your xenpaging series; nor does setting runlevel 6 as the default runlevel seem very sensible other than perhaps for some kind of stress testing.

Perhaps this patch snuck into the series by mistake ?

Ian.

#16 lore raw

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

Quoted 11 lines · Ian Jackson

Olaf Hering writes ("[Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):

Quoted 1 line

Allow runlevel 6 as default runlevel.

...

tools/python/xen/xend/XendConstants.py DOM_STATES_OLD = [ · line 95

Quoted 2 lines
tools/python/xen/xend/XendConstants.py +1 -1
-MINIMUM_RESTART_TIME = 60
+MINIMUM_RESTART_TIME = 5

I don't understand what this change is doing in your xenpaging series; nor does setting runlevel 6 as the default runlevel seem very sensible other than perhaps for some kind of stress testing.

Perhaps this patch snuck into the series by mistake ?

Its there intentional in my series, just to get the startup code tested automatically. The runtime code seems to work ok now that the granttable part got fixed.

Last time I tried to get this variable into the vm config file, it did not work as expected. Appearently its a global variable, not per guest.

And its value was changed from 20 (I think) to 60 without a good changelog entry, not too long ago. For my testing, even 20 would have been to long. If I use 'reboot' quickly in grub, guest runtime is around 7 seconds I think.

Too bad, this variable is only a bandaid because it doesnt know any context of the guest. And it cant know any context. When is a guest in a good or bad state? Should it panic, or crash? Its not easy to detect a bad condition.

A better change would be to allow an override in the guest config file.

Olaf

#17 lore raw

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

Olaf Hering writes ("Re: [Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):

Quoted 4 lines

Too bad, this variable is only a bandaid because it doesnt know any context of the guest. And it cant know any context. When is a guest in a good or bad state? Should it panic, or crash? Its not easy to detect a bad condition.

Absolutely. Also some guests may reboot rapidly the first time (or two) eg due to fsck, and thereafter be fine. It ought to be more sophisticated.

Quoted 1 line

A better change would be to allow an override in the guest config file.

Yes :-).

Ian.

#18 lore raw

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

Hi :)

Quoted 9 lines · Ian Jackson

Olaf Hering writes ("Re: [Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):

Quoted 4 lines

Too bad, this variable is only a bandaid because it doesnt know any context of the guest. And it cant know any context. When is a guest in a good or bad state? Should it panic, or crash? Its not easy to detect a bad condition.

Absolutely. Also some guests may reboot rapidly the first time (or two) eg due to fsck, and thereafter be fine. It ought to be more sophisticated.

Or even more, this is needed for example for Windows to register theirselves to KMS. Sometimes I need to add a sleep() to avoid this going into action.

Quoted 3 lines
Quoted 1 line

A better change would be to allow an override in the guest config file.

Yes :-).

I'd really like this too :)

Cheers,
Xavier

--
Xavier Beaudouin - [email protected] - http://www.soprive.net/
So Privé - Le premier acteur dédié au cloud computing en France GPG Fingerprints : A6B2 D563 F93B A3AF C08A CBAC 6BC6 79EB DCC9 9867

#7 lore raw

Re: [PATCH 05/16] xenpaging: update machine_to_phys_mapping during page-in and page-out

In reply to Olaf Hering

The machine_to_phys_mapping array needs updating during page-out. If a page is gone, a call to get_gpfn_from_mfn will still return the old gfn for an already paged-out page. This happens when the entire guest ram is paged-out before xen_vga_populate_vram() runs. Then XENMEM_populate_physmap is called with gfn 0xff000. A new page is allocated with alloc_domheap_pages. This new page does not have a gfn yet. However, in guest_physmap_add_entry() the passed mfn maps still to an old gfn. This old gfn is paged-out and has no mfn anymore. As a result, the ASSERT() triggers because p2m_is_ram() is true for p2m_ram_paging* types.

If the machine_to_phys_mapping array is updated properly, both loops in guest_physmap_add_entry() turn into no-ops for the new page and the mfn/gfn mapping will be done at the end of the function.

The same thing needs to happen dring a page-in.
If XENMEM_add_to_physmap is used with XENMAPSPACE_gmfn, get_gpfn_from_mfn() will return an appearently valid gfn. As a result, guest_physmap_remove_page() is called. The ASSERT in p2m_remove_page triggers because the passed mfn does not match the old mfn for the passed gfn.

---
v2:
call set_gpfn_from_mfn only if mfn is valid

 xen/arch/x86/mm/p2m.c |    3 +++
 1 file changed, 3 insertions(+)
xen/arch/x86/mm/p2m.c +3 -0
@@ -2742,6 +2742,7 @@ int p2m_mem_paging_evict(struct p2m_doma
     /* Remove mapping from p2m table */
     p2m_lock(p2m);
     set_p2m_entry(p2m, gfn, _mfn(PAGING_MFN), 0, p2m_ram_paged);
+    set_gpfn_from_mfn(mfn_x(mfn), INVALID_M2P_ENTRY);
     p2m_unlock(p2m);
 
     /* Put the page back so it gets freed */
@@ -2820,6 +2821,8 @@ void p2m_mem_paging_resume(struct p2m_do
     mfn = gfn_to_mfn(p2m, rsp.gfn, &p2mt);
     p2m_lock(p2m);
     set_p2m_entry(p2m, rsp.gfn, mfn, 0, p2m_ram_rw);
+    if ( mfn_valid(mfn) )
+        set_gpfn_from_mfn(mfn_x(mfn), rsp.gfn);
     p2m_unlock(p2m);
 
     /* Unpause domain */
#9 lore raw

Re: [PATCH 06/16] xenpaging: drop paged pages in guest_remove_page

In reply to Olaf Hering

Simply drop paged-pages in guest_remove_page(), and notify xenpaging to drop reference to the gfn.

---
v2:
resume dropped page to unpause vcpus

 tools/xenpaging/xenpaging.c    |   17 ++++++++++----
 xen/arch/x86/mm/p2m.c          |   47 ++++++++++++++++++++++++++++++++++-------
 xen/common/memory.c            |    6 +++++
 xen/include/asm-x86/p2m.h      |    4 +++
 xen/include/public/mem_event.h |    1 
 5 files changed, 63 insertions(+), 12 deletions(-)
tools/xenpaging/xenpaging.c +12 -5
@@ -586,12 +586,19 @@ int main(int argc, char *argv[])
                     goto out;
                 }
                 
-                /* Populate the page */
-                rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
-                if ( rc != 0 )
+                if ( req.flags & MEM_EVENT_FLAG_DROP_PAGE )
                 {
-                    ERROR("Error populating page");
-                    goto out;
+                    DPRINTF("Dropping page %"PRIx64" p2mt %x\n", req.gfn, req.p2mt);
+                }
+                else
+                {
+                    /* Populate the page */
+                    rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
+                    if ( rc != 0 )
+                    {
+                        ERROR("Error populating page");
+                        goto out;
+                    }
                 }
 
                 /* Prepare the response */
xen/arch/x86/mm/p2m.c +38 -5

56 lines not on this page · raw message

xen/common/memory.c +6 -0
@@ -162,6 +162,12 @@ int guest_remove_page(struct domain *d,
 
 #ifdef CONFIG_X86
     mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt)); 
+    if ( unlikely(p2m_is_paging(p2mt)) )
+    {
+        guest_physmap_remove_page(d, gmfn, mfn, 0);
+        p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
+        return 1;
+    }
 #else
     mfn = gmfn_to_mfn(d, gmfn);
 #endif
xen/include/asm-x86/p2m.h +4 -0
@@ -480,6 +480,8 @@ int set_shared_p2m_entry(struct p2m_doma
 int p2m_mem_paging_nominate(struct p2m_domain *p2m, unsigned long gfn);
 /* Evict a frame */
 int p2m_mem_paging_evict(struct p2m_domain *p2m, unsigned long gfn);
+/* Tell xenpaging to drop a paged out frame */
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn);
 /* Start populating a paged out frame */
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn);
 /* Prepare the p2m for paging a frame in */
@@ -487,6 +489,8 @@ int p2m_mem_paging_prep(struct p2m_domai
 /* Resume normal operation (in case a domain was paused) */
 void p2m_mem_paging_resume(struct p2m_domain *p2m);
 #else
+static inline void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{ }
 static inline void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 { }
 #endif
xen/include/public/mem_event.h +1 -0
@@ -37,6 +37,7 @@
 #define MEM_EVENT_FLAG_VCPU_PAUSED  (1 << 0)
 #define MEM_EVENT_FLAG_DOM_PAUSED   (1 << 1)
 #define MEM_EVENT_FLAG_OUT_OF_MEM   (1 << 2)
+#define MEM_EVENT_FLAG_DROP_PAGE    (1 << 3)
 
 
 typedef struct mem_event_shared_page {
#29 lore raw

[PATCH 17/16] xenpaging: notify policy only on resume

In reply to Olaf Hering

If a page is requested more than once, the policy is also notified more than once about the page-in. However, a page-in happens only once. Any further resume will only unpause the other vcpu. The multiple notify will put the page into the mru list multiple times and it will unlock other already resumed pages too early. In the worst case, a page that was just resumed can be evicted right away, causing a deadlock in the guest.

---
This happend since some time already, today I found the cause.

 tools/xenpaging/xenpaging.c |    9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
tools/xenpaging/xenpaging.c +5 -4
@@ -382,7 +382,7 @@ int xenpaging_evict_page(xc_interface *x
     return ret;
 }
 
-static int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp)
+static int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp, int notify_policy)
 {
     int ret;
 
@@ -392,7 +392,8 @@ static int xenpaging_resume_page(xenpagi
         goto out;
 
     /* Notify policy of page being paged in */
-    policy_notify_paged_in(paging->mem_event.domain_id, rsp->gfn);
+    if ( notify_policy )
+        policy_notify_paged_in(paging->mem_event.domain_id, rsp->gfn);
 
     /* Tell Xen page is ready */
     ret = xc_mem_paging_resume(paging->xc_handle, paging->mem_event.domain_id,
@@ -626,7 +627,7 @@ int main(int argc, char *argv[])
                 rsp.vcpu_id = req.vcpu_id;
                 rsp.flags = req.flags;
 
-                rc = xenpaging_resume_page(paging, &rsp);
+                rc = xenpaging_resume_page(paging, &rsp, 1);
                 if ( rc != 0 )
                 {
                     ERROR("Error resuming page");
@@ -655,7 +656,7 @@ int main(int argc, char *argv[])
                     rsp.vcpu_id = req.vcpu_id;
                     rsp.flags = req.flags;
 
-                    rc = xenpaging_resume_page(paging, &rsp);
+                    rc = xenpaging_resume_page(paging, &rsp, 0);
                     if ( rc != 0 )
                     {
                         ERROR("Error resuming");
#19 lore raw

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

In reply to Xavier Beaudouin

Xavier Beaudouin writes ("Re: [Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):

Quoted 3 lines

Or even more, this is needed for example for Windows to register theirselves to KMS. Sometimes I need to add a sleep() to avoid this going into action.

Also, I notice that xl does not yet have any machinery for detecting infinite guest reboots. I suggest that we take this opportunity to invent a new mechanism for that, and it could be configurable.

xl could record the times of the last N restarts in an array, quite easily, and spot when there have been more than N restarts in time T (for default values of N=5 and T= fifteen minutes, perhaps).

Ian.

#12 lore raw

Re: [PATCH 07/16] xenpaging: populate only paged-out pages

In reply to Ian Jackson
Quoted 6 lines · Ian Jackson

Olaf Hering writes ("[Xen-devel] [PATCH 07/16] xenpaging: populate only paged-out pages"):

Quoted 4 lines

populdate a paged-out page only once to reduce pressure in the ringbuffer. Several cpus may still request a page at once. xenpaging can handle this.

But: maybe this will miss pages in paging-out state?

Quoted 8 lines

I can't answer your second question, but surely this pattern:

xen/arch/x86/hvm/emulate.c static int hvmemul_do_io( · line 70

Quoted 3 lines
xen/arch/x86/hvm/emulate.c +2 -1
-        p2m_mem_paging_populate(p2m, ram_gfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, ram_gfn);

would better be done inside p2m_mem_paging_populate ? Unless there are some calls to _populate which should skip the check ?

Ian,

thanks for bringing this up. After revisiting this patch and the reasons for it, two different things need changing.

One thing is that p2m_mem_paging_populate needs to be called unconditionally. The reason is that the vcpu may need stopping and resuming until the page content is usable again.
A change for grant_table.c:__get_paged_frame is required.

The other thing is that p2m_mem_paging_populate must only invalidate the mfn if either the page is still in the process of being paged out, or if the page is already in the process of being paged in. The latter does not happen right now. The result is that the page may have state p2m_ram_paging_in and a new valid mfn. Then that new mfn is invalidated, the gfn content in the xenpaging process was already overwritten and there is no wakeup of the vcpu. This situation is not recoverable. p2m.c:p2m_mem_paging_populate needs a change to no call set_p2m_entry if the p2mt state is already p2m_ram_paging_in

I will send two new patches for grant_table.c and p2m.c

Olaf

PS:

Below is a custom xenalyze debug log for the this. It translates to:
At some point gfn 3bdb7 was paged out, and later requested on vcpu 0. It got a new mfn 1120b9 in p2m_mem_paging_prep on vcpu 5.
Later vcpu 3 called p2m_mem_paging_populate again, and it reset the new mfn 1120b9. The xenpaging binary noticed that gfn 3bdb7 was already populated and therefore could not prep/resume this gfn. However, the first p2m_mem_paging_resume call ran into the new invalid mfn and could not set the gfn state back to p2m_ram_rw. I have a change to check the mfn in p2m_mem_paging_resume.
If p2m_mem_paging_populate had checked the p2mt properly, p2m_mem_paging_resume would have succeed and everything could proceed.

19699656: 2073.683567637 -x--|--- d0v3 olh p2m_mem_paging_nominate gfn 3bdb7 mfn f5b7 flags 0 p2mt 1 ret 0
19699705: 2073.683601641 -x--|--- d0v3 olh p2m_mem_paging_evict gfn 3bdb7 mfn f5b7 flags 41 p2mt 9 ret 0
24430211: 2107.837821159 x-|||--| d0v0 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt a line 159
24430212: 2107.837822145 x-|||--| d0v0 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt a vcpu 0
24434493: 2107.845357003 |-x--||| d0v5 olh p2m_mem_paging_prep gfn 3bdb7 mfn 1120b9 ret 0
24434542: 2107.845409334 |-|--||x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn 1120b9 p2mt b line 159
24434544: 2107.845411875 |-|--||x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn 1120b9 flags 20 p2mt b vcpu 3
24435286: 2107.856054450 |||--||x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24435287: 2107.856056334 |||--||x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24435461: 2107.860575628 --x--||- d0v5 olh p2m_mem_paging_resume gfn 3bdb7 mfn ffffffff flags 0 p2mt c
24439829: 2107.864952416 -|--|||x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24439840: 2107.864955042 -|--|||x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24449379: 2107.873837210 --|---|x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24449389: 2107.873839448 --|----x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24455031: 2107.882722451 ---|-|-x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24455033: 2107.882724440 ---|-|-x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24462725: 2107.896056767 -|---|-x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24462730: 2107.896059777 -|---|-x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3

...

#13 lore raw

Re: [PATCH 07/16] xenpaging: populate only paged-out pages

Quoted 9 lines · Olaf Hering

The other thing is that p2m_mem_paging_populate must only invalidate the mfn if either the page is still in the process of being paged out, or if the page is already in the process of being paged in. The latter does not happen right now. The result is that the page may have state p2m_ram_paging_in and a new valid mfn. Then that new mfn is invalidated, the gfn content in the xenpaging process was already overwritten and there is no wakeup of the vcpu. This situation is not recoverable. p2m.c:p2m_mem_paging_populate needs a change to no call set_p2m_entry if the p2mt state is already p2m_ram_paging_in

And even this is not perfect.
p2m_mem_paging_populate() could be called again even if the p2m state was just set to p2m_ram_rw in p2m_mem_paging_resume(). In this case the thread who calls p2m_mem_paging_populate() cant make any progress anymore.
So in the end, p2m_mem_paging_populate() can only invalidate the mfn if the p2mt happens to be p2m_ram_paged.

Olaf