Populate-on-demand: Check pages being returned by the balloon driver

@@ -3644,6 +3644,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
         p2m_type_t t;
         p2m_access_t ac;
         mfn_t mfn;
+        int l;
 
         /* Interface access to internal p2m accesses */
         hvmmem_access_t memaccess[] = {

@@ -3681,7 +3682,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
                 goto param_fail6;
 
             rc = -ESRCH;
-            mfn = p2m->get_entry(p2m, a.pfn, &t, &ac, p2m_query);
+            mfn = p2m->get_entry(p2m, a.pfn, &t, &ac, &l, p2m_query);
 
             if ( mfn_x(mfn) == INVALID_MFN )
                 goto param_fail6;

@@ -527,14 +527,14 @@ out:
 /* Read ept p2m entries */
 static mfn_t ept_get_entry(struct p2m_domain *p2m,
                            unsigned long gfn, p2m_type_t *t, p2m_access_t* a,
-                           p2m_query_t q)
+                           int *level, p2m_query_t q)
 {
     struct domain *d = p2m->domain;
     ept_entry_t *table = map_domain_page(ept_get_asr(d));
     unsigned long gfn_remainder = gfn;
     ept_entry_t *ept_entry;
     u32 index;
-    int i;
+    int i=5;
     int ret = 0;
     mfn_t mfn = _mfn(INVALID_MFN);
 

@@ -616,6 +616,7 @@ static mfn_t ept_get_entry(struct p2m_do
     }
 
 out:
+    *level=i+1;
     unmap_domain_page(table);
     return mfn;
 }

@@ -709,9 +710,10 @@ out:
 
 static mfn_t ept_get_entry_current(struct p2m_domain *p2m,
                                    unsigned long gfn, p2m_type_t *t, p2m_access_t *a,
+                                   int *l,
                                    p2m_query_t q)
 {
-    return ept_get_entry(p2m, gfn, t, a, q);
+    return ept_get_entry(p2m, gfn, t, a, l, q);
 }
 
 /*

@@ -1579,7 +1579,7 @@ out:
 
 static mfn_t
 p2m_gfn_to_mfn(struct p2m_domain *p2m, unsigned long gfn, p2m_type_t *t, p2m_access_t *a,
-               p2m_query_t q)
+               int *level, p2m_query_t q)
 {
     mfn_t mfn;
     paddr_t addr = ((paddr_t)gfn) << PAGE_SHIFT;

@@ -1594,7 +1594,8 @@ p2m_gfn_to_mfn(struct p2m_domain *p2m, u
      * XXX we will return p2m_invalid for unmapped gfns */
     *t = p2m_mmio_dm;
     /* Not implemented except with EPT */
-    *a = p2m_access_rwx; 
+    *a = p2m_access_rwx;
+    *level = 5;
 
     mfn = pagetable_get_mfn(p2m_get_pagetable(p2m));
 

@@ -1602,6 +1603,7 @@ p2m_gfn_to_mfn(struct p2m_domain *p2m, u
         /* This pfn is higher than the highest the p2m map currently holds */
         return _mfn(INVALID_MFN);
 
+    *level = 4;
 #if CONFIG_PAGING_LEVELS >= 4
     {
         l4_pgentry_t *l4e = map_domain_page(mfn_x(mfn));

@@ -1615,6 +1617,7 @@ p2m_gfn_to_mfn(struct p2m_domain *p2m, u
         unmap_domain_page(l4e);
     }
 #endif
+    *level=3;
     {
         l3_pgentry_t *l3e = map_domain_page(mfn_x(mfn));
 #if CONFIG_PAGING_LEVELS == 3

@@ -1662,6 +1665,7 @@ pod_retry_l3:
 
     l2e = map_domain_page(mfn_x(mfn));
     l2e += l2_table_offset(addr);
+    *level=2;
 
 pod_retry_l2:
     if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )

@@ -1695,6 +1699,7 @@ pod_retry_l2:
 
     l1e = map_domain_page(mfn_x(mfn));
     l1e += l1_table_offset(addr);
+    *level=1;
 pod_retry_l1:
     if ( (l1e_get_flags(*l1e) & _PAGE_PRESENT) == 0 )
     {

@@ -1723,7 +1728,7 @@ pod_retry_l1:
 /* Read the current domain's p2m table (through the linear mapping). */
 static mfn_t p2m_gfn_to_mfn_current(struct p2m_domain *p2m,
                                     unsigned long gfn, p2m_type_t *t, p2m_access_t *a,
-                                    p2m_query_t q)
+                                    int *level, p2m_query_t q)
 {
     mfn_t mfn = _mfn(INVALID_MFN);
     p2m_type_t p2mt = p2m_mmio_dm;

@@ -1735,6 +1740,7 @@ static mfn_t p2m_gfn_to_mfn_current(stru
 
     /* Not currently implemented except for EPT */
     *a = p2m_access_rwx;
+    *level=5;
 
     if ( gfn <= p2m->max_mapped_pfn )
     {

@@ -1749,6 +1755,7 @@ static mfn_t p2m_gfn_to_mfn_current(stru
                / sizeof(l1_pgentry_t));
 
 #if CONFIG_PAGING_LEVELS >= 4
+    *level=3;
         /*
          * Read & process L3
          */

@@ -1802,6 +1809,7 @@ static mfn_t p2m_gfn_to_mfn_current(stru
         p2m_entry = &__linear_l1_table[l1_linear_offset(RO_MPT_VIRT_START)
                                        + l2_linear_offset(addr)];
 
+    *level=2;
     pod_retry_l2:
         ret = __copy_from_user(&l2e,
                                p2m_entry,

@@ -1855,6 +1863,7 @@ static mfn_t p2m_gfn_to_mfn_current(stru
 
         /* Need to __copy_from_user because the p2m is sparse and this
          * part might not exist */
+    *level=1;
     pod_retry_l1:
         p2m_entry = &phys_to_machine_mapping[gfn];
 

@@ -2069,6 +2078,7 @@ void p2m_teardown(struct p2m_domain *p2m
     unsigned long gfn;
     p2m_type_t t;
     p2m_access_t a;
+    int l;
     mfn_t mfn;
 #endif
 

@@ -2077,7 +2087,7 @@ void p2m_teardown(struct p2m_domain *p2m
 #ifdef __x86_64__
     for ( gfn=0; gfn < p2m->max_mapped_pfn; gfn++ )
     {
-        mfn = p2m->get_entry(p2m, gfn, &t, &a, p2m_query);
+        mfn = p2m->get_entry(p2m, gfn, &t, &a, &l, p2m_query);
         if ( mfn_valid(mfn) && (t == p2m_ram_shared) )
             BUG_ON(mem_sharing_unshare_page(p2m, gfn, MEM_SHARING_DESTROY_GFN));
     }

@@ -2375,6 +2385,7 @@ p2m_remove_page(struct p2m_domain *p2m, 
     mfn_t mfn_return;
     p2m_type_t t;
     p2m_access_t a;
+    int l;
 
     if ( !paging_mode_translate(p2m->domain) )
     {

@@ -2390,7 +2401,7 @@ p2m_remove_page(struct p2m_domain *p2m, 
     {
         for ( i = 0; i < (1UL << page_order); i++ )
         {
-            mfn_return = p2m->get_entry(p2m, gfn + i, &t, &a, p2m_query);
+            mfn_return = p2m->get_entry(p2m, gfn + i, &t, &a, &l, p2m_query);
             if ( !p2m_is_grant(t) )
                 set_gpfn_from_mfn(mfn+i, INVALID_M2P_ENTRY);
             ASSERT( !p2m_is_valid(t) || mfn + i == mfn_x(mfn_return) );

@@ -3079,10 +3090,11 @@ void p2m_mem_access_check(unsigned long 
     mfn_t mfn;
     p2m_type_t p2mt;
     p2m_access_t p2ma;
+    int p2ml;
     
     /* First, handle rx2rw conversion automatically */
     p2m_lock(p2m);
-    mfn = p2m->get_entry(p2m, gfn, &p2mt, &p2ma, p2m_query);
+    mfn = p2m->get_entry(p2m, gfn, &p2mt, &p2ma, &p2ml, p2m_query);
 
     if ( access_w && p2ma == p2m_access_rx2rw ) 
     {

@@ -216,11 +216,13 @@ struct p2m_domain {
                                        unsigned long gfn,
                                        p2m_type_t *p2mt,
                                        p2m_access_t *p2ma,
+                                       int *level,
                                        p2m_query_t q);
     mfn_t              (*get_entry_current)(struct p2m_domain *p2m,
                                             unsigned long gfn,
                                             p2m_type_t *p2mt,
                                             p2m_access_t *p2ma,
+                                            int *level,
                                             p2m_query_t q);
     void               (*change_entry_type_global)(struct p2m_domain *p2m,
                                                    p2m_type_t ot,

@@ -334,7 +336,8 @@ static inline mfn_t gfn_to_mfn_type_curr
                                             p2m_access_t *a,
                                             p2m_query_t q)
 {
-    return p2m->get_entry_current(p2m, gfn, t, a, q);
+    int l;
+    return p2m->get_entry_current(p2m, gfn, t, a, &l, q);
 }
 
 /* Read P2M table, mapping pages as we go.

@@ -344,7 +347,8 @@ gfn_to_mfn_type_p2m(struct p2m_domain *p
                               p2m_type_t *t, p2m_query_t q)
 {
     p2m_access_t a = 0;
-    return p2m->get_entry(p2m, gfn, t, &a, q);
+    int l;
+    return p2m->get_entry(p2m, gfn, t, &a, &l, q);
 }

@@ -74,6 +74,7 @@ boolean_param("hap_2mb", opt_hap_2mb);
 
 #define SUPERPAGE_PAGES (1UL << 9)
 #define superpage_aligned(_x)  (((_x)&(SUPERPAGE_PAGES-1))==0)
+#define order_aligned(_x,_o)  (((_x)&((1<<(_o))-1))==0)
 
 static unsigned long p2m_type_to_flags(p2m_type_t t, mfn_t mfn)
 {

@@ -767,6 +768,9 @@ p2m_pod_offline_or_broken_replace(struct
     return;
 }
 
+static int p2m_pod_zero_check_superpage(struct p2m_domain *p2m, unsigned long gfn);
+
+
 /* This function is needed for two reasons:
  * + To properly handle clearing of PoD entries
  * + To "steal back" memory being freed for the PoD cache, rather than

@@ -774,6 +778,9 @@ p2m_pod_offline_or_broken_replace(struct
  *
  * Once both of these functions have been completed, we can return and
  * allow decrease_reservation() to handle everything else.
+ *
+ * Because the balloon driver races with the page scrubber, we also
+ * scan for zeroed superpages we can reclaim.
  */
 int
 p2m_pod_decrease_reservation(struct domain *d,

@@ -781,11 +788,16 @@ p2m_pod_decrease_reservation(struct doma
                              unsigned int order)
 {
     int ret=0;
-    int i;
+    int i, entry_size=0;
     struct p2m_domain *p2m = p2m_get_hostp2m(d);
 
     int steal_for_cache = 0;
-    int pod = 0, nonpod = 0, ram = 0;
+    int nonpod = 0;
+#if 0
+    unsigned long gfn_aligned;
+
+    int nonpod = 0, ram = 0;
+#endif
     
 
     /* If we don't have any outstanding PoD entries, let things take their

@@ -794,7 +806,8 @@ p2m_pod_decrease_reservation(struct doma
         goto out;
 
     /* Figure out if we need to steal some freed memory for our cache */
-    steal_for_cache =  ( p2m->pod.entry_count > p2m->pod.count );
+    if ( p2m->pod.entry_count > p2m->pod.count )
+        steal_for_cache = ( p2m->pod.entry_count - p2m->pod.count );
 
     p2m_lock(p2m);
     audit_p2m(p2m, 1);

@@ -802,7 +815,7 @@ p2m_pod_decrease_reservation(struct doma
     if ( unlikely(d->is_dying) )
         goto out_unlock;
 
-    /* See what's in here. */
+#if 0
     /* FIXME: Add contiguous; query for PSE entries? */
     for ( i=0; i<(1<<order); i++)
     {

@@ -876,12 +889,105 @@ p2m_pod_decrease_reservation(struct doma
         }
     }    
 
+#else
+/*
+ * while(more pages to look at)
+ *  - Find the next page unit
+ *    - if req_decrease >= page_size
+ *         or page is zero
+ *       - Reclaim whole page
+ *    - else
+ *       - Reclaim req_decrease         
+ *       - break out of the loop
+ */
+    /* FIXME: Add contiguous; query for PSE entries? */
+    for ( i=0; i<(1<<order); i+=entry_size)
+    {
+        mfn_t mfn;
+        p2m_type_t t;
+        p2m_access_t a;
+        int l, entry_order;
+
+        /* See what's in here. */
+        mfn = p2m->get_entry(p2m, gpfn + i, &t, &a, &l, p2m_query);
+
+        /* NB: entry_size is used by the loop update; so if you change the
+         * level at which you're working (i.e., decide to work w/ 4k
+         * pages instead of a superpage), you must change entry_size
+         * so that the loop logic works right. */
+        entry_order = (l-1)*9;
+        entry_size = 1<<(entry_order);
+
+        /* If this is in a superpage and we need ram, try to nab it */
+        if ( steal_for_cache
+             && entry_order == 9
+             && p2m_is_ram(t)
+             && p2m_pod_zero_check_superpage(p2m, gpfn & ~(SUPERPAGE_PAGES-1)) )
+        {
+            /* Now we have a PoD entry to deal with; update
+             * steal_for_cache, and set entry_size to 0 so that we
+             * re-process the entry. */
+            entry_size=0;
+
+            /* Update steal_for_cache */
+            if ( p2m->pod.entry_count > p2m->pod.count )
+                steal_for_cache = ( p2m->pod.entry_count - p2m->pod.count );
+            else
+                steal_for_cache = 0;
+            
+            continue;
+        }
+
+        /* Switch to singleton pages if we don't want to reclaim the
+         * whole page, or if the start is not SP aligned.  Future
+         * iterations to this superpage frame will then be */
+        if ( entry_order > 0
+             && ( i + entry_size > (1<<order)
+                  || !superpage_aligned(gpfn + i) ) )
+        {
+            entry_order = 0;
+            entry_size = 1;
+        }
+
+        if ( t == p2m_populate_on_demand )
+        {
+            set_p2m_entry(p2m, gpfn+i, _mfn(INVALID_MFN), entry_order, p2m_invalid, p2m->default_access);
+            p2m->pod.entry_count-=(1<<entry_order); /* Lock: p2m */
+            BUG_ON(p2m->pod.entry_count < 0);
+        }
+        else if( p2m_is_ram(t) )
+        {
+            if ( steal_for_cache )
+            {
+                struct page_info *page;
+                
+                ASSERT(mfn_valid(mfn));
+                
+                page = mfn_to_page(mfn);
+                
+                set_p2m_entry(p2m, gpfn + i, _mfn(INVALID_MFN), entry_order,
+                              p2m_invalid, p2m->default_access);
+                set_gpfn_from_mfn(mfn_x(mfn), INVALID_M2P_ENTRY);
+                
+                p2m_pod_cache_add(p2m, page, entry_order);
+                
+                /* Update steal_for_cache */
+                if ( p2m->pod.entry_count > p2m->pod.count )
+                    steal_for_cache = ( p2m->pod.entry_count - p2m->pod.count );
+                else
+                    steal_for_cache = 0;
+            }
+            else
+                nonpod++;
+        }
+    }
+#endif
     /* If there are no more non-PoD entries, tell decrease_reservation() that
      * there's nothing left to do. */
     if ( nonpod == 0 )
         ret = 1;
 
-out_entry_check:
+//out_entry_check:
     /* If we've reduced our "liabilities" beyond our "assets", free some */
     if ( p2m->pod.entry_count < p2m->pod.count )
     {

@@ -1040,6 +1146,8 @@ p2m_pod_zero_check_superpage(struct p2m_
     p2m_pod_cache_add(p2m, mfn_to_page(mfn0), 9);
     p2m->pod.entry_count += SUPERPAGE_PAGES;
 
+    ret = SUPERPAGE_PAGES;
+
 out_reset:
     if ( reset )
         set_p2m_entry(p2m, gfn, mfn0, 9, type0, p2m->default_access);

@@ -87,8 +87,8 @@ static ssize_t rdexact(xc_interface *xch
         if ( (len == -1) && ((errno == EINTR) || (errno == EAGAIN)) )
             continue;
         if ( len == 0 ) {
-            ERROR("0-length read");
-            errno = 0;
+            errno = EPIPE;
+            return -1;
         }
         if ( len <= 0 ) {
             ERROR("read_exact_timed failed (read rc: %d, errno: %d, size %d, offset %d)",

@@ -1253,7 +1253,7 @@ int xc_domain_restore(xc_interface *xch,
         if ( !ctx->completed ) {
             pagebuf.nr_physpages = pagebuf.nr_pages = 0;
             if ( pagebuf_get_one(xch, ctx, &pagebuf, io_fd, dom) < 0 ) {
-	        DPERROR(tdom, "Error when reading batch");
+	        /* pagebuf_get_one already returned a proper error */
                 goto out;
             }
         }