p2m fine-grained concurrency control

diff -r a33af75083c7 -r 25b9a9966368 xen/arch/x86/mm/mm-locks.h
@@ -28,6 +28,7 @@
 
 /* Per-CPU variable for enforcing the lock ordering */
 DECLARE_PER_CPU(int, mm_lock_level);
+#define __get_lock_level()  (this_cpu(mm_lock_level))
 
 static inline void mm_lock_init(mm_lock_t *l)
 {

@@ -42,22 +43,32 @@ static inline int mm_locked_by_me(mm_loc
     return (l->lock.recurse_cpu == current->processor);
 }
 
+/* If you see this crash, the numbers printed are lines in this file 
+ * where the offending locks are declared. */
+#define __check_lock_level(l)                           \
+do {                                                    \
+    if ( unlikely(__get_lock_level()) > (l) )           \
+        panic("mm locking order violation: %i > %i\n",  \
+              __get_lock_level(), (l));                 \
+} while(0)
+
+#define __set_lock_level(l)         \
+do {                                \
+    __get_lock_level() = (l);       \
+} while(0)
+
 static inline void _mm_lock(mm_lock_t *l, const char *func, int level, int rec)
 {
-    /* If you see this crash, the numbers printed are lines in this file 
-     * where the offending locks are declared. */
-    if ( unlikely(this_cpu(mm_lock_level) > level) )
-        panic("mm locking order violation: %i > %i\n", 
-              this_cpu(mm_lock_level), level);
+    __check_lock_level(level);
     spin_lock_recursive(&l->lock);
     if ( l->lock.recurse_cnt == 1 )
     {
         l->locker_function = func;
-        l->unlock_level = this_cpu(mm_lock_level);
+        l->unlock_level = __get_lock_level();
     }
     else if ( (unlikely(!rec)) )
         panic("mm lock already held by %s\n", l->locker_function);
-    this_cpu(mm_lock_level) = level;
+    __set_lock_level(level);
 }
 /* This wrapper uses the line number to express the locking order below */
 #define declare_mm_lock(name)                                                 \

@@ -72,7 +83,7 @@ static inline void mm_unlock(mm_lock_t *
     if ( l->lock.recurse_cnt == 1 )
     {
         l->locker_function = "nobody";
-        this_cpu(mm_lock_level) = l->unlock_level;
+        __set_lock_level(l->unlock_level);
     }
     spin_unlock_recursive(&l->lock);
 }

diff -r 25b9a9966368 -r c915609e4235 xen/arch/x86/mm/mm-locks.h
@@ -70,6 +70,18 @@ static inline void _mm_lock(mm_lock_t *l
         panic("mm lock already held by %s\n", l->locker_function);
     __set_lock_level(level);
 }
+
+static inline void _mm_enforce_order_lock_pre(int level)
+{
+    __check_lock_level(level);
+}
+
+static inline void _mm_enforce_order_lock_post(int level, int *unlock_level)
+{
+    *unlock_level = __get_lock_level();
+    __set_lock_level(level);
+}
+
 /* This wrapper uses the line number to express the locking order below */
 #define declare_mm_lock(name)                                                 \
     static inline void mm_lock_##name(mm_lock_t *l, const char *func, int rec)\

@@ -78,6 +90,16 @@ static inline void _mm_lock(mm_lock_t *l
 #define mm_lock(name, l) mm_lock_##name(l, __func__, 0)
 #define mm_lock_recursive(name, l) mm_lock_##name(l, __func__, 1)
 
+/* This wrapper is intended for "external" locks which do not use
+ * the mm_lock_t types. Such locks inside the mm code are also subject
+ * to ordering constraints. They cannot be recursive (yet, additional
+ * bookkepping is necessary) */
+#define declare_mm_order_constraint(name)                                   \
+    static inline void mm_enforce_order_lock_pre_##name(void)               \
+    { _mm_enforce_order_lock_pre(__LINE__); }                               \
+    static inline void mm_enforce_order_lock_post_##name(int *unlock_level) \
+    { _mm_enforce_order_lock_post(__LINE__, unlock_level); }                \
+
 static inline void mm_unlock(mm_lock_t *l)
 {
     if ( l->lock.recurse_cnt == 1 )

@@ -88,6 +110,11 @@ static inline void mm_unlock(mm_lock_t *
     spin_unlock_recursive(&l->lock);
 }
 
+static inline void mm_enforce_order_unlock(int unlock_level)
+{
+    __set_lock_level(unlock_level);
+}
+
 /************************************************************************
  *                                                                      *
  * To avoid deadlocks, these locks _MUST_ be taken in the order they're *

diff -r c915609e4235 -r 332775f72a30 xen/arch/x86/mm/mm-locks.h
@@ -155,6 +155,17 @@ declare_mm_lock(p2m)
 #define p2m_unlock(p)         mm_unlock(&(p)->lock)
 #define p2m_locked_by_me(p)   mm_locked_by_me(&(p)->lock)
 
+/* Page alloc lock (per-domain)
+ *
+ * This is an external lock, not represented by an mm_lock_t. However, 
+ * pod code uses it in conjunction with the p2m lock, and expecting
+ * the ordering which we enforce here */
+
+declare_mm_order_constraint(page_alloc)
+#define page_alloc_mm_pre_lock()   mm_enforce_order_lock_pre_page_alloc()
+#define page_alloc_mm_post_lock(l) mm_enforce_order_lock_post_page_alloc(&(l))
+#define page_alloc_mm_unlock(l)    mm_enforce_order_unlock((l))
+
 /* Paging lock (per-domain)
  *
  * For shadow pagetables, this lock protects

diff -r c915609e4235 -r 332775f72a30 xen/arch/x86/mm/p2m-pod.c
@@ -45,6 +45,20 @@
 
 #define superpage_aligned(_x)  (((_x)&(SUPERPAGE_PAGES-1))==0)
 
+/* Enforce lock ordering when grabbing the "external" page_alloc lock */
+static inline void lock_page_alloc(struct p2m_domain *p2m)
+{
+    page_alloc_mm_pre_lock();
+    spin_lock(&(p2m->domain->page_alloc_lock));
+    page_alloc_mm_post_lock(p2m->pod.page_alloc_unlock_level);
+}
+
+static inline void unlock_page_alloc(struct p2m_domain *p2m)
+{
+    page_alloc_mm_unlock(p2m->pod.page_alloc_unlock_level);
+    spin_unlock(&(p2m->domain->page_alloc_lock));
+}
+
 /*
  * Populate-on-demand functionality
  */

@@ -100,7 +114,7 @@ p2m_pod_cache_add(struct p2m_domain *p2m
         unmap_domain_page(b);
     }
 
-    spin_lock(&d->page_alloc_lock);
+    lock_page_alloc(p2m);
 
     /* First, take all pages off the domain list */
     for(i=0; i < 1 << order ; i++)

@@ -128,7 +142,7 @@ p2m_pod_cache_add(struct p2m_domain *p2m
      * This may cause "zombie domains" since the page will never be freed. */
     BUG_ON( d->arch.relmem != RELMEM_not_started );
 
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
 
     return 0;
 }

@@ -245,7 +259,7 @@ p2m_pod_set_cache_target(struct p2m_doma
 
         /* Grab the lock before checking that pod.super is empty, or the last
          * entries may disappear before we grab the lock. */
-        spin_lock(&d->page_alloc_lock);
+        lock_page_alloc(p2m);
 
         if ( (p2m->pod.count - pod_target) > SUPERPAGE_PAGES
              && !page_list_empty(&p2m->pod.super) )

@@ -257,7 +271,7 @@ p2m_pod_set_cache_target(struct p2m_doma
 
         ASSERT(page != NULL);
 
-        spin_unlock(&d->page_alloc_lock);
+        unlock_page_alloc(p2m);
 
         /* Then free them */
         for ( i = 0 ; i < (1 << order) ; i++ )

@@ -378,7 +392,7 @@ p2m_pod_empty_cache(struct domain *d)
     BUG_ON(!d->is_dying);
     spin_barrier(&p2m->lock.lock);
 
-    spin_lock(&d->page_alloc_lock);
+    lock_page_alloc(p2m);
 
     while ( (page = page_list_remove_head(&p2m->pod.super)) )
     {

@@ -403,7 +417,7 @@ p2m_pod_empty_cache(struct domain *d)
 
     BUG_ON(p2m->pod.count != 0);
 
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
 }
 
 int

@@ -417,7 +431,7 @@ p2m_pod_offline_or_broken_hit(struct pag
     if ( !(d = page_get_owner(p)) || !(p2m = p2m_get_hostp2m(d)) )
         return 0;
 
-    spin_lock(&d->page_alloc_lock);
+    lock_page_alloc(p2m);
     bmfn = mfn_x(page_to_mfn(p));
     page_list_for_each_safe(q, tmp, &p2m->pod.super)
     {

@@ -448,12 +462,12 @@ p2m_pod_offline_or_broken_hit(struct pag
         }
     }
 
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
     return 0;
 
 pod_hit:
     page_list_add_tail(p, &d->arch.relmem_list);
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
     return 1;
 }
 

@@ -994,7 +1008,7 @@ p2m_pod_demand_populate(struct p2m_domai
     if ( q == p2m_guest && gfn > p2m->pod.max_guest )
         p2m->pod.max_guest = gfn;
 
-    spin_lock(&d->page_alloc_lock);
+    lock_page_alloc(p2m);
 
     if ( p2m->pod.count == 0 )
         goto out_of_memory;

@@ -1008,7 +1022,7 @@ p2m_pod_demand_populate(struct p2m_domai
 
     BUG_ON((mfn_x(mfn) & ((1 << order)-1)) != 0);
 
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
 
     gfn_aligned = (gfn >> order) << order;
 

@@ -1040,7 +1054,7 @@ p2m_pod_demand_populate(struct p2m_domai
 
     return 0;
 out_of_memory:
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
 
     printk("%s: Out of populate-on-demand memory! tot_pages %" PRIu32 " pod_entries %" PRIi32 "\n",
            __func__, d->tot_pages, p2m->pod.entry_count);

@@ -1049,7 +1063,7 @@ out_fail:
     return -1;
 remap_and_retry:
     BUG_ON(order != PAGE_ORDER_2M);
-    spin_unlock(&d->page_alloc_lock);
+    unlock_page_alloc(p2m);
 
     /* Remap this 2-meg region in singleton chunks */
     gfn_aligned = (gfn>>order)<<order;

diff -r c915609e4235 -r 332775f72a30 xen/include/asm-x86/p2m.h
@@ -270,6 +270,10 @@ struct p2m_domain {
      * + p2m_pod_demand_populate() grabs both; the p2m lock to avoid
      *   double-demand-populating of pages, the page_alloc lock to
      *   protect moving stuff from the PoD cache to the domain page list.
+     *
+     * We enforce this lock ordering through a construct in mm-locks.h.
+     * This demands, however, that we store the previous lock-ordering
+     * level in effect before grabbing the page_alloc lock.
      */
     struct {
         struct page_list_head super,   /* List of superpages                */

@@ -279,6 +283,7 @@ struct p2m_domain {
         unsigned         reclaim_super; /* Last gpfn of a scan */
         unsigned         reclaim_single; /* Last gpfn of a scan */
         unsigned         max_guest;    /* gpfn of max guest demand-populate */
+        int              page_alloc_unlock_level; /* To enforce lock ordering */
     } pod;
 };

diff -r 332775f72a30 -r 981073d78f7f xen/arch/x86/mm/mm-locks.h
@@ -155,6 +155,15 @@ declare_mm_lock(p2m)
 #define p2m_unlock(p)         mm_unlock(&(p)->lock)
 #define p2m_locked_by_me(p)   mm_locked_by_me(&(p)->lock)
 
+/* PoD lock (per-p2m-table)
+ * 
+ * Protects private PoD data structs. */
+
+declare_mm_lock(pod)
+#define pod_lock(p)           mm_lock(pod, &(p)->pod.lock)
+#define pod_unlock(p)         mm_unlock(&(p)->pod.lock)
+#define pod_locked_by_me(p)   mm_locked_by_me(&(p)->pod.lock)
+
 /* Page alloc lock (per-domain)
  *
  * This is an external lock, not represented by an mm_lock_t. However, 

424 lines not on this page · raw message

diff -r 332775f72a30 -r 981073d78f7f xen/arch/x86/mm/p2m-pt.c
@@ -1001,6 +1001,7 @@ void audit_p2m(struct p2m_domain *p2m, i
     if ( !paging_mode_translate(d) )
         return;
 
+    pod_lock(p2m);
     //P2M_PRINTK("p2m audit starts\n");
 
     test_linear = ( (d == current->domain)

@@ -1247,6 +1248,8 @@ void audit_p2m(struct p2m_domain *p2m, i
                    pmbad, mpbad);
         WARN();
     }
+
+    pod_unlock(p2m);
 }
 #endif /* P2M_AUDIT */
 

diff -r 332775f72a30 -r 981073d78f7f xen/arch/x86/mm/p2m.c
@@ -72,6 +72,7 @@ boolean_param("hap_2mb", opt_hap_2mb);
 static void p2m_initialise(struct domain *d, struct p2m_domain *p2m)
 {
     mm_lock_init(&p2m->lock);
+    mm_lock_init(&p2m->pod.lock);
     INIT_LIST_HEAD(&p2m->np2m_list);
     INIT_PAGE_LIST_HEAD(&p2m->pages);
     INIT_PAGE_LIST_HEAD(&p2m->pod.super);

@@ -506,8 +507,10 @@ guest_physmap_add_entry(struct domain *d
             rc = -EINVAL;
         else
         {
-            p2m->pod.entry_count -= pod_count; /* Lock: p2m */
+            pod_lock(p2m);
+            p2m->pod.entry_count -= pod_count;
             BUG_ON(p2m->pod.entry_count < 0);
+            pod_unlock(p2m);
         }
     }
 

@@ -1125,8 +1128,10 @@ p2m_flush_table(struct p2m_domain *p2m)
     /* "Host" p2m tables can have shared entries &c that need a bit more 
      * care when discarding them */
     ASSERT(p2m_is_nestedp2m(p2m));
+    pod_lock(p2m);
     ASSERT(page_list_empty(&p2m->pod.super));
     ASSERT(page_list_empty(&p2m->pod.single));
+    pod_unlock(p2m);
 
     /* This is no longer a valid nested p2m for any address space */
     p2m->cr3 = CR3_EADDR;

diff -r 332775f72a30 -r 981073d78f7f xen/include/asm-x86/p2m.h
@@ -257,24 +257,13 @@ struct p2m_domain {
     unsigned long max_mapped_pfn;
 
     /* Populate-on-demand variables
-     * NB on locking.  {super,single,count} are
-     * covered by d->page_alloc_lock, since they're almost always used in
-     * conjunction with that functionality.  {entry_count} is covered by
-     * the domain p2m lock, since it's almost always used in conjunction
-     * with changing the p2m tables.
      *
-     * At this point, both locks are held in two places.  In both,
-     * the order is [p2m,page_alloc]:
-     * + p2m_pod_decrease_reservation() calls p2m_pod_cache_add(),
-     *   which grabs page_alloc
-     * + p2m_pod_demand_populate() grabs both; the p2m lock to avoid
-     *   double-demand-populating of pages, the page_alloc lock to
-     *   protect moving stuff from the PoD cache to the domain page list.
-     *
-     * We enforce this lock ordering through a construct in mm-locks.h.
-     * This demands, however, that we store the previous lock-ordering
-     * level in effect before grabbing the page_alloc lock.
-     */
+     * All variables are protected with the pod lock. We cannot rely on
+     * the p2m lock if it's turned into a fine-grained lock.
+     * We only use the domain page_alloc lock for additions and 
+     * deletions to the domain's page list. Because we use it nested
+     * within the PoD lock, we enforce it's ordering (by remembering
+     * the unlock level). */
     struct {
         struct page_list_head super,   /* List of superpages                */
                          single;       /* Non-super lists                   */

@@ -283,6 +272,8 @@ struct p2m_domain {
         unsigned         reclaim_super; /* Last gpfn of a scan */
         unsigned         reclaim_single; /* Last gpfn of a scan */
         unsigned         max_guest;    /* gpfn of max guest demand-populate */
+        mm_lock_t        lock;         /* Locking of private pod structs,   *
+                                        * not relying on the p2m lock.      */
         int              page_alloc_unlock_level; /* To enforce lock ordering */
     } pod;
 };

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/hap/private.h
@@ -21,6 +21,7 @@
 #define __HAP_PRIVATE_H__
 
 #include "../mm-locks.h"
+#include "../p2m-lock.h"
 
 /********************************************/
 /*          GUEST TRANSLATION FUNCS         */

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/mm-locks.h
@@ -146,14 +146,22 @@ declare_mm_lock(nestedp2m)
 
 /* P2M lock (per-p2m-table)
  * 
- * This protects all updates to the p2m table.  Updates are expected to
- * be safe against concurrent reads, which do *not* require the lock. */
+ * This protects all updates to the p2m table.
+ * 
+ * In 64 bit mode we disable this because the lock becomes fine-grained,
+ * and several code paths cause inversion/deadlock:
+ *   -- PoD sweeps
+ *   -- mem_sharing_unshare_page
+ *   -- generally widespread recursive locking, which we don't support
+ *      (yet, I guess) on an "external" mm lock. */
 
+#ifndef __x86_64__
 declare_mm_lock(p2m)
-#define p2m_lock(p)           mm_lock(p2m, &(p)->lock)
-#define p2m_lock_recursive(p) mm_lock_recursive(p2m, &(p)->lock)
-#define p2m_unlock(p)         mm_unlock(&(p)->lock)
-#define p2m_locked_by_me(p)   mm_locked_by_me(&(p)->lock)
+#define _p2m_lock(p)           mm_lock(p2m, &(p)->lock)
+#define _p2m_lock_recursive(p) mm_lock_recursive(p2m, &(p)->lock)
+#define _p2m_unlock(p)         mm_unlock(&(p)->lock)
+#define _p2m_locked_by_me(p)   mm_locked_by_me(&(p)->lock)
+#endif /* __x86_64__ */
 
 /* PoD lock (per-p2m-table)
  * 

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-ept.c
@@ -33,6 +33,7 @@
 #include <xen/softirq.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 #define atomic_read_ept_entry(__pepte)                              \
     ( (ept_entry_t) { .epte = atomic_read64(&(__pepte)->epte) } )

613 lines not on this page · raw message

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-pod.c
@@ -34,6 +34,7 @@
 #include <asm/hvm/svm/amd-iommu-proto.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 /* Override macros from asm/page.h to make them work with mfn_t */
 #undef mfn_to_page

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m-pt.c
@@ -39,6 +39,7 @@
 #include <asm/hvm/svm/amd-iommu-proto.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 /* Override macros from asm/page.h to make them work with mfn_t */
 #undef mfn_to_page

diff -r 981073d78f7f -r a23e1262b124 xen/arch/x86/mm/p2m.c
@@ -38,6 +38,7 @@
 #include <asm/hvm/svm/amd-iommu-proto.h>
 
 #include "mm-locks.h"
+#include "p2m-lock.h"
 
 /* turn on/off 1GB host page table support for hap, default on */
 static bool_t __read_mostly opt_hap_1gb = 1;

@@ -69,9 +70,12 @@ boolean_param("hap_2mb", opt_hap_2mb);
 
 
 /* Init the datastructures for later use by the p2m code */
-static void p2m_initialise(struct domain *d, struct p2m_domain *p2m)
+static int p2m_initialise(struct domain *d, struct p2m_domain *p2m)
 {
-    mm_lock_init(&p2m->lock);
+    if (p2m_lock_init(p2m))
+    {
+        return -ENOMEM;
+    }
     mm_lock_init(&p2m->pod.lock);
     INIT_LIST_HEAD(&p2m->np2m_list);
     INIT_PAGE_LIST_HEAD(&p2m->pages);

@@ -89,7 +93,7 @@ static void p2m_initialise(struct domain
     else
         p2m_pt_init(p2m);
 
-    return;
+    return 0;
 }
 
 static int

@@ -103,7 +107,11 @@ p2m_init_nestedp2m(struct domain *d)
         d->arch.nested_p2m[i] = p2m = xzalloc(struct p2m_domain);
         if (p2m == NULL)
             return -ENOMEM;
-        p2m_initialise(d, p2m);
+        if (p2m_initialise(d, p2m))
+        {
+            xfree(p2m);
+            return -ENOMEM;
+        }
         p2m->write_p2m_entry = nestedp2m_write_p2m_entry;
         list_add(&p2m->np2m_list, &p2m_get_hostp2m(d)->np2m_list);
     }

@@ -118,7 +126,11 @@ int p2m_init(struct domain *d)
     p2m_get_hostp2m(d) = p2m = xzalloc(struct p2m_domain);
     if ( p2m == NULL )
         return -ENOMEM;
-    p2m_initialise(d, p2m);
+    if (p2m_initialise(d, p2m))
+    {
+        xfree(p2m);
+        return -ENOMEM;
+    }
 
     /* Must initialise nestedp2m unconditionally
      * since nestedhvm_enabled(d) returns false here.

@@ -331,6 +343,7 @@ static void p2m_teardown_nestedp2m(struc
     uint8_t i;
 
     for (i = 0; i < MAX_NESTEDP2M; i++) {
+        p2m_lock_destroy(d->arch.nested_p2m[i]);
         xfree(d->arch.nested_p2m[i]);
         d->arch.nested_p2m[i] = NULL;
     }

@@ -338,6 +351,7 @@ static void p2m_teardown_nestedp2m(struc
 
 void p2m_final_teardown(struct domain *d)
 {
+    p2m_lock_destroy(d->arch.p2m); 
     /* Iterate over all p2m tables per domain */
     xfree(d->arch.p2m);
     d->arch.p2m = NULL;

diff -r 981073d78f7f -r a23e1262b124 xen/include/asm-x86/p2m.h
@@ -187,9 +187,10 @@ typedef enum {
 #define p2m_is_broken(_t)   (p2m_to_mask(_t) & P2M_BROKEN_TYPES)
 
 /* Per-p2m-table state */
+struct __p2m_lock;
 struct p2m_domain {
     /* Lock that protects updates to the p2m */
-    mm_lock_t          lock;
+    struct __p2m_lock *lock;
 
     /* Shadow translated domain: p2m mapping */
     pagetable_t        phys_table;

diff -r a23e1262b124 -r 0a97d62c2d41 xen/arch/x86/mm/p2m-ept.c
@@ -163,6 +163,7 @@ static int ept_set_middle_entry(struct p
 }
 
 /* free ept sub tree behind an entry */
+/* Lock on this superpage (if any) held on entry */
 static void ept_free_entry(struct p2m_domain *p2m, ept_entry_t *ept_entry, int level)
 {
     /* End if the entry is a leaf entry. */

@@ -181,6 +182,7 @@ static void ept_free_entry(struct p2m_do
     p2m_free_ptp(p2m, mfn_to_page(ept_entry->mfn));
 }
 
+/* Lock on this superpage held on entry */
 static int ept_split_super_page(struct p2m_domain *p2m, ept_entry_t *ept_entry,
                                 int level, int target)
 {

@@ -315,6 +317,7 @@ ept_set_entry(struct p2m_domain *p2m, un
     int needs_sync = 1;
     struct domain *d = p2m->domain;
     ept_entry_t old_entry = { .epte = 0 };
+    p2m_lock_t *p2ml = p2m->lock;
 
     /*
      * the caller must make sure:

@@ -361,6 +364,8 @@ ept_set_entry(struct p2m_domain *p2m, un
      * with a leaf entry (a 1GiB or 2MiB page), and handle things appropriately.
      */
 
+    if ( target == 2 )
+        lock_p2m_1G(p2ml, index);
     if ( i == target )
     {
         /* We reached the target level. */

@@ -373,7 +378,7 @@ ept_set_entry(struct p2m_domain *p2m, un
         /* If we're replacing a non-leaf entry with a leaf entry (1GiB or 2MiB),
          * the intermediate tables will be freed below after the ept flush
          *
-         * Read-then-write is OK because we hold the p2m lock. */
+         * Read-then-write is OK because we hold the 1G or 2M lock. */
         old_entry = *ept_entry;
 
         if ( mfn_valid(mfn_x(mfn)) || direct_mmio || p2m_is_paged(p2mt) ||

@@ -412,6 +417,8 @@ ept_set_entry(struct p2m_domain *p2m, un
         if ( !ept_split_super_page(p2m, &split_ept_entry, i, target) )
         {
             ept_free_entry(p2m, &split_ept_entry, i);
+            if ( target == 2 )
+                unlock_p2m_1G(p2ml, index);
             goto out;
         }
 

@@ -440,7 +447,7 @@ ept_set_entry(struct p2m_domain *p2m, un
         /* the caller should take care of the previous page */
         new_entry.mfn = mfn_x(mfn);
 
-        /* Safe to read-then-write because we hold the p2m lock */
+        /* Safe to read-then-write because we hold the 1G or 2M lock */
         if ( ept_entry->mfn == new_entry.mfn )
              need_modify_vtd_table = 0;
 

@@ -448,6 +455,8 @@ ept_set_entry(struct p2m_domain *p2m, un
 
         atomic_write_ept_entry(ept_entry, new_entry);
     }
+    if ( target == 2 )
+        unlock_p2m_1G(p2ml, index);
 
     /* Track the highest gfn for which we have ever had a valid mapping */
     if ( mfn_valid(mfn_x(mfn)) &&

@@ -642,6 +651,8 @@ static ept_entry_t ept_get_entry_content
     return content;
 }
 
+/* This is called before crashing a domain, so we're not particularly 
+ * concerned with locking */
 void ept_walk_table(struct domain *d, unsigned long gfn)
 {
     struct p2m_domain *p2m = p2m_get_hostp2m(d);

diff -r a23e1262b124 -r 0a97d62c2d41 xen/arch/x86/mm/p2m-lock.h
@@ -109,6 +109,11 @@ typedef struct __p2m_lock {
     mm_lock_t lock;
 } p2m_lock_t;
 
+/* We do not need sub-locking on 1G superpages because we have a 
+ * global lock */
+#define lock_p2m_1G(l, gfn)     ((void)l) 
+#define unlock_p2m_1G(l, gfn)   ((void)l)
+
 static inline int p2m_lock_init(struct p2m_domain *p2m)
 {
     p2m_lock_t *p2ml = xmalloc(p2m_lock_t);

@@ -271,7 +276,8 @@ typedef struct __p2m_lock
     /* To enforce ordering in mm-locks */
     int unlock_level;
     /* To protect on-demand allocation of locks 
-     * (yeah you heard that right) */
+     * (yeah you heard that right) 
+     * Also protects 1GB superpage splitting. */
     spinlock_t alloc_lock;
     /* Global lock */
     p2m_inner_lock_t global;

@@ -429,6 +435,9 @@ static inline void put_p2m_2m(struct p2m
     unlock_p2m_leaf(__get_2m_lock(p2m->lock, gfn_1g, gfn_2m));
 }
 
+#define lock_p2m_1G(l, gfn)     spin_lock(&l->alloc_lock)
+#define unlock_p2m_1G(l, gfn)   spin_unlock(&l->alloc_lock)
+
 /* Allocate 2M locks we may not have allocated yet for this 1G superpage */
 static inline int alloc_locks_2m(struct p2m_domain *p2m, unsigned long gfn_1g)
 {

186 lines not on this page · raw message

diff -r 0a97d62c2d41 -r 8a98179666de xen/arch/x86/mm/p2m.c
@@ -148,6 +148,44 @@ void p2m_change_entry_type_global(struct
     p2m_unlock(p2m);
 }
 
+mfn_t gfn_to_mfn_type_p2m(struct p2m_domain *p2m, unsigned long gfn,
+                    p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
+                    unsigned int *page_order)
+{
+    mfn_t mfn;
+
+    if ( !p2m || !paging_mode_translate(p2m->domain) )
+    {
+        /* Not necessarily true, but for non-translated guests, we claim
+         * it's the most generic kind of memory */
+        *t = p2m_ram_rw;
+        return _mfn(gfn);
+    }
+
+    mfn = p2m->get_entry(p2m, gfn, t, a, q, page_order);
+
+#ifdef __x86_64__
+    if ( q == p2m_unshare && p2m_is_shared(*t) )
+    {
+        ASSERT(!p2m_is_nestedp2m(p2m));
+        mem_sharing_unshare_page(p2m->domain, gfn, 0);
+        mfn = p2m->get_entry(p2m, gfn, t, a, q, page_order);
+    }
+#endif
+
+#ifdef __x86_64__
+    if (unlikely((p2m_is_broken(*t))))
+    {
+        /* Return invalid_mfn to avoid caller's access */
+        mfn = _mfn(INVALID_MFN);
+        if (q == p2m_guest)
+            domain_crash(p2m->domain);
+    }
+#endif
+
+    return mfn;
+}
+
 int set_p2m_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn, 
                   unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma)
 {

diff -r 0a97d62c2d41 -r 8a98179666de xen/include/asm-x86/p2m.h
@@ -305,45 +305,9 @@ struct p2m_domain *p2m_get_p2m(struct vc
  * If the lookup succeeds, the return value is != INVALID_MFN and 
  * *page_order is filled in with the order of the superpage (if any) that
  * the entry was found in.  */
-static inline mfn_t
-gfn_to_mfn_type_p2m(struct p2m_domain *p2m, unsigned long gfn,
+mfn_t gfn_to_mfn_type_p2m(struct p2m_domain *p2m, unsigned long gfn,
                     p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
-                    unsigned int *page_order)
-{
-    mfn_t mfn;
-
-    if ( !p2m || !paging_mode_translate(p2m->domain) )
-    {
-        /* Not necessarily true, but for non-translated guests, we claim
-         * it's the most generic kind of memory */
-        *t = p2m_ram_rw;
-        return _mfn(gfn);
-    }
-
-    mfn = p2m->get_entry(p2m, gfn, t, a, q, page_order);
-
-#ifdef __x86_64__
-    if ( q == p2m_unshare && p2m_is_shared(*t) )
-    {
-        ASSERT(!p2m_is_nestedp2m(p2m));
-        mem_sharing_unshare_page(p2m->domain, gfn, 0);
-        mfn = p2m->get_entry(p2m, gfn, t, a, q, page_order);
-    }
-#endif
-
-#ifdef __x86_64__
-    if (unlikely((p2m_is_broken(*t))))
-    {
-        /* Return invalid_mfn to avoid caller's access */
-        mfn = _mfn(INVALID_MFN);
-        if (q == p2m_guest)
-            domain_crash(p2m->domain);
-    }
-#endif
-
-    return mfn;
-}
-
+                    unsigned int *page_order);
 
 /* General conversion function from gfn to mfn */
 static inline mfn_t gfn_to_mfn_type(struct domain *d,

diff -r 8a98179666de -r 471d4f2754d6 xen/arch/x86/mm/hap/hap.c
@@ -861,7 +861,7 @@ hap_write_p2m_entry(struct vcpu *v, unsi
     old_flags = l1e_get_flags(*p);
 
     if ( nestedhvm_enabled(d) && (old_flags & _PAGE_PRESENT) 
-         && !p2m_get_hostp2m(d)->defer_nested_flush ) {
+         && !atomic_read(&(p2m_get_hostp2m(d)->defer_nested_flush)) ) {
         /* We are replacing a valid entry so we need to flush nested p2ms,
          * unless the only change is an increase in access rights. */
         mfn_t omfn = _mfn(l1e_get_pfn(*p));

diff -r 8a98179666de -r 471d4f2754d6 xen/arch/x86/mm/hap/nested_hap.c
@@ -105,8 +105,6 @@ nestedhap_fix_p2m(struct vcpu *v, struct
     ASSERT(p2m);
     ASSERT(p2m->set_entry);
 
-    p2m_lock(p2m);
-
     /* If this p2m table has been flushed or recycled under our feet, 
      * leave it alone.  We'll pick up the right one as we try to 
      * vmenter the guest. */

@@ -122,11 +120,13 @@ nestedhap_fix_p2m(struct vcpu *v, struct
         gfn = (L2_gpa >> PAGE_SHIFT) & mask;
         mfn = _mfn((L0_gpa >> PAGE_SHIFT) & mask);
 
+        /* Not bumping refcount of pages underneath because we're getting
+         * rid of whatever was there */
+        get_p2m(p2m, gfn, page_order);
         rv = set_p2m_entry(p2m, gfn, mfn, page_order, p2mt, p2ma);
+        put_p2m(p2m, gfn, page_order);
     }
 
-    p2m_unlock(p2m);
-
     if (rv == 0) {
         gdprintk(XENLOG_ERR,
 		"failed to set entry for 0x%"PRIx64" -> 0x%"PRIx64"\n",

@@ -146,19 +146,26 @@ nestedhap_walk_L0_p2m(struct p2m_domain 
     mfn_t mfn;
     p2m_type_t p2mt;
     p2m_access_t p2ma;
+    int rc;
 
     /* walk L0 P2M table */
     mfn = gfn_to_mfn_type_p2m(p2m, L1_gpa >> PAGE_SHIFT, &p2mt, &p2ma, 
                               p2m_query, page_order);
 
+    rc = NESTEDHVM_PAGEFAULT_ERROR;
     if ( p2m_is_paging(p2mt) || p2m_is_shared(p2mt) || !p2m_is_ram(p2mt) )
-        return NESTEDHVM_PAGEFAULT_ERROR;
+        goto out;
 
+    rc = NESTEDHVM_PAGEFAULT_ERROR;
     if ( !mfn_valid(mfn) )
-        return NESTEDHVM_PAGEFAULT_ERROR;
+        goto out;
 
     *L0_gpa = (mfn_x(mfn) << PAGE_SHIFT) + (L1_gpa & ~PAGE_MASK);
-    return NESTEDHVM_PAGEFAULT_DONE;
+    rc = NESTEDHVM_PAGEFAULT_DONE;
+
+out:
+    drop_p2m_gfn(p2m, L1_gpa >> PAGE_SHIFT, mfn_x(mfn));
+    return rc;
 }
 
 /* This function uses L2_gpa to walk the P2M page table in L1. If the 

diff -r 8a98179666de -r 471d4f2754d6 xen/arch/x86/mm/p2m-ept.c
@@ -43,29 +43,16 @@
 #define is_epte_present(ept_entry)      ((ept_entry)->epte & 0x7)
 #define is_epte_superpage(ept_entry)    ((ept_entry)->sp)
 
-/* Non-ept "lock-and-check" wrapper */
+/* Ept-specific check wrapper */
 static int ept_pod_check_and_populate(struct p2m_domain *p2m, unsigned long gfn,
                                       ept_entry_t *entry, int order,
                                       p2m_query_t q)
 {
-    int r;
-
-    /* This is called from the p2m lookups, which can happen with or 
-     * without the lock hed. */
-    p2m_lock_recursive(p2m);
-
     /* Check to make sure this is still PoD */
     if ( entry->sa_p2mt != p2m_populate_on_demand )
-    {
-        p2m_unlock(p2m);
         return 0;
-    }
 
-    r = p2m_pod_demand_populate(p2m, gfn, order, q);
-
-    p2m_unlock(p2m);
-
-    return r;
+    return p2m_pod_demand_populate(p2m, gfn, order, q);
 }
 
 static void ept_p2m_type_to_flags(ept_entry_t *entry, p2m_type_t type, p2m_access_t access)

@@ -265,9 +252,9 @@ static int ept_next_level(struct p2m_dom
 
     ept_entry = (*table) + index;
 
-    /* ept_next_level() is called (sometimes) without a lock.  Read
+    /* ept_next_level() is called (never) without a lock.  Read
      * the entry once, and act on the "cached" entry after that to
-     * avoid races. */
+     * avoid races. AAA */
     e = atomic_read_ept_entry(ept_entry);
 
     if ( !is_epte_present(&e) )

@@ -733,7 +720,8 @@ void ept_change_entry_emt_with_range(str
     int order = 0;
     struct p2m_domain *p2m = p2m_get_hostp2m(d);
 
-    p2m_lock(p2m);
+    /* This is a global operation, essentially */
+    get_p2m_global(p2m);
     for ( gfn = start_gfn; gfn <= end_gfn; gfn++ )
     {
         int level = 0;

@@ -773,7 +761,7 @@ void ept_change_entry_emt_with_range(str
                 ept_set_entry(p2m, gfn, mfn, order, e.sa_p2mt, e.access);
         }
     }
-    p2m_unlock(p2m);
+    put_p2m_global(p2m);
 }
 
 /*

132 lines not on this page · raw message

diff -r 8a98179666de -r 471d4f2754d6 xen/arch/x86/mm/p2m-pt.c
@@ -487,31 +487,16 @@ out:
 }
 
 
-/* Non-ept "lock-and-check" wrapper */
+/* PT-specific check wrapper */
 static int p2m_pod_check_and_populate(struct p2m_domain *p2m, unsigned long gfn,
                                       l1_pgentry_t *p2m_entry, int order,
                                       p2m_query_t q)
 {
-    int r;
-
-    /* This is called from the p2m lookups, which can happen with or 
-     * without the lock hed. */
-    p2m_lock_recursive(p2m);
-    audit_p2m(p2m, 1);
-
     /* Check to make sure this is still PoD */
     if ( p2m_flags_to_type(l1e_get_flags(*p2m_entry)) != p2m_populate_on_demand )
-    {
-        p2m_unlock(p2m);
         return 0;
-    }
 
-    r = p2m_pod_demand_populate(p2m, gfn, order, q);
-
-    audit_p2m(p2m, 1);
-    p2m_unlock(p2m);
-
-    return r;
+    return p2m_pod_demand_populate(p2m, gfn, order, q);
 }
 
 /* Read the current domain's p2m table (through the linear mapping). */

@@ -894,6 +879,7 @@ static void p2m_change_type_global(struc
     if ( pagetable_get_pfn(p2m_get_pagetable(p2m)) == 0 )
         return;
 
+    /* Checks for exclusive lock */
     ASSERT(p2m_locked_by_me(p2m));
 
 #if CONFIG_PAGING_LEVELS == 4

567 lines not on this page · raw message

diff -r 8a98179666de -r 471d4f2754d6 xen/include/asm-ia64/mm.h
@@ -561,6 +561,11 @@ extern u64 translate_domain_pte(u64 ptev
     ((get_gpfn_from_mfn((madr) >> PAGE_SHIFT) << PAGE_SHIFT) | \
     ((madr) & ~PAGE_MASK))
 
+/* Because x86-specific p2m fine-grained lock releases are called from common
+ * code, we put a dummy placeholder here */
+#define drop_p2m_gfn(p, g, m)           ((void)0)
+#define drop_p2m_gfn_domain(p, g, m)    ((void)0)
+
 /* Internal use only: returns 0 in case of bad address.  */
 extern unsigned long paddr_to_maddr(unsigned long paddr);
 

74 lines not on this page · raw message

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/cpu/mcheck/vmce.c
@@ -574,6 +574,7 @@ int unmmap_broken_page(struct domain *d,
 {
     mfn_t r_mfn;
     p2m_type_t pt;
+    int rc;
 
     /* Always trust dom0's MCE handler will prevent future access */
     if ( d == dom0 )

@@ -585,14 +586,16 @@ int unmmap_broken_page(struct domain *d,
     if ( !is_hvm_domain(d) || !paging_mode_hap(d) )
         return -ENOSYS;
 
+    rc = -1;
     r_mfn = gfn_to_mfn_query(d, gfn, &pt);
     if ( p2m_to_mask(pt) & P2M_UNMAP_TYPES)
     {
         ASSERT(mfn_x(r_mfn) == mfn_x(mfn));
         p2m_change_type(d, gfn, pt, p2m_ram_broken);
-        return 0;
+        rc = 0;
     }
+    drop_p2m_gfn_domain(d, gfn, mfn_x(r_mfn));
 
-    return -1;
+    return rc;
 }
 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/debug.c
@@ -45,7 +45,8 @@
 static unsigned long 
 dbg_hvm_va2mfn(dbgva_t vaddr, struct domain *dp, int toaddr)
 {
-    unsigned long mfn, gfn;
+    unsigned long gfn;
+    mfn_t mfn;
     uint32_t pfec = PFEC_page_present;
     p2m_type_t gfntype;
 

@@ -58,7 +59,7 @@ dbg_hvm_va2mfn(dbgva_t vaddr, struct dom
         return INVALID_MFN;
     }
 
-    mfn = mfn_x(gfn_to_mfn(dp, gfn, &gfntype)); 
+    mfn = gfn_to_mfn_unlocked(dp, gfn, &gfntype); 
     if ( p2m_is_readonly(gfntype) && toaddr )
     {
         DBGP2("kdb:p2m_is_readonly: gfntype:%x\n", gfntype);

@@ -66,7 +67,7 @@ dbg_hvm_va2mfn(dbgva_t vaddr, struct dom
     }
 
     DBGP2("X: vaddr:%lx domid:%d mfn:%lx\n", vaddr, dp->domain_id, mfn);
-    return mfn;
+    return mfn_x(mfn);
 }
 
 #if defined(__x86_64__)

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/domain.c
@@ -720,6 +720,7 @@ int arch_set_info_guest(
     struct vcpu *v, vcpu_guest_context_u c)
 {
     struct domain *d = v->domain;
+    unsigned long cr3_gfn;
     unsigned long cr3_pfn = INVALID_MFN;
     unsigned long flags, cr4;
     unsigned int i;

@@ -931,7 +932,8 @@ int arch_set_info_guest(
 
     if ( !compat )
     {
-        cr3_pfn = gmfn_to_mfn(d, xen_cr3_to_pfn(c.nat->ctrlreg[3]));
+        cr3_gfn = xen_cr3_to_pfn(c.nat->ctrlreg[3]);
+        cr3_pfn = gmfn_to_mfn(d, cr3_gfn);
 
         if ( !mfn_valid(cr3_pfn) ||
              (paging_mode_refcounts(d)

@@ -939,16 +941,18 @@ int arch_set_info_guest(
               : !get_page_and_type(mfn_to_page(cr3_pfn), d,
                                    PGT_base_page_table)) )
         {
+            drop_p2m_gfn_domain(d, cr3_gfn, cr3_pfn);
             destroy_gdt(v);
             return -EINVAL;
         }
 
         v->arch.guest_table = pagetable_from_pfn(cr3_pfn);
-
+        drop_p2m_gfn_domain(d, cr3_gfn, cr3_pfn);
 #ifdef __x86_64__
         if ( c.nat->ctrlreg[1] )
         {
-            cr3_pfn = gmfn_to_mfn(d, xen_cr3_to_pfn(c.nat->ctrlreg[1]));
+            cr3_gfn = xen_cr3_to_pfn(c.nat->ctrlreg[1]);
+            cr3_pfn = gmfn_to_mfn(d, cr3_gfn);
 
             if ( !mfn_valid(cr3_pfn) ||
                  (paging_mode_refcounts(d)

@@ -962,11 +966,13 @@ int arch_set_info_guest(
                     put_page(mfn_to_page(cr3_pfn));
                 else
                     put_page_and_type(mfn_to_page(cr3_pfn));
+                drop_p2m_gfn_domain(d, cr3_gfn, cr3_pfn); 
                 destroy_gdt(v);
                 return -EINVAL;
             }
 
             v->arch.guest_table_user = pagetable_from_pfn(cr3_pfn);
+            drop_p2m_gfn_domain(d, cr3_gfn, cr3_pfn); 
         }
         else if ( !(flags & VGCF_in_kernel) )
         {

@@ -978,7 +984,8 @@ int arch_set_info_guest(
     {
         l4_pgentry_t *l4tab;
 
-        cr3_pfn = gmfn_to_mfn(d, compat_cr3_to_pfn(c.cmp->ctrlreg[3]));
+        cr3_gfn = compat_cr3_to_pfn(c.cmp->ctrlreg[3]);
+        cr3_pfn = gmfn_to_mfn(d, cr3_gfn);
 
         if ( !mfn_valid(cr3_pfn) ||
              (paging_mode_refcounts(d)

@@ -986,6 +993,7 @@ int arch_set_info_guest(
               : !get_page_and_type(mfn_to_page(cr3_pfn), d,
                                    PGT_l3_page_table)) )
         {
+            drop_p2m_gfn_domain(d, cr3_gfn, cr3_pfn); 
             destroy_gdt(v);
             return -EINVAL;
         }

@@ -993,6 +1001,7 @@ int arch_set_info_guest(
         l4tab = __va(pagetable_get_paddr(v->arch.guest_table));
         *l4tab = l4e_from_pfn(
             cr3_pfn, _PAGE_PRESENT|_PAGE_RW|_PAGE_USER|_PAGE_ACCESSED);
+        drop_p2m_gfn_domain(d, cr3_gfn, cr3_pfn); 
 #endif
     }
 

@@ -1058,11 +1067,12 @@ unmap_vcpu_info(struct vcpu *v)
  * event doesn't get missed.
  */
 static int
-map_vcpu_info(struct vcpu *v, unsigned long mfn, unsigned offset)
+map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset)
 {
     struct domain *d = v->domain;
     void *mapping;
     vcpu_info_t *new_info;
+    unsigned long mfn;
     int i;
 
     if ( offset > (PAGE_SIZE - sizeof(vcpu_info_t)) )

@@ -1075,7 +1085,7 @@ map_vcpu_info(struct vcpu *v, unsigned l
     if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) )
         return -EINVAL;
 
-    mfn = gmfn_to_mfn(d, mfn);
+    mfn = gmfn_to_mfn(d, gfn);
     if ( !mfn_valid(mfn) ||
          !get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
         return -EINVAL;

@@ -1084,6 +1094,7 @@ map_vcpu_info(struct vcpu *v, unsigned l
     if ( mapping == NULL )
     {
         put_page_and_type(mfn_to_page(mfn));
+        drop_p2m_gfn_domain(d, gfn, mfn); 
         return -ENOMEM;
     }
 

@@ -1113,6 +1124,7 @@ map_vcpu_info(struct vcpu *v, unsigned l
     for ( i = 0; i < BITS_PER_EVTCHN_WORD(d); i++ )
         set_bit(i, &vcpu_info(v, evtchn_pending_sel));
 
+    drop_p2m_gfn_domain(d, gfn, mfn); 
     return 0;
 }
 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/domctl.c
@@ -235,6 +235,7 @@ long arch_do_domctl(
                         type = XEN_DOMCTL_PFINFO_XTAB;
 
                     arr[j] = type;
+                    drop_p2m_gfn_domain(d, arr[j], mfn);
                 }
 
                 if ( copy_to_guest_offset(domctl->u.getpageframeinfo3.array,

@@ -299,6 +300,7 @@ long arch_do_domctl(
             for ( j = 0; j < k; j++ )
             {      
                 struct page_info *page;
+                unsigned long gfn = arr32[j];
                 unsigned long mfn = gmfn_to_mfn(d, arr32[j]);
 
                 page = mfn_to_page(mfn);

@@ -310,8 +312,10 @@ long arch_do_domctl(
                      unlikely(is_xen_heap_mfn(mfn)) )
                     arr32[j] |= XEN_DOMCTL_PFINFO_XTAB;
                 else if ( xsm_getpageframeinfo(page) != 0 )
+                {
+                    drop_p2m_gfn_domain(d, gfn, mfn); 
                     continue;
-                else if ( likely(get_page(page, d)) )
+                } else if ( likely(get_page(page, d)) )
                 {
                     unsigned long type = 0;
 

@@ -339,6 +343,7 @@ long arch_do_domctl(
                 else
                     arr32[j] |= XEN_DOMCTL_PFINFO_XTAB;
 
+                drop_p2m_gfn_domain(d, gfn, mfn); 
             }
 
             if ( copy_to_guest_offset(domctl->u.getpageframeinfo2.array,

@@ -431,6 +436,7 @@ long arch_do_domctl(
         if ( !mfn_valid(mfn) ||
              !get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
         {
+            drop_p2m_gfn_domain(d, gmfn, mfn); 
             rcu_unlock_domain(d);
             break;
         }

@@ -443,6 +449,7 @@ long arch_do_domctl(
 
         put_page_and_type(mfn_to_page(mfn));
 
+        drop_p2m_gfn_domain(d, gmfn, mfn); 
         rcu_unlock_domain(d);
     }
     break;

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/emulate.c
@@ -66,10 +66,14 @@ static int hvmemul_do_io(
     if ( p2m_is_paging(p2mt) )
     {
         p2m_mem_paging_populate(curr->domain, ram_gfn);
+        drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
         return X86EMUL_RETRY;
     }
     if ( p2m_is_shared(p2mt) )
+    {
+        drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
         return X86EMUL_RETRY;
+    }
 
     /*
      * Weird-sized accesses have undefined behaviour: we discard writes

@@ -81,6 +85,7 @@ static int hvmemul_do_io(
         ASSERT(p_data != NULL); /* cannot happen with a REP prefix */
         if ( dir == IOREQ_READ )
             memset(p_data, ~0, size);
+        drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
         return X86EMUL_UNHANDLEABLE;
     }
 

@@ -98,7 +103,10 @@ static int hvmemul_do_io(
             paddr_t pa = curr->arch.hvm_vcpu.mmio_large_write_pa;
             unsigned int bytes = curr->arch.hvm_vcpu.mmio_large_write_bytes;
             if ( (addr >= pa) && ((addr + size) <= (pa + bytes)) )
+            {
+                drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
                 return X86EMUL_OKAY;
+            }
         }
         else
         {

@@ -108,6 +116,7 @@ static int hvmemul_do_io(
             {
                 memcpy(p_data, &curr->arch.hvm_vcpu.mmio_large_read[addr - pa],
                        size);
+                drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
                 return X86EMUL_OKAY;
             }
         }

@@ -120,15 +129,22 @@ static int hvmemul_do_io(
     case HVMIO_completed:
         curr->arch.hvm_vcpu.io_state = HVMIO_none;
         if ( p_data == NULL )
+        {
+            drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
             return X86EMUL_UNHANDLEABLE;
+        }
         goto finish_access;
     case HVMIO_dispatched:
         /* May have to wait for previous cycle of a multi-write to complete. */
         if ( is_mmio && !value_is_ptr && (dir == IOREQ_WRITE) &&
              (addr == (curr->arch.hvm_vcpu.mmio_large_write_pa +
                        curr->arch.hvm_vcpu.mmio_large_write_bytes)) )
+        {
+            drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
             return X86EMUL_RETRY;
+        }
     default:
+        drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
         return X86EMUL_UNHANDLEABLE;
     }
 

@@ -136,6 +152,7 @@ static int hvmemul_do_io(
     {
         gdprintk(XENLOG_WARNING, "WARNING: io already pending (%d)?\n",
                  p->state);
+        drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
         return X86EMUL_UNHANDLEABLE;
     }
 

@@ -186,7 +203,10 @@ static int hvmemul_do_io(
     }
 
     if ( rc != X86EMUL_OKAY )
+    {
+        drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
         return rc;
+    }
 
  finish_access:
     if ( p_data != NULL )

@@ -221,6 +241,7 @@ static int hvmemul_do_io(
         }
     }
 
+    drop_p2m_gfn_domain(curr->domain, ram_gfn, mfn_x(ram_mfn)); 
     return X86EMUL_OKAY;
 }
 

@@ -669,12 +690,12 @@ static int hvmemul_rep_movs(
     if ( rc != X86EMUL_OKAY )
         return rc;
 
-    (void)gfn_to_mfn(current->domain, sgpa >> PAGE_SHIFT, &p2mt);
+    (void)gfn_to_mfn_unlocked(current->domain, sgpa >> PAGE_SHIFT, &p2mt);
     if ( !p2m_is_ram(p2mt) && !p2m_is_grant(p2mt) )
         return hvmemul_do_mmio(
             sgpa, reps, bytes_per_rep, dgpa, IOREQ_READ, df, NULL);
 
-    (void)gfn_to_mfn(current->domain, dgpa >> PAGE_SHIFT, &p2mt);
+    (void)gfn_to_mfn_unlocked(current->domain, dgpa >> PAGE_SHIFT, &p2mt);
     if ( !p2m_is_ram(p2mt) && !p2m_is_grant(p2mt) )
         return hvmemul_do_mmio(
             dgpa, reps, bytes_per_rep, sgpa, IOREQ_WRITE, df, NULL);

371 lines not on this page · raw message

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/mtrr.c
@@ -389,7 +389,7 @@ uint32_t get_pat_flags(struct vcpu *v,
     {
         struct domain *d = v->domain;
         p2m_type_t p2mt;
-        gfn_to_mfn_query(d, paddr_to_pfn(gpaddr), &p2mt);
+        gfn_to_mfn_query_unlocked(d, paddr_to_pfn(gpaddr), &p2mt);
         if (p2m_is_ram(p2mt))
             gdprintk(XENLOG_WARNING,
                     "Conflict occurs for a given guest l1e flags:%x "

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/nestedhvm.c
@@ -56,7 +56,7 @@ nestedhvm_vcpu_reset(struct vcpu *v)
     nv->nv_ioportED = 0;
 
     if (nv->nv_vvmcx)
-        hvm_unmap_guest_frame(nv->nv_vvmcx);
+        hvm_unmap_guest_frame(nv->nv_vvmcx, nv->nv_vvmcxaddr >> PAGE_SHIFT, 0);
     nv->nv_vvmcx = NULL;
     nv->nv_vvmcxaddr = VMCX_EADDR;
     nv->nv_flushp2m = 0;

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/stdvga.c
@@ -482,7 +482,7 @@ static int mmio_move(struct hvm_hw_stdvg
                 if ( hvm_copy_to_guest_phys(data, &tmp, p->size) !=
                      HVMCOPY_okay )
                 {
-                    (void)gfn_to_mfn(d, data >> PAGE_SHIFT, &p2mt);
+                    (void)gfn_to_mfn_unlocked(d, data >> PAGE_SHIFT, &p2mt);
                     /*
                      * The only case we handle is vga_mem <-> vga_mem.
                      * Anything else disables caching and leaves it to qemu-dm.

@@ -504,7 +504,7 @@ static int mmio_move(struct hvm_hw_stdvg
                 if ( hvm_copy_from_guest_phys(&tmp, data, p->size) !=
                      HVMCOPY_okay )
                 {
-                    (void)gfn_to_mfn(d, data >> PAGE_SHIFT, &p2mt);
+                    (void)gfn_to_mfn_unlocked(d, data >> PAGE_SHIFT, &p2mt);
                     if ( (p2mt != p2m_mmio_dm) || (data < VGA_MEM_BASE) ||
                          ((data + p->size) > (VGA_MEM_BASE + VGA_MEM_SIZE)) )
                         return 0;

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/svm/nestedsvm.c
@@ -71,7 +71,7 @@ int nestedsvm_vmcb_map(struct vcpu *v, u
     if (nv->nv_vvmcx != NULL && nv->nv_vvmcxaddr != vmcbaddr) {
         ASSERT(nv->nv_vvmcx != NULL);
         ASSERT(nv->nv_vvmcxaddr != VMCX_EADDR);
-        hvm_unmap_guest_frame(nv->nv_vvmcx);
+        hvm_unmap_guest_frame(nv->nv_vvmcx, nv->nv_vvmcxaddr >> PAGE_SHIFT, 0);
         nv->nv_vvmcx = NULL;
         nv->nv_vvmcxaddr = VMCX_EADDR;
     }

@@ -353,7 +353,7 @@ static int nsvm_vmrun_permissionmap(stru
     ASSERT(ns_viomap != NULL);
     ioport_80 = test_bit(0x80, ns_viomap);
     ioport_ed = test_bit(0xed, ns_viomap);
-    hvm_unmap_guest_frame(ns_viomap);
+    hvm_unmap_guest_frame(ns_viomap, svm->ns_iomap_pa >> PAGE_SHIFT, 0);
 
     svm->ns_iomap = nestedhvm_vcpu_iomap_get(ioport_80, ioport_ed);
 

@@ -857,23 +857,25 @@ nsvm_vmcb_guest_intercepts_ioio(paddr_t 
     ioio_info_t ioinfo;
     uint16_t port;
     bool_t enabled;
+    unsigned long gfn = 0; /* gcc ... */
 
     ioinfo.bytes = exitinfo1;
     port = ioinfo.fields.port;
 
     switch (port) {
     case 0 ... 32767: /* first 4KB page */
-        io_bitmap = hvm_map_guest_frame_ro(iopm_gfn);
+        gfn = iopm_gfn;
         break;
     case 32768 ... 65535: /* second 4KB page */
         port -= 32768;
-        io_bitmap = hvm_map_guest_frame_ro(iopm_gfn+1);
+        gfn = iopm_gfn + 1;
         break;
     default:
         BUG();
         break;
     }
 
+    io_bitmap = hvm_map_guest_frame_ro(gfn);
     if (io_bitmap == NULL) {
         gdprintk(XENLOG_ERR,
             "IOIO intercept: mapping of permission map failed\n");

@@ -881,7 +883,7 @@ nsvm_vmcb_guest_intercepts_ioio(paddr_t 
     }
 
     enabled = test_bit(port, io_bitmap);
-    hvm_unmap_guest_frame(io_bitmap);
+    hvm_unmap_guest_frame(io_bitmap, gfn, 0);
     if (!enabled)
         return NESTEDHVM_VMEXIT_HOST;
 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/svm/svm.c
@@ -247,6 +247,8 @@ static int svm_vmcb_restore(struct vcpu 
             mfn = mfn_x(gfn_to_mfn(v->domain, c->cr3 >> PAGE_SHIFT, &p2mt));
             if ( !p2m_is_ram(p2mt) || !get_page(mfn_to_page(mfn), v->domain) )
             {
+                drop_p2m_gfn_domain(v->domain, 
+                                                c->cr3 >> PAGE_SHIFT, mfn);
                 gdprintk(XENLOG_ERR, "Invalid CR3 value=0x%"PRIx64"\n",
                          c->cr3);
                 return -EINVAL;

@@ -257,6 +259,10 @@ static int svm_vmcb_restore(struct vcpu 
             put_page(pagetable_get_page(v->arch.guest_table));
 
         v->arch.guest_table = pagetable_from_pfn(mfn);
+        if ( c->cr0 & X86_CR0_PG )
+        {
+            drop_p2m_gfn_domain(v->domain, c->cr3 >> PAGE_SHIFT, mfn);
+        }
     }
 
     v->arch.hvm_vcpu.guest_cr[0] = c->cr0 | X86_CR0_ET;

@@ -1160,7 +1166,9 @@ static void svm_do_nested_pgfault(struct
         p2m = p2m_get_p2m(v);
         _d.gpa = gpa;
         _d.qualification = 0;
-        _d.mfn = mfn_x(gfn_to_mfn_type_p2m(p2m, gfn, &_d.p2mt, &p2ma, p2m_query, NULL));
+        mfn = gfn_to_mfn_type_p2m(p2m, gfn, &_d.p2mt, &p2ma, p2m_query, NULL);
+        _d.mfn = mfn_x(mfn);
+        drop_p2m_gfn(p2m, gfn, mfn_x(mfn));
         
         __trace_var(TRC_HVM_NPF, 0, sizeof(_d), &_d);
     }

@@ -1184,6 +1192,7 @@ static void svm_do_nested_pgfault(struct
     gdprintk(XENLOG_ERR,
          "SVM violation gpa %#"PRIpaddr", mfn %#lx, type %i\n",
          gpa, mfn_x(mfn), p2mt);
+    drop_p2m_gfn(p2m, gfn, mfn_x(mfn));
     domain_crash(v->domain);
 }
 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/viridian.c
@@ -140,6 +140,7 @@ static void enable_hypercall_page(struct
     if ( !mfn_valid(mfn) ||
          !get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
     {
+        drop_p2m_gfn_domain(d, gmfn, mfn); 
         gdprintk(XENLOG_WARNING, "Bad GMFN %lx (MFN %lx)\n", gmfn, mfn);
         return;
     }

@@ -162,6 +163,7 @@ static void enable_hypercall_page(struct
     unmap_domain_page(p);
 
     put_page_and_type(mfn_to_page(mfn));
+    drop_p2m_gfn_domain(d, gmfn, mfn); 
 }
 
 void initialize_apic_assist(struct vcpu *v)

@@ -184,6 +186,7 @@ void initialize_apic_assist(struct vcpu 
     if ( !mfn_valid(mfn) ||
          !get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
     {
+        drop_p2m_gfn_domain(d, gmfn, mfn); 
         gdprintk(XENLOG_WARNING, "Bad GMFN %lx (MFN %lx)\n", gmfn, mfn);
         return;
     }

@@ -195,6 +198,7 @@ void initialize_apic_assist(struct vcpu 
     unmap_domain_page(p);
 
     put_page_and_type(mfn_to_page(mfn));
+    drop_p2m_gfn_domain(d, gmfn, mfn); 
 }
 
 int wrmsr_viridian_regs(uint32_t idx, uint64_t val)

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/vmx/vmx.c
@@ -490,6 +490,7 @@ static int vmx_restore_cr0_cr3(
             mfn = mfn_x(gfn_to_mfn(v->domain, cr3 >> PAGE_SHIFT, &p2mt));
             if ( !p2m_is_ram(p2mt) || !get_page(mfn_to_page(mfn), v->domain) )
             {
+                drop_p2m_gfn_domain(v->domain, cr3 >> PAGE_SHIFT, mfn);
                 gdprintk(XENLOG_ERR, "Invalid CR3 value=0x%lx\n", cr3);
                 return -EINVAL;
             }

@@ -499,6 +500,10 @@ static int vmx_restore_cr0_cr3(
             put_page(pagetable_get_page(v->arch.guest_table));
 
         v->arch.guest_table = pagetable_from_pfn(mfn);
+        if ( cr0 & X86_CR0_PG )
+        {
+            drop_p2m_gfn_domain(v->domain, cr3 >> PAGE_SHIFT, mfn);
+        }
     }
 
     v->arch.hvm_vcpu.guest_cr[0] = cr0 | X86_CR0_ET;

@@ -1009,7 +1014,10 @@ static void vmx_load_pdptrs(struct vcpu 
 
     mfn = mfn_x(gfn_to_mfn(v->domain, cr3 >> PAGE_SHIFT, &p2mt));
     if ( !p2m_is_ram(p2mt) )
+    {
+        drop_p2m_gfn_domain(v->domain, cr3 >> PAGE_SHIFT, mfn);
         goto crash;
+    }
 
     p = map_domain_page(mfn);
 

@@ -1037,6 +1045,7 @@ static void vmx_load_pdptrs(struct vcpu 
     vmx_vmcs_exit(v);
 
     unmap_domain_page(p);
+    drop_p2m_gfn_domain(v->domain, cr3 >> PAGE_SHIFT, mfn);
     return;
 
  crash:

@@ -2088,7 +2097,7 @@ static void ept_handle_violation(unsigne
 
         _d.gpa = gpa;
         _d.qualification = qualification;
-        _d.mfn = mfn_x(gfn_to_mfn_query(d, gfn, &_d.p2mt));
+        _d.mfn = mfn_x(gfn_to_mfn_query_unlocked(d, gfn, &_d.p2mt));
         
         __trace_var(TRC_HVM_NPF, 0, sizeof(_d), &_d);
     }

@@ -2104,7 +2113,7 @@ static void ept_handle_violation(unsigne
         return;
 
     /* Everything else is an error. */
-    mfn = gfn_to_mfn_guest(d, gfn, &p2mt);
+    mfn = gfn_to_mfn_guest_unlocked(d, gfn, &p2mt);
     gdprintk(XENLOG_ERR, "EPT violation %#lx (%c%c%c/%c%c%c), "
              "gpa %#"PRIpaddr", mfn %#lx, type %i.\n", 
              qualification, 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/hvm/vmx/vvmx.c
@@ -558,8 +558,10 @@ static void __map_io_bitmap(struct vcpu 
 
     index = vmcs_reg == IO_BITMAP_A ? 0 : 1;
     if (nvmx->iobitmap[index])
-        hvm_unmap_guest_frame (nvmx->iobitmap[index]);
+        hvm_unmap_guest_frame (nvmx->iobitmap[index], 
+                               nvmx->iobitmap_gfn[index], 0);
     gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, vmcs_reg);
+    nvmx->iobitmap_gfn[index] = gpa >> PAGE_SHIFT;
     nvmx->iobitmap[index] = hvm_map_guest_frame_ro (gpa >> PAGE_SHIFT);
 }
 

@@ -577,13 +579,14 @@ static void nvmx_purge_vvmcs(struct vcpu
 
     __clear_current_vvmcs(v);
     if ( nvcpu->nv_vvmcxaddr != VMCX_EADDR )
-        hvm_unmap_guest_frame (nvcpu->nv_vvmcx);
+        hvm_unmap_guest_frame (nvcpu->nv_vvmcx, nvcpu->nv_vvmcxaddr >> PAGE_SHIFT, 0);
     nvcpu->nv_vvmcx == NULL;
     nvcpu->nv_vvmcxaddr = VMCX_EADDR;
     for (i=0; i<2; i++) {
         if ( nvmx->iobitmap[i] ) {
-            hvm_unmap_guest_frame (nvmx->iobitmap[i]);
+            hvm_unmap_guest_frame (nvmx->iobitmap[i], nvmx->iobitmap_gfn[i], 0);
             nvmx->iobitmap[i] = NULL;
+            nvmx->iobitmap_gfn[i] = 0;
         }
     }
 }

@@ -1198,7 +1201,7 @@ int nvmx_handle_vmclear(struct cpu_user_
         vvmcs = hvm_map_guest_frame_rw(gpa >> PAGE_SHIFT);
         if ( vvmcs ) 
             __set_vvmcs(vvmcs, NVMX_LAUNCH_STATE, 0);
-        hvm_unmap_guest_frame(vvmcs);
+        hvm_unmap_guest_frame(vvmcs, gpa >> PAGE_SHIFT, 0);
     }
 
     vmreturn(regs, VMSUCCEED);

515 lines not on this page · raw message

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/mm/guest_walk.c
@@ -86,6 +86,8 @@ static uint32_t set_ad_bits(void *guest_
     return 0;
 }
 
+/* We leave this function with a lock on the p2m and a ref on the 
+ * mapped page. Regardless of the map, you need to call drop_p2m_gfn. */
 static inline void *map_domain_gfn(struct p2m_domain *p2m,
                                    gfn_t gfn, 
                                    mfn_t *mfn,

@@ -120,6 +122,9 @@ static inline void *map_domain_gfn(struc
 
 
 /* Walk the guest pagetables, after the manner of a hardware walker. */
+/* Because the walk is essentially random, it can cause a deadlock 
+ * warning in the p2m locking code. Highly unlikely this is an actual
+ * deadlock, because who would walk page table in the opposite order? */
 uint32_t
 guest_walk_tables(struct vcpu *v, struct p2m_domain *p2m,
                   unsigned long va, walk_t *gw, 

@@ -348,11 +353,17 @@ set_ad:
  out:
 #if GUEST_PAGING_LEVELS == 4
     if ( l3p ) unmap_domain_page(l3p);
+    drop_p2m_gfn(p2m, gfn_x(guest_l4e_get_gfn(gw->l4e)), 
+                    mfn_x(gw->l3mfn));
 #endif
 #if GUEST_PAGING_LEVELS >= 3
     if ( l2p ) unmap_domain_page(l2p);
+    drop_p2m_gfn(p2m, gfn_x(guest_l3e_get_gfn(gw->l3e)), 
+                    mfn_x(gw->l2mfn));
 #endif
     if ( l1p ) unmap_domain_page(l1p);
+    drop_p2m_gfn(p2m, gfn_x(guest_l2e_get_gfn(gw->l2e)), 
+                    mfn_x(gw->l1mfn));
 
     return rc;
 }

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/mm/hap/guest_walk.c
@@ -56,9 +56,11 @@ unsigned long hap_p2m_ga_to_gfn(GUEST_PA
     p2m_type_t p2mt;
     p2m_access_t p2ma;
     walk_t gw;
+    unsigned long top_gfn;
 
     /* Get the top-level table's MFN */
-    top_mfn = gfn_to_mfn_type_p2m(p2m, cr3 >> PAGE_SHIFT, 
+    top_gfn = cr3 >> PAGE_SHIFT;
+    top_mfn = gfn_to_mfn_type_p2m(p2m, top_gfn,
                                   &p2mt, &p2ma, p2m_unshare, NULL);
     if ( p2m_is_paging(p2mt) )
     {

@@ -66,16 +68,19 @@ unsigned long hap_p2m_ga_to_gfn(GUEST_PA
         p2m_mem_paging_populate(p2m->domain, cr3 >> PAGE_SHIFT);
 
         pfec[0] = PFEC_page_paged;
+        drop_p2m_gfn(p2m, top_gfn, mfn_x(top_mfn));
         return INVALID_GFN;
     }
     if ( p2m_is_shared(p2mt) )
     {
         pfec[0] = PFEC_page_shared;
+        drop_p2m_gfn(p2m, top_gfn, mfn_x(top_mfn));
         return INVALID_GFN;
     }
     if ( !p2m_is_ram(p2mt) )
     {
         pfec[0] &= ~PFEC_page_present;
+        drop_p2m_gfn(p2m, top_gfn, mfn_x(top_mfn));
         return INVALID_GFN;
     }
 

@@ -87,26 +92,32 @@ unsigned long hap_p2m_ga_to_gfn(GUEST_PA
 #endif
     missing = guest_walk_tables(v, p2m, ga, &gw, pfec[0], top_mfn, top_map);
     unmap_domain_page(top_map);
+    drop_p2m_gfn(p2m, top_gfn, mfn_x(top_mfn));
 
     /* Interpret the answer */
     if ( missing == 0 )
     {
         gfn_t gfn = guest_l1e_get_gfn(gw.l1e);
-        gfn_to_mfn_type_p2m(p2m, gfn_x(gfn), &p2mt, &p2ma, p2m_unshare, NULL);
+        mfn_t eff_l1_mfn = gfn_to_mfn_type_p2m(p2m, gfn_x(gfn), &p2mt, 
+                                                &p2ma, p2m_unshare, NULL);
         if ( p2m_is_paging(p2mt) )
         {
             ASSERT(!p2m_is_nestedp2m(p2m));
             p2m_mem_paging_populate(p2m->domain, gfn_x(gfn));
 
             pfec[0] = PFEC_page_paged;
+            drop_p2m_gfn(p2m, gfn_x(gfn), mfn_x(eff_l1_mfn));
             return INVALID_GFN;
         }
         if ( p2m_is_shared(p2mt) )
         {
             pfec[0] = PFEC_page_shared;
+            drop_p2m_gfn(p2m, gfn_x(gfn), mfn_x(eff_l1_mfn));
             return INVALID_GFN;
         }
 
+        drop_p2m_gfn(p2m, gfn_x(gfn), mfn_x(eff_l1_mfn));
+
         if ( page_order )
             *page_order = guest_walk_to_page_order(&gw);
 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/mm/mem_event.c
@@ -47,7 +47,7 @@ static int mem_event_enable(struct domai
     unsigned long ring_addr = mec->ring_addr;
     unsigned long shared_addr = mec->shared_addr;
     l1_pgentry_t l1e;
-    unsigned long gfn;
+    unsigned long shared_gfn = 0, ring_gfn = 0; /* gcc ... */
     p2m_type_t p2mt;
     mfn_t ring_mfn;
     mfn_t shared_mfn;

@@ -60,23 +60,41 @@ static int mem_event_enable(struct domai
 
     /* Get MFN of ring page */
     guest_get_eff_l1e(v, ring_addr, &l1e);
-    gfn = l1e_get_pfn(l1e);
-    ring_mfn = gfn_to_mfn(dom_mem_event, gfn, &p2mt);
+    ring_gfn = l1e_get_pfn(l1e);
+    /* We're grabbing these two in an order that could deadlock
+     * dom0 if 1. it were an hvm 2. there were two concurrent
+     * enables 3. the two gfn's in each enable criss-crossed
+     * 2MB regions. Duly noted.... */
+    ring_mfn = gfn_to_mfn(dom_mem_event, ring_gfn, &p2mt);
 
     if ( unlikely(!mfn_valid(mfn_x(ring_mfn))) )
+    {
+        drop_p2m_gfn_domain(dom_mem_event, 
+                        ring_gfn, mfn_x(ring_mfn));
         return -EINVAL;
+    }
 
     /* Get MFN of shared page */
     guest_get_eff_l1e(v, shared_addr, &l1e);
-    gfn = l1e_get_pfn(l1e);
-    shared_mfn = gfn_to_mfn(dom_mem_event, gfn, &p2mt);
+    shared_gfn = l1e_get_pfn(l1e);
+    shared_mfn = gfn_to_mfn(dom_mem_event, shared_gfn, &p2mt);
 
     if ( unlikely(!mfn_valid(mfn_x(shared_mfn))) )
+    {
+        drop_p2m_gfn_domain(dom_mem_event, 
+                        ring_gfn, mfn_x(ring_mfn));
+        drop_p2m_gfn_domain(dom_mem_event, 
+                        shared_gfn, mfn_x(shared_mfn));
         return -EINVAL;
+    }
 
     /* Map ring and shared pages */
     med->ring_page = map_domain_page(mfn_x(ring_mfn));
     med->shared_page = map_domain_page(mfn_x(shared_mfn));
+    drop_p2m_gfn_domain(dom_mem_event, ring_gfn, 
+                    mfn_x(ring_mfn));
+    drop_p2m_gfn_domain(dom_mem_event, shared_gfn, 
+                    mfn_x(shared_mfn));
 
     /* Allocate event channel */
     rc = alloc_unbound_xen_event_channel(d->vcpu[0],

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/mm/mem_sharing.c
@@ -227,7 +227,7 @@ static void mem_sharing_audit(void)
                             g->domain, g->gfn, mfn_x(e->mfn));
                     continue;
                 }
-                mfn = gfn_to_mfn(d, g->gfn, &t); 
+                mfn = gfn_to_mfn_unlocked(d, g->gfn, &t); 
                 if(mfn_x(mfn) != mfn_x(e->mfn))
                     MEM_SHARING_DEBUG("Incorrect P2M for d=%d, PFN=%lx."
                                       "Expecting MFN=%ld, got %ld\n",

@@ -335,7 +335,7 @@ int mem_sharing_debug_gfn(struct domain 
     p2m_type_t p2mt;
     mfn_t mfn;
 
-    mfn = gfn_to_mfn(d, gfn, &p2mt);
+    mfn = gfn_to_mfn_unlocked(d, gfn, &p2mt);
 
     printk("Debug for domain=%d, gfn=%lx, ", 
             d->domain_id, 

@@ -524,6 +524,7 @@ int mem_sharing_nominate_page(struct dom
     ret = 0;
 
 out:
+    drop_p2m_gfn_domain(d, gfn, mfn_x(mfn));
     shr_unlock();
     return ret;
 }

@@ -593,14 +594,18 @@ int mem_sharing_unshare_page(struct doma
     shr_handle_t handle;
     struct list_head *le;
 
+    /* Remove the gfn_info from the list */
+   
+    /* This is one of the reasons why we can't enforce ordering
+     * between shr_lock and p2m fine-grained locks in mm-lock. 
+     * Callers may walk in here already holding the lock for this gfn */
     shr_lock();
     mem_sharing_audit();
-    
-    /* Remove the gfn_info from the list */
     mfn = gfn_to_mfn(d, gfn, &p2mt);
     
     /* Has someone already unshared it? */
     if (!p2m_is_shared(p2mt)) {
+        drop_p2m_gfn_domain(d, gfn, mfn_x(mfn));
         shr_unlock();
         return 0;
     }

@@ -634,6 +639,7 @@ gfn_found:
             /* Even though we don't allocate a private page, we have to account
              * for the MFN that originally backed this PFN. */
             atomic_dec(&nr_saved_mfns);
+        drop_p2m_gfn_domain(d, gfn, mfn_x(mfn));
         shr_unlock();
         put_page_and_type(page);
         if(last_gfn && 

@@ -653,6 +659,7 @@ gfn_found:
         /* We've failed to obtain memory for private page. Need to re-add the
          * gfn_info to relevant list */
         list_add(&gfn_info->list, &hash_entry->gfns);
+        drop_p2m_gfn_domain(d, gfn, mfn_x(mfn));
         shr_unlock();
         return -ENOMEM;
     }

@@ -665,6 +672,13 @@ gfn_found:
 
     BUG_ON(set_shared_p2m_entry(d, gfn, page_to_mfn(page)) == 0);
     put_page_and_type(old_page);
+    /* After switching the p2m entry we still hold it locked, and
+     * we have a ref count to the old page (mfn). Drop the ref
+     * on the old page, and set mfn to invalid, so the refcount is
+     * no further decremented. We are the only cpu who knows about
+     * the new page, so we don't need additional refs on it. */
+    put_page(mfn_to_page(mfn));
+    mfn = _mfn(INVALID_MFN);
 
 private_page_found:    
     /* We've got a private page, we can commit the gfn destruction */

@@ -683,6 +697,7 @@ private_page_found:
     /* Update m2p entry */
     set_gpfn_from_mfn(mfn_x(page_to_mfn(page)), gfn);
 
+    drop_p2m_gfn_domain(d, gfn, mfn_x(mfn));
     shr_unlock();
     return 0;
 }

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/mm/shadow/common.c
@@ -3741,6 +3741,8 @@ int shadow_track_dirty_vram(struct domai
                 }
             }
 
+            drop_p2m_gfn_domain(d, begin_pfn + i, mfn_x(mfn));
+
             if ( dirty )
             {
                 dirty_vram->dirty_bitmap[i / 8] |= 1 << (i % 8);

@@ -3761,7 +3763,7 @@ int shadow_track_dirty_vram(struct domai
                 /* was clean for more than two seconds, try to disable guest
                  * write access */
                 for ( i = begin_pfn; i < end_pfn; i++ ) {
-                    mfn_t mfn = gfn_to_mfn_query(d, i, &t);
+                    mfn_t mfn = gfn_to_mfn_query_unlocked(d, i, &t);
                     if (mfn_x(mfn) != INVALID_MFN)
                         flush_tlb |= sh_remove_write_access(d->vcpu[0], mfn, 1, 0);
                 }

262 lines not on this page · raw message

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/physdev.c
@@ -288,12 +288,18 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_H
         if ( !mfn_valid(mfn) ||
              !get_page_and_type(mfn_to_page(mfn), v->domain,
                                 PGT_writable_page) )
+        {
+            drop_p2m_gfn_domain(current->domain, 
+                            info.gmfn, mfn);
             break;
+        }
 
         if ( cmpxchg(&v->domain->arch.pv_domain.pirq_eoi_map_mfn,
                      0, mfn) != 0 )
         {
             put_page_and_type(mfn_to_page(mfn));
+            drop_p2m_gfn_domain(current->domain, 
+                            info.gmfn, mfn);
             ret = -EBUSY;
             break;
         }

@@ -303,10 +309,13 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_H
         {
             v->domain->arch.pv_domain.pirq_eoi_map_mfn = 0;
             put_page_and_type(mfn_to_page(mfn));
+            drop_p2m_gfn_domain(current->domain, 
+                            info.gmfn, mfn);
             ret = -ENOSPC;
             break;
         }
 
+        drop_p2m_gfn_domain(current->domain, info.gmfn, mfn);
         ret = 0;
         break;
     }

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/arch/x86/traps.c
@@ -678,6 +678,7 @@ int wrmsr_hypervisor_regs(uint32_t idx, 
         if ( !mfn_valid(mfn) ||
              !get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
         {
+            drop_p2m_gfn_domain(d, gmfn, mfn);
             gdprintk(XENLOG_WARNING,
                      "Bad GMFN %lx (MFN %lx) to MSR %08x\n",
                      gmfn, mfn, base + idx);

@@ -689,6 +690,7 @@ int wrmsr_hypervisor_regs(uint32_t idx, 
         unmap_domain_page(hypercall_page);
 
         put_page_and_type(mfn_to_page(mfn));
+        drop_p2m_gfn_domain(d, gmfn, mfn);
         break;
     }
 

@@ -2347,18 +2349,25 @@ static int emulate_privileged_op(struct 
             arch_set_cr2(v, *reg);
             break;
 
-        case 3: /* Write CR3 */
+        case 3: {/* Write CR3 */
+            unsigned long mfn, gfn;
             domain_lock(v->domain);
             if ( !is_pv_32on64_vcpu(v) )
-                rc = new_guest_cr3(gmfn_to_mfn(v->domain, xen_cr3_to_pfn(*reg)));
+            {
+                gfn = xen_cr3_to_pfn(*reg);
 #ifdef CONFIG_COMPAT
-            else
-                rc = new_guest_cr3(gmfn_to_mfn(v->domain, compat_cr3_to_pfn(*reg)));
+            } else {
+                gfn = compat_cr3_to_pfn(*reg);
 #endif
+            }
+            mfn = gmfn_to_mfn(v->domain, gfn);
+            rc = new_guest_cr3(mfn);
+            drop_p2m_gfn_domain(v->domain, gfn, mfn);
             domain_unlock(v->domain);
             if ( rc == 0 ) /* not okay */
                 goto fail;
             break;
+        }
 
         case 4: /* Write CR4 */
             v->arch.pv_vcpu.ctrlreg[4] = pv_guest_cr4_fixup(v, *reg);

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/common/grant_table.c
@@ -164,9 +164,11 @@ static int __get_paged_frame(unsigned lo
         if ( p2m_is_paging(p2mt) )
         {
             p2m_mem_paging_populate(rd, gfn);
+            drop_p2m_gfn_domain(rd, gfn, mfn_x(mfn));
             rc = GNTST_eagain;
         }
     } else {
+       drop_p2m_gfn_domain(rd, gfn, mfn_x(mfn));
        *frame = INVALID_MFN;
        rc = GNTST_bad_page;
     }

@@ -474,7 +476,7 @@ __gnttab_map_grant_ref(
     u32            old_pin;
     u32            act_pin;
     unsigned int   cache_flags;
-    struct active_grant_entry *act;
+    struct active_grant_entry *act = NULL; /* gcc ... */
     struct grant_mapping *mt;
     grant_entry_v1_t *sha1;
     grant_entry_v2_t *sha2;

@@ -698,6 +700,7 @@ __gnttab_map_grant_ref(
     op->handle       = handle;
     op->status       = GNTST_okay;
 
+    drop_p2m_gfn_domain(rd, act->gfn, act->frame);
     rcu_unlock_domain(rd);
     return;
 

@@ -735,6 +738,7 @@ __gnttab_map_grant_ref(
         gnttab_clear_flag(_GTF_reading, status);
 
  unlock_out:
+    drop_p2m_gfn_domain(rd, act->gfn, act->frame);
     spin_unlock(&rd->grant_table->lock);
     op->status = rc;
     put_maptrack_handle(ld->grant_table, handle);

@@ -1454,7 +1458,7 @@ gnttab_transfer(
     struct page_info *page;
     int i;
     struct gnttab_transfer gop;
-    unsigned long mfn;
+    unsigned long mfn, drop_mfn;
     unsigned int max_bitsize;
 
     for ( i = 0; i < count; i++ )

@@ -1475,6 +1479,7 @@ gnttab_transfer(
         /* Check the passed page frame for basic validity. */
         if ( unlikely(!mfn_valid(mfn)) )
         { 
+            drop_p2m_gfn_domain(d, gop.mfn, mfn);
             gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
                     (unsigned long)gop.mfn);
             gop.status = GNTST_bad_page;

@@ -1484,6 +1489,7 @@ gnttab_transfer(
         page = mfn_to_page(mfn);
         if ( unlikely(is_xen_heap_page(page)) )
         { 
+            drop_p2m_gfn_domain(d, gop.mfn, mfn);
             gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
                     (unsigned long)gop.mfn);
             gop.status = GNTST_bad_page;

@@ -1492,6 +1498,7 @@ gnttab_transfer(
 
         if ( steal_page(d, page, 0) < 0 )
         {
+            drop_p2m_gfn_domain(d, gop.mfn, mfn);
             gop.status = GNTST_bad_page;
             goto copyback;
         }

@@ -1504,6 +1511,7 @@ gnttab_transfer(
         /* Find the target domain. */
         if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) )
         {
+            drop_p2m_gfn_domain(d, gop.mfn, mfn);
             gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
                     gop.domid);
             page->count_info &= ~(PGC_count_mask|PGC_allocated);

@@ -1514,6 +1522,7 @@ gnttab_transfer(
 
         if ( xsm_grant_transfer(d, e) )
         {
+            drop_p2m_gfn_domain(d, gop.mfn, mfn);
             gop.status = GNTST_permission_denied;
         unlock_and_copyback:
             rcu_unlock_domain(e);

@@ -1542,9 +1551,15 @@ gnttab_transfer(
             unmap_domain_page(dp);
             unmap_domain_page(sp);
 
+            /* We took a ref on acquiring the p2m entry. Drop the ref */
+            put_page(page);
+            drop_mfn = INVALID_MFN; /* Further drops of the p2m entry won't drop anyone's refcount */
             page->count_info &= ~(PGC_count_mask|PGC_allocated);
             free_domheap_page(page);
             page = new_page;
+            /* BY the way, this doesn't update mfn, which is used later below ... */
+        } else {
+            drop_mfn = mfn;
         }
 
         spin_lock(&e->page_alloc_lock);

@@ -1566,6 +1581,7 @@ gnttab_transfer(
                         e->tot_pages, e->max_pages, gop.ref, e->is_dying);
             spin_unlock(&e->page_alloc_lock);
             rcu_unlock_domain(e);
+            drop_p2m_gfn_domain(d, gop.mfn, drop_mfn);
             page->count_info &= ~(PGC_count_mask|PGC_allocated);
             free_domheap_page(page);
             gop.status = GNTST_general_error;

@@ -1579,6 +1595,7 @@ gnttab_transfer(
         page_set_owner(page, e);
 
         spin_unlock(&e->page_alloc_lock);
+        drop_p2m_gfn_domain(d, gop.mfn, drop_mfn);
 
         TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
 

@@ -1852,6 +1869,8 @@ __acquire_grant_for_copy(
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
                 goto unlock_out;
+            /* We drop this immediately per the comments at the top */
+            drop_p2m_gfn_domain(rd, gfn, grant_frame);
             act->gfn = gfn;
             is_sub_page = 0;
             trans_page_off = 0;

@@ -1864,6 +1883,7 @@ __acquire_grant_for_copy(
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
                 goto unlock_out;
+            drop_p2m_gfn_domain(rd, gfn, grant_frame);
             act->gfn = gfn;
             is_sub_page = 0;
             trans_page_off = 0;

@@ -1876,6 +1896,7 @@ __acquire_grant_for_copy(
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
                 goto unlock_out;
+            drop_p2m_gfn_domain(rd, gfn, grant_frame);
             act->gfn = gfn;
             is_sub_page = 1;
             trans_page_off = sha2->sub_page.page_off;

@@ -1973,6 +1994,7 @@ __gnttab_copy(
     {
 #ifdef CONFIG_X86
         rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
+        drop_p2m_gfn_domain(sd, op->source.u.gmfn, s_frame);
         if ( rc != GNTST_okay )
             goto error_out;
 #else

@@ -2012,6 +2034,7 @@ __gnttab_copy(
     {
 #ifdef CONFIG_X86
         rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
+        drop_p2m_gfn_domain(dd, op->dest.u.gmfn, d_frame);
         if ( rc != GNTST_okay )
             goto error_out;
 #else

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/common/memory.c
@@ -167,6 +167,7 @@ int guest_remove_page(struct domain *d, 
     {
         guest_physmap_remove_page(d, gmfn, mfn, 0);
         p2m_mem_paging_drop_page(d, gmfn);
+        drop_p2m_gfn_domain(d, gmfn, mfn);
         return 1;
     }
 #else

@@ -174,6 +175,7 @@ int guest_remove_page(struct domain *d, 
 #endif
     if ( unlikely(!mfn_valid(mfn)) )
     {
+        drop_p2m_gfn_domain(d, gmfn, mfn);
         gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
                 d->domain_id, gmfn);
         return 0;

@@ -187,12 +189,14 @@ int guest_remove_page(struct domain *d, 
     {
         put_page_and_type(page);
         guest_physmap_remove_page(d, gmfn, mfn, 0);
+        drop_p2m_gfn_domain(d, gmfn, mfn);
         return 1;
     }
 
 #endif /* CONFIG_X86 */
     if ( unlikely(!get_page(page, d)) )
     {
+        drop_p2m_gfn_domain(d, gmfn, mfn);
         gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
         return 0;
     }

@@ -204,6 +208,7 @@ int guest_remove_page(struct domain *d, 
         put_page(page);
 
     guest_physmap_remove_page(d, gmfn, mfn, 0);
+    drop_p2m_gfn_domain(d, gmfn, mfn);
 
     put_page(page);
 

@@ -366,6 +371,7 @@ static long memory_exchange(XEN_GUEST_HA
                 mfn = mfn_x(gfn_to_mfn_unshare(d, gmfn + k, &p2mt));
                 if ( p2m_is_shared(p2mt) )
                 {
+                    drop_p2m_gfn_domain(d, gmfn + k, mfn);
                     rc = -ENOMEM;
                     goto fail; 
                 }

@@ -374,6 +380,7 @@ static long memory_exchange(XEN_GUEST_HA
 #endif
                 if ( unlikely(!mfn_valid(mfn)) )
                 {
+                    drop_p2m_gfn_domain(d, gmfn + k, mfn);
                     rc = -EINVAL;
                     goto fail;
                 }

@@ -382,11 +389,13 @@ static long memory_exchange(XEN_GUEST_HA
 
                 if ( unlikely(steal_page(d, page, MEMF_no_refcount)) )
                 {
+                    drop_p2m_gfn_domain(d, gmfn + k, mfn);
                     rc = -EINVAL;
                     goto fail;
                 }
 
                 page_list_add(page, &in_chunk_list);
+                drop_p2m_gfn_domain(d, gmfn + k, mfn);
             }
         }
 

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/common/tmem_xen.c
@@ -111,20 +111,28 @@ static inline void *cli_get_page(tmem_cl
 
     cli_mfn = mfn_x(gfn_to_mfn(current->domain, cmfn, &t));
     if ( t != p2m_ram_rw || !mfn_valid(cli_mfn) )
+    {
+            drop_p2m_gfn_domain(current->domain, 
+                            (unsigned long) cmfn, cli_mfn);
             return NULL;
+    }
     page = mfn_to_page(cli_mfn);
     if ( cli_write )
         ret = get_page_and_type(page, current->domain, PGT_writable_page);
     else
         ret = get_page(page, current->domain);
     if ( !ret )
+    {
+        drop_p2m_gfn_domain(current->domain, 
+                        (unsigned long) cmfn, cli_mfn);
         return NULL;
+    }
     *pcli_mfn = cli_mfn;
     *pcli_pfp = (pfp_t *)page;
     return map_domain_page(cli_mfn);
 }
 
-static inline void cli_put_page(void *cli_va, pfp_t *cli_pfp,
+static inline void cli_put_page(tmem_cli_mfn_t cmfn, void *cli_va, pfp_t *cli_pfp,
                                 unsigned long cli_mfn, bool_t mark_dirty)
 {
     if ( mark_dirty )

@@ -135,6 +143,7 @@ static inline void cli_put_page(void *cl
     else
         put_page((struct page_info *)cli_pfp);
     unmap_domain_page(cli_va);
+    drop_p2m_gfn_domain(current->domain, (unsigned long) cmfn, cli_mfn);
 }
 #endif
 

@@ -169,7 +178,7 @@ EXPORT int tmh_copy_from_client(pfp_t *p
               (pfn_offset+len <= PAGE_SIZE) )
         memcpy((char *)tmem_va+tmem_offset,(char *)cli_va+pfn_offset,len);
     if ( !tmemc )
-        cli_put_page(cli_va, cli_pfp, cli_mfn, 0);
+        cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 0);
     unmap_domain_page(tmem_va);
     return 1;
 }

@@ -197,7 +206,7 @@ EXPORT int tmh_compress_from_client(tmem
     ASSERT(ret == LZO_E_OK);
     *out_va = dmem;
     if ( !tmemc )
-        cli_put_page(cli_va, cli_pfp, cli_mfn, 0);
+        cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 0);
     unmap_domain_page(cli_va);
     return 1;
 }

@@ -225,7 +234,7 @@ EXPORT int tmh_copy_to_client(tmem_cli_m
         memcpy((char *)cli_va+pfn_offset,(char *)tmem_va+tmem_offset,len);
     unmap_domain_page(tmem_va);
     if ( !tmemc )
-        cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
+        cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 1);
     mb();
     return 1;
 }

@@ -249,7 +258,7 @@ EXPORT int tmh_decompress_to_client(tmem
     ASSERT(ret == LZO_E_OK);
     ASSERT(out_len == PAGE_SIZE);
     if ( !tmemc )
-        cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
+        cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 1);
     mb();
     return 1;
 }

@@ -271,7 +280,7 @@ EXPORT int tmh_copy_tze_to_client(tmem_c
         memcpy((char *)cli_va,(char *)tmem_va,len);
     if ( len < PAGE_SIZE )
         memset((char *)cli_va+len,0,PAGE_SIZE-len);
-    cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
+    cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 1);
     mb();
     return 1;
 }

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/include/asm-x86/hvm/hvm.h
@@ -394,7 +394,10 @@ int hvm_virtual_to_linear_addr(
 
 void *hvm_map_guest_frame_rw(unsigned long gfn);
 void *hvm_map_guest_frame_ro(unsigned long gfn);
-void hvm_unmap_guest_frame(void *p);
+/* We pass back either the guest virtual or physical frame mapped,
+ * in order to drop any locks/refcounts we may have had on p2m 
+ * entries or underlying mfn's while using the map */
+void hvm_unmap_guest_frame(void *p, unsigned long addr, int is_va);
 
 static inline void hvm_set_info_guest(struct vcpu *v)
 {

diff -r 471d4f2754d6 -r d13f91c2fe18 xen/include/asm-x86/hvm/vmx/vvmx.h
@@ -25,6 +25,7 @@
 
 struct nestedvmx {
     paddr_t    vmxon_region_pa;
+    unsigned long iobitmap_gfn[2];
     void       *iobitmap[2];		/* map (va) of L1 guest I/O bitmap */
     /* deferred nested interrupt */
     struct {