Skip to contentSkip to conversation

Nested Virtualization: hap-on-hap

PATCH In review Christoph Egger 2 participants 8 messages Nov 12, 2010 – Dec 20, 2010

lore
Share & tools
lore
#0 lore raw

PATCH 13/13 Nested Virtualization: hap-on-hap

Signature · 7 lines
-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

# HG changeset patch
# User cegger
# Date 1289584020 -3600
Implement Nested-on-Nested.
This allows the guest to run nested guest with hap enabled.

xen/arch/x86/hvm/hvm.c +56 -7
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/hvm/hvm.c
@@ -1014,12 +1014,40 @@ void hvm_inject_exception(unsigned int t
     hvm_funcs.inject_exception(trapnr, errcode, cr2);
 }
 
-bool_t hvm_hap_nested_page_fault(unsigned long gfn)
+int hvm_hap_nested_page_fault(paddr_t gpa, struct cpu_user_regs *regs)
 {
     p2m_type_t p2mt;
     mfn_t mfn;
     struct vcpu *v = current;
     struct p2m_domain *p2m = p2m_get_hostp2m(v->domain);
+    unsigned long gfn = gpa >> PAGE_SHIFT;
+    int rv;
+
+    /* On Nested Virtualization, walk the guest page table.
+     * If this succeeds, all is fine.
+     * If this fails, inject a nested page fault into the guest.
+     */
+    if ( nestedhvm_enabled(v->domain)
+        && nestedhvm_vcpu_in_guestmode(v)
+        && nestedhvm_paging_mode_hap(v) )
+    {
+        /* The vcpu is in guest mode and the l1 guest
+         * uses hap. That means 'gpa' is in l2 guest
+         * physical address space.
+         * Fix the nested p2m or inject nested page fault
+         * into l1 guest if not fixable. The algorithm is
+         * the same as for shadow paging.
+         */
+        rv = nestedhvm_hap_nested_page_fault(v, gpa);
+        switch (rv) {
+        case NESTEDHVM_PAGEFAULT_DONE:
+            return 1;
+        case NESTEDHVM_PAGEFAULT_ERROR:
+            return 0;
+        case NESTEDHVM_PAGEFAULT_INJECT:
+            return -1;
+        }
+    }
 
     mfn = gfn_to_mfn_guest(p2m, gfn, &p2mt);
 
@@ -1097,6 +1125,15 @@ int hvm_set_efer(uint64_t value)
         return X86EMUL_EXCEPTION;
     }
 
+    if ( nestedhvm_enabled(v->domain) && cpu_has_svm &&
+       ((value & EFER_SVME) == 0 ) &&
+       ((value ^ v->arch.hvm_vcpu.guest_efer) & EFER_SVME) )
+    {
+        /* Cleared EFER.SVME: Flush all nestedp2m tables */
+        p2m_flush_nestedp2m(v->domain);
+        nestedhvm_vcpu_reset(v);
+    }
+
     value |= v->arch.hvm_vcpu.guest_efer & EFER_LMA;
     v->arch.hvm_vcpu.guest_efer = value;
     hvm_update_guest_efer(v);
@@ -1247,8 +1284,12 @@ int hvm_set_cr0(unsigned long value)
     v->arch.hvm_vcpu.guest_cr[0] = value;
     hvm_update_guest_cr(v, 0);
 
-    if ( (value ^ old_value) & X86_CR0_PG )
-        paging_update_paging_modes(v);
+    if ( (value ^ old_value) & X86_CR0_PG ) {
+        if ( !nestedhvm_vmswitch_in_progress(v) && nestedhvm_vcpu_in_guestmode(v) )
+            paging_update_nestedmode(v);
+        else
+            paging_update_paging_modes(v);
+    }
 
     return X86EMUL_OKAY;
 
@@ -1315,8 +1356,12 @@ int hvm_set_cr4(unsigned long value)
     hvm_update_guest_cr(v, 4);
 
     /* Modifying CR4.{PSE,PAE,PGE} invalidates all TLB entries, inc. Global. */
-    if ( (old_cr ^ value) & (X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE) )
-        paging_update_paging_modes(v);
+    if ( (old_cr ^ value) & (X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE) ) {
+        if ( !nestedhvm_vmswitch_in_progress(v) && nestedhvm_vcpu_in_guestmode(v) )
+            paging_update_nestedmode(v);
+        else
+            paging_update_paging_modes(v);
+    }
 
     return X86EMUL_OKAY;
 
@@ -1829,11 +1874,15 @@ static enum hvm_copy_result __hvm_copy(
     void *buf, paddr_t addr, int size, unsigned int flags, uint32_t pfec)
 {
     struct vcpu *curr = current;
-    struct p2m_domain *p2m = p2m_get_hostp2m(curr->domain);
+    struct p2m_domain *p2m;
     unsigned long gfn, mfn;
     p2m_type_t p2mt;
     char *p;
     int count, todo = size;
+    unsigned long cr3 = curr->arch.hvm_vcpu.guest_cr[3];
+    const struct paging_mode *mode = paging_get_mode(curr);
+
+    p2m = p2m_get_p2m(curr);
 
     while ( todo > 0 )
     {
@@ -1841,7 +1890,7 @@ static enum hvm_copy_result __hvm_copy(
 
         if ( flags & HVMCOPY_virt )
         {
-            gfn = paging_gva_to_gfn(curr, addr, &pfec);
+            gfn = paging_p2m_ga_to_gfn(curr, p2m, mode, cr3, addr, &pfec);
             if ( gfn == INVALID_GFN )
             {
                 if ( pfec == PFEC_page_paged )
xen/arch/x86/hvm/nestedhvm.c +17 -0
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/hvm/nestedhvm.c
@@ -20,6 +20,7 @@
 #include <asm/msr.h>
 #include <asm/hvm/support.h>	/* for HVM_DELIVER_NO_ERROR_CODE */
 #include <asm/hvm/hvm.h>
+#include <asm/p2m.h>    /* for struct p2m_domain */
 #include <asm/hvm/nestedhvm.h>
 #include <asm/event.h>  /* for local_event_delivery_(en|dis)able */
 #include <asm/paging.h> /* for paging_mode_hap() */
@@ -100,11 +101,27 @@ nestedhvm_vcpu_destroy(struct vcpu *v)
 void
 nestedhvm_vcpu_enter_guestmode(struct vcpu *v)
 {
+	struct p2m_domain *p2m;
 	vcpu_nestedhvm(v).nv_guestmode = 1;
+
+	p2m = vcpu_nestedhvm(v).nv_p2m;
+	if (p2m == NULL)
+		/* p2m has either been invalidated or not yet assigned. */
+		return;
+
+	cpu_set(v->processor, p2m->p2m_dirty_cpumask);
 }
 
 void
 nestedhvm_vcpu_exit_guestmode(struct vcpu *v)
 {
+	struct p2m_domain *p2m;
 	vcpu_nestedhvm(v).nv_guestmode = 0;
+
+	p2m = vcpu_nestedhvm(v).nv_p2m;
+	if (p2m == NULL)
+		/* p2m has either been invalidated or not yet assigned. */
+		return;
+
+	cpu_clear(v->processor, p2m->p2m_dirty_cpumask);
 }
xen/arch/x86/hvm/svm/nestedsvm.c +6 -0
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/hvm/svm/nestedsvm.c
@@ -25,6 +25,7 @@
 #include <asm/hvm/svm/svmdebug.h>
 #include <asm/paging.h> /* paging_mode_hap */
 #include <asm/event.h> /* for local_event_delivery_(en|dis)able */
+#include <asm/p2m.h> /* p2m_get_pagetable, p2m_get_nestedp2m */
 
 static int
 nestedsvm_vcpu_clgi(struct vcpu *v)
@@ -427,6 +428,10 @@ static int nsvm_vmcb_prepare4vmrun(struc
     /* Nested paging mode */
     if (nestedhvm_paging_mode_hap(v)) {
         /* host nested paging + guest nested paging. */
+        host_vmcb->np_enable = 1;
+
+        host_vmcb->h_cr3 =
+            pagetable_get_paddr(p2m_get_pagetable(p2m_get_nestedp2m(v, ns_vmcb->h_cr3)));
 
         /* hvm_set_cr3() below sets v->arch.hvm_vcpu.guest_cr[3] for us. */
         rc = hvm_set_cr3(ns_vmcb->cr3);
@@ -1168,6 +1173,7 @@ nestedsvm_vcpu_vmexit(struct vcpu *v, st
     struct nestedvcpu *nv = &vcpu_nestedhvm(v);
 
     nv->nv_hostflags.fields.vmswitch_in_progress = 1;
+    paging_update_nestedmode(v);
 
     ASSERT(nv->nv_vmcx != NULL);
 
xen/arch/x86/hvm/svm/svm.c +27 -8
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/hvm/svm/svm.c
@@ -955,14 +955,17 @@ struct hvm_function_table * __init start
     return &svm_function_table;
 }
 
-static void svm_do_nested_pgfault(paddr_t gpa)
+static void svm_do_nested_pgfault(struct vcpu *v,
+    struct cpu_user_regs *regs, paddr_t gpa)
 {
+    int ret;
     unsigned long gfn = gpa >> PAGE_SHIFT;
     mfn_t mfn;
     p2m_type_t p2mt;
-    struct p2m_domain *p2m;
+    struct p2m_domain *p2m = NULL;
+    struct nestedsvm *svm = &vcpu_nestedsvm(v);
 
-    p2m = p2m_get_hostp2m(current->domain);
+    ret = hvm_hap_nested_page_fault(gpa, regs);
 
     if ( tb_init_done )
     {
@@ -973,6 +976,7 @@ static void svm_do_nested_pgfault(paddr_
             uint32_t p2mt;
         } _d;
 
+        p2m = p2m_get_p2m(v);
         _d.gpa = gpa;
         _d.qualification = 0;
         _d.mfn = mfn_x(gfn_to_mfn_query(p2m, gfn, &_d.p2mt));
@@ -980,14 +984,29 @@ static void svm_do_nested_pgfault(paddr_
         __trace_var(TRC_HVM_NPF, 0, sizeof(_d), &_d);
     }
 
-    if ( hvm_hap_nested_page_fault(gfn) )
+    switch (ret) {
+    case 0:
+        break;
+    case 1:
         return;
+    case -1:
+        ASSERT(nestedhvm_enabled(v->domain) && nestedhvm_vcpu_in_guestmode(v));
+        /* inject #VMEXIT(NPF) into guest. */
+        svm->ns_vmexit.exitcode = VMEXIT_NPF;
+        svm->ns_vmexit.exitinfo1 = regs->error_code;
+        svm->ns_vmexit.exitinfo2 = gpa;
+        vcpu_nestedhvm(v).nv_hostflags.fields.vmexit_pending = 1;
+        return;
+    }
 
+    if ( p2m == NULL )
+        p2m = p2m_get_p2m(v);
     /* Everything else is an error. */
     mfn = gfn_to_mfn_guest(p2m, gfn, &p2mt);
-    gdprintk(XENLOG_ERR, "SVM violation gpa %#"PRIpaddr", mfn %#lx, type %i\n",
-             gpa, mfn_x(mfn), p2mt);
-    domain_crash(current->domain);
+    gdprintk(XENLOG_ERR,
+         "SVM violation gpa %#"PRIpaddr", mfn %#lx, type %i\n",
+         gpa, mfn_x(mfn), p2mt);
+    domain_crash(v->domain);
 }
 
 static void svm_fpu_dirty_intercept(void)
@@ -1817,7 +1836,7 @@ asmlinkage void svm_vmexit_handler(struc
     case VMEXIT_NPF:
         perfc_incra(svmexits, VMEXIT_NPF_PERFC);
         regs->error_code = vmcb->exitinfo1;
-        svm_do_nested_pgfault(vmcb->exitinfo2);
+        svm_do_nested_pgfault(v, regs, vmcb->exitinfo2);
         break;
 
     case VMEXIT_IRET:
xen/arch/x86/hvm/vmx/vmx.c +1 -1
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/hvm/vmx/vmx.c
@@ -2083,7 +2083,7 @@ static void ept_handle_violation(unsigne
     }
 
     if ( (qualification & EPT_GLA_VALID) &&
-         hvm_hap_nested_page_fault(gfn) )
+         hvm_hap_nested_page_fault(gpa, guest_cpu_user_regs()) )
         return;
 
     /* Everything else is an error. */
xen/arch/x86/mm/hap/Makefile +1 -0
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/mm/hap/Makefile
@@ -3,6 +3,7 @@ obj-y += guest_walk_2level.o
 obj-y += guest_walk_3level.o
 obj-y += guest_walk_4level.o
 obj-y += p2m-ept.o
+obj-y += nested_hap.o
 
 guest_levels  = $(subst level,,$(filter %level,$(subst ., ,$(subst _, ,$(1)))))
 guest_walk_defns = -DGUEST_PAGING_LEVELS=$(call guest_levels,$(1))
xen/arch/x86/mm/hap/guest_walk.c +23 -3
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/mm/hap/guest_walk.c
@@ -29,6 +29,9 @@
 #define _hap_gva_to_gfn(levels) hap_gva_to_gfn_##levels##_levels
 #define hap_gva_to_gfn(levels) _hap_gva_to_gfn(levels)
 
+#define _hap_p2m_ga_to_gfn(levels) hap_p2m_ga_to_gfn_##levels##_levels
+#define hap_p2m_ga_to_gfn(levels) _hap_p2m_ga_to_gfn(levels)
+
 #if GUEST_PAGING_LEVELS <= CONFIG_PAGING_LEVELS
 
 #include <asm/guest_pt.h>
@@ -38,15 +41,23 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
     struct vcpu *v, unsigned long gva, uint32_t *pfec)
 {
     unsigned long cr3;
+    struct p2m_domain *p2m = p2m_get_hostp2m(v->domain);
+
+    cr3 = v->arch.hvm_vcpu.guest_cr[3];
+    return hap_p2m_ga_to_gfn(GUEST_PAGING_LEVELS)(v, p2m, cr3, gva, pfec);
+}
+
+unsigned long hap_p2m_ga_to_gfn(GUEST_PAGING_LEVELS)(
+    struct vcpu *v, struct p2m_domain *p2m, unsigned long cr3,
+    paddr_t ga, uint32_t *pfec)
+{
     uint32_t missing;
     mfn_t top_mfn;
     void *top_map;
     p2m_type_t p2mt;
     walk_t gw;
-    struct p2m_domain *p2m = p2m_get_hostp2m(v->domain);
 
     /* Get the top-level table's MFN */
-    cr3 = v->arch.hvm_vcpu.guest_cr[3];
     top_mfn = gfn_to_mfn_unshare(p2m, cr3 >> PAGE_SHIFT, &p2mt, 0);
     if ( p2m_is_paging(p2mt) )
     {
@@ -72,7 +83,7 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
 #if GUEST_PAGING_LEVELS == 3
     top_map += (cr3 & ~(PAGE_MASK | 31));
 #endif
-    missing = guest_walk_tables(v, p2m, gva, &gw, pfec[0], top_mfn, top_map);
+    missing = guest_walk_tables(v, p2m, ga, &gw, pfec[0], top_mfn, top_map);
     unmap_domain_page(top_map);
 
     /* Interpret the answer */
@@ -122,6 +133,15 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
     return INVALID_GFN;
 }
 
+unsigned long hap_p2m_ga_to_gfn(GUEST_PAGING_LEVELS)(
+    struct vcpu *v, struct p2m_domain *p2m, unsigned long cr3,
+    paddr_t ga, uint32_t *pfec)
+{
+    gdprintk(XENLOG_ERR,
+             "Guest paging level is greater than host paging level!\n");
+    domain_crash(v->domain);
+    return INVALID_GFN;
+}
 #endif
 
 
xen/arch/x86/mm/hap/hap.c +86 -11

200 lines not on this page · raw message

xen/arch/x86/mm/hap/nested_hap.c new file +423 -0

423 lines not on this page · raw message

xen/arch/x86/mm/hap/private.h +10 -0
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/arch/x86/mm/hap/private.h
@@ -30,4 +30,14 @@ unsigned long hap_gva_to_gfn_3_levels(st
 unsigned long hap_gva_to_gfn_4_levels(struct vcpu *v, unsigned long gva,
                                      uint32_t *pfec);
 
+unsigned long hap_p2m_ga_to_gfn_2_levels(struct vcpu *v,
+    struct p2m_domain *p2m, unsigned long cr3,
+    paddr_t ga, uint32_t *pfec);
+unsigned long hap_p2m_ga_to_gfn_3_levels(struct vcpu *v,
+    struct p2m_domain *p2m, unsigned long cr3,
+    paddr_t ga, uint32_t *pfec);
+unsigned long hap_p2m_ga_to_gfn_4_levels(struct vcpu *v,
+    struct p2m_domain *p2m, unsigned long cr3,
+    paddr_t ga, uint32_t *pfec);
+
 #endif /* __HAP_PRIVATE_H__ */
xen/arch/x86/mm/p2m.c +221 -4

301 lines not on this page · raw message

xen/arch/x86/mm/paging.c +37 -5

64 lines not on this page · raw message

xen/include/asm-x86/domain.h +7 -0
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/include/asm-x86/domain.h
@@ -206,6 +206,8 @@ struct paging_domain {
 struct paging_vcpu {
     /* Pointers to mode-specific entry points. */
     const struct paging_mode *mode;
+    /* Nested Virtualization: paging mode of nested guest */
+    const struct paging_mode *nestedmode;
     /* HVM guest: last emulate was to a pagetable */
     unsigned int last_write_was_pt:1;
     /* HVM guest: last write emulation succeeds */
@@ -221,6 +223,7 @@ struct paging_vcpu {
 #define MAX_CPUID_INPUT 40
 typedef xen_domctl_cpuid_t cpuid_input_t;
 
+#define MAX_NESTEDP2M 10
 struct p2m_domain;
 struct time_scale {
     int shift;
@@ -254,6 +257,10 @@ struct arch_domain
     struct paging_domain paging;
     struct p2m_domain *p2m;
 
+    /* nestedhvm: translate l2 guest physical to host physical */
+    struct p2m_domain *nested_p2m[MAX_NESTEDP2M];
+    spinlock_t nested_p2m_lock;
+
     /* NB. protected by d->event_lock and by irq_desc[irq].lock */
     int *irq_pirq;
     int *pirq_irq;
xen/include/asm-x86/hvm/hvm.h +1 -1
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/include/asm-x86/hvm/hvm.h
@@ -374,7 +374,7 @@ static inline void hvm_set_info_guest(st
 
 int hvm_debug_op(struct vcpu *v, int32_t op);
 
-bool_t hvm_hap_nested_page_fault(unsigned long gfn);
+int hvm_hap_nested_page_fault(paddr_t gpa, struct cpu_user_regs *regs);
 
 #define hvm_msr_tsc_aux(v) ({                                               \
     struct domain *__d = (v)->domain;                                       \
xen/include/asm-x86/hvm/nestedhvm.h +2 -2
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/include/asm-x86/hvm/nestedhvm.h
@@ -82,7 +82,7 @@ int nestedhvm_hap_nested_page_fault(stru
 #define nestedhvm_paging_mode_hap(v) (!!nhvm_vmcx_hap_enabled(v))
 #define nestedhvm_vmswitch_in_progress(v)   \
     (!!vcpu_nestedhvm((v)).nv_hostflags.fields.vmswitch_in_progress)
-#define nestedhvm_vmcx_flushtlb(d) \
-    flush_tlb_mask(&(d)->arch.hvm_domain.nh_dirty_cpumask)
+#define nestedhvm_vmcx_flushtlb(p2m) \
+    flush_tlb_mask(&((p2m)->p2m_dirty_cpumask))
 
 #endif /* _HVM_NESTEDHVM_H */
xen/include/asm-x86/p2m.h +37 -0
diff -r 2aafa38f2390 -r 5e8e3c570bb3 xen/include/asm-x86/p2m.h
@@ -177,7 +177,14 @@ struct p2m_domain {
     /* VRAM dirty support. */
     struct sh_dirty_vram *dirty_vram;
 
+    /* Same as domain_dirty_cpumask but limited to
+     * this p2m and those physical cpus whose vcpu's are in
+     * guestmode.
+     */
+    cpumask_t          p2m_dirty_cpumask;
+
     struct domain     *domain;   /* back pointer to domain */
+    uint64_t           cr3;      /* to identify this p2m for re-use */
 
     /* Pages used to construct the p2m */
     struct page_list_head pages;
@@ -234,8 +241,26 @@ struct p2m_domain {
 /* get host p2m table */
 #define p2m_get_hostp2m(d)      ((d)->arch.p2m)
 
+/* Get p2m table (re)usable for specified cr3.
+ * Automatically destroys and re-initializes a p2m if none found.
+ * If cr3 == 0 then v->arch.hvm_vcpu.guest_cr[3] is used.
+ */
+struct p2m_domain *p2m_get_nestedp2m(struct vcpu *v, uint64_t cr3);
+
+/* If vcpu is in host mode then behaviour matches p2m_get_hostp2m().
+ * If vcpu is in guest mode then behaviour matches p2m_get_nestedp2m().
+ */
+struct p2m_domain *p2m_get_p2m(struct vcpu *v);
+
+#define p2m_is_nestedp2m(p2m)   ((p2m) != p2m_get_hostp2m((p2m->domain)))
+
 #define p2m_get_pagetable(p2m)  ((p2m)->phys_table)
 
+/* Flushes specified p2m table */
+void p2m_flush(struct vcpu *v, struct p2m_domain *p2m);
+/* Flushes all nested p2m tables */
+void p2m_flush_nestedp2m(struct domain *d);
+
 /*
  * The P2M lock.  This protects all updates to the p2m table.
  * Updates are expected to be safe against concurrent reads,
@@ -391,6 +416,9 @@ static inline unsigned long mfn_to_gfn(s
 /* Init the datastructures for later use by the p2m code */
 int p2m_init(struct domain *d);
 
+/* PTE flags for various types of p2m entry */
+unsigned long p2m_type_to_flags(p2m_type_t t, mfn_t mfn);
+
 /* Allocate a new p2m table for a domain. 
  *
  * The alloc_page and free_page functions will be used to get memory to
@@ -401,6 +429,13 @@ int p2m_alloc_table(struct p2m_domain *p
                struct page_info * (*alloc_page)(struct p2m_domain *p2m),
                void (*free_page)(struct p2m_domain *p2m, struct page_info *pg));
 
+/* Find the next level's P2M entry, checking for out-of-range gfn's...
+ * Returns NULL on error.
+ */
+l1_pgentry_t *
+p2m_find_entry(void *table, unsigned long *gfn_remainder,
+               unsigned long gfn, uint32_t shift, uint32_t max);
+
 /* Return all the p2m resources to Xen. */
 void p2m_teardown(struct p2m_domain *p2m);
 void p2m_final_teardown(struct domain *d);
@@ -474,6 +509,8 @@ p2m_type_t p2m_change_type(struct p2m_do
 int set_mmio_p2m_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn);
 int clear_mmio_p2m_entry(struct p2m_domain *p2m, unsigned long gfn);
 
+int nestedp2m_set_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
+                        unsigned int page_order, p2m_type_t p2mt);
 
 #ifdef __x86_64__
 /* Modify p2m table for shared gfn */
xen/include/asm-x86/paging.h +42 -12

127 lines not on this page · raw message

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#1 lore raw

Hi,

This patch doesn't address most of my concerns with the last version.

My comments about callers of gva_to_gfn still stand -- basically, gva_to_gfn should return an N1 GFN, and most callers (including hvm_copy) should not need to know about N2 GFNs or multiple p2ms.

Also, you're still copying large parts of the existing HAP walker (e.g. the next_level function). Can you avoid the duplication by using a write_p2m_entry() callback, since that seems to be the part that's different?

Skipping a flush when p2m->cr3 == 0 is not safe. I suggested using -1 as the magic value last time.

My comments on why p2m_flush_locked() isn't enough to reclaim an in-use p2m for recycling still stand.

I have one more comment on the new code:

Quoted 13 lines · Christoph Egger

hap_write_p2m_entry(struct vcpu *v, unsigned long gfn, l1_pgentry_t *p,
mfn_t table_mfn, l1_pgentry_t new, unsigned int level)
{

xen/arch/x86/mm/hap/hap.c +5 -2
+    struct domain *d = v->domain;
     uint32_t old_flags;
 
-    hap_lock(v->domain);
+    old_flags = l1e_get_flags(*p);
 
-    old_flags = l1e_get_flags(*p);
+    /* We know always use the host p2m here, regardless if the vcpu
+     * is in host or guest mode. The vcpu can be in guest mode by
+     * a hypercall which passes a domain and chooses mostly the first

xen/arch/x86/mm/hap/hap.c static void · line 875

Quoted 10 lines
xen/arch/x86/mm/hap/hap.c +10 -0
+     * vcpu.
+     * XXX This is the reason why this function can not be used re-used
+     * for updating the nestedp2m. Otherwise, hypercalls would randomly
+     * operate on host p2m and nested p2m.
+     */
+    if ( nestedhvm_enabled(d) ) {
+        mfn_t omfn = _mfn(l1e_get_pfn(*p));
+        p2m_type_t op2mt = p2m_flags_to_type(old_flags);
+
+        if ( p2m_is_valid(op2mt) && mfn_valid(omfn) ) {

I think you need to do this flush even if !mfn_valid(omfn) - for example if you're removing a mapping of an MMIO area.

xen/arch/x86/mm/hap/hap.c static void · line 881

Quoted 6 lines
xen/arch/x86/mm/hap/hap.c +6 -0
+            mfn_t nmfn = _mfn(l1e_get_pfn(new));
+            p2m_type_t np2mt = p2m_flags_to_type(l1e_get_flags(new));
+
+            if ( p2m_is_valid(np2mt)
+                && mfn_valid(nmfn)
+                && !(l1e_get_flags(new) & _PAGE_PRESENT) )

I don't understand this test at all - you need to flush if you're removing an old present entry regardless of what replaces it. The only case where you can skip the flush is if old == new.

Cheers,

Tim.

Quoted 13 lines
xen/arch/x86/mm/hap/hap.c +13 -0
+            {
+                /* This GFN -> MFN is going to get removed. */
+                /* XXX There is a more efficient way to do that
+                 * but it works for now.
+                 * Note, p2m_flush_nestedp2m calls hap_lock() internally.
+                 */
+                p2m_flush_nestedp2m(d);
+            }
+        }
+    }
+
+    hap_lock(d);
+
Signature · 3 lines
Tim Deegan <[email protected]>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)
#2 lore raw
Quoted 7 lines · On Tuesday 16 November 2010 17:58:25 Tim Deegan wrote:

Hi,

This patch doesn't address most of my concerns with the last version.

My comments about callers of gva_to_gfn still stand -- basically, gva_to_gfn should return an N1 GFN, and most callers (including hvm_copy) should not need to know about N2 GFNs or multiple p2ms.

Done.

Quoted 5 lines

Also, you're still copying large parts of the existing HAP walker (e.g. the next_level function). Can you avoid the duplication by using a write_p2m_entry() callback, since that seems to be the part that's different?

Yes, I did it with a callback.

Quoted 3 lines

Skipping a flush when p2m->cr3 == 0 is not safe. I suggested using -1 as the magic value last time.

Done.

Quoted 3 lines

My comments on why p2m_flush_locked() isn't enough to reclaim an in-use p2m for recycling still stand.

Can you point me to the mail in the ML archive you refer to, please?

Quoted 28 lines

I have one more comment on the new code:

Quoted 13 lines · Christoph Egger

hap_write_p2m_entry(struct vcpu *v, unsigned long gfn, l1_pgentry_t *p,
mfn_t table_mfn, l1_pgentry_t new, unsigned int
level) {

xen/arch/x86/mm/hap/hap.c +5 -2
+    struct domain *d = v->domain;
     uint32_t old_flags;
 
-    hap_lock(v->domain);
+    old_flags = l1e_get_flags(*p);
 
-    old_flags = l1e_get_flags(*p);
+    /* We know always use the host p2m here, regardless if the vcpu
+     * is in host or guest mode. The vcpu can be in guest mode by
+     * a hypercall which passes a domain and chooses mostly the first

xen/arch/x86/mm/hap/hap.c static void · line 875

Quoted 10 lines
xen/arch/x86/mm/hap/hap.c +10 -0
+     * vcpu.
+     * XXX This is the reason why this function can not be used re-used
+     * for updating the nestedp2m. Otherwise, hypercalls would randomly
+     * operate on host p2m and nested p2m.
+     */
+    if ( nestedhvm_enabled(d) ) {
+        mfn_t omfn = _mfn(l1e_get_pfn(*p));
+        p2m_type_t op2mt = p2m_flags_to_type(old_flags);
+
+        if ( p2m_is_valid(op2mt) && mfn_valid(omfn) ) {
Quoted 2 lines

I think you need to do this flush even if !mfn_valid(omfn) - for example if you're removing a mapping of an MMIO area.

Fixed.

Quoted 11 lines

xen/arch/x86/mm/hap/hap.c static void · line 881

Quoted 7 lines
xen/arch/x86/mm/hap/hap.c +6 -0
+            mfn_t nmfn = _mfn(l1e_get_pfn(new));
+            p2m_type_t np2mt = p2m_flags_to_type(l1e_get_flags(new));
+
+            if ( p2m_is_valid(np2mt)
+                && mfn_valid(nmfn)
+                && !(l1e_get_flags(new) & _PAGE_PRESENT) )

I don't understand this test at all - you need to flush if you're removing an old present entry regardless of what replaces it. The only case where you can skip the flush is if old == new.

Fixed.

Quoted 17 lines

Cheers,

Tim.

Quoted 13 lines
xen/arch/x86/mm/hap/hap.c +5 -0
+            {
+                /* This GFN -> MFN is going to get removed. */
+                /* XXX There is a more efficient way to do that
+                 * but it works for now.
+                 * Note, p2m_flush_nestedp2m calls hap_lock()

internally. + */

xen/arch/x86/mm/hap/hap.c +7 -0
+                p2m_flush_nestedp2m(d);
+            }
+        }
+    }
+
+    hap_lock(d);
+
Signature · 6 lines
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#3 lore raw
Quoted 4 lines · Christoph Egger
Quoted 2 lines

My comments on why p2m_flush_locked() isn't enough to reclaim an in-use p2m for recycling still stand.

Can you point me to the mail in the ML archive you refer to, please?

It's the discussion at the end of this email:
http://lists.xensource.com/archives/html/xen-devel/2010-09/msg00624.html

Cheers,

Tim.

Signature · 3 lines
Tim Deegan <[email protected]>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)
#4 lore raw
Quoted 8 lines · On Wednesday 08 December 2010 11:17:15 Tim Deegan wrote:
Quoted 4 lines · Christoph Egger
Quoted 2 lines

My comments on why p2m_flush_locked() isn't enough to reclaim an in-use p2m for recycling still stand.

Can you point me to the mail in the ML archive you refer to, please?

It's the discussion at the end of this email:
http://lists.xensource.com/archives/html/xen-devel/2010-09/msg00624.html

Tnx. I see it is related to your suggestion to check cr3 against -1 you already mentioned. This is already fixed in my local tree.

Christoph

Signature · 6 lines
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#5 lore raw
Quoted 5 lines · Christoph Egger
Quoted 4 lines · On Wednesday 08 December 2010 11:17:15 Tim Deegan wrote:
Quoted 3 lines · Christoph Egger
Quoted 2 lines

My comments on why p2m_flush_locked() isn't enough to reclaim an in-use p2m for recycling still stand.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Can you point me to the mail in the ML archive you refer to, please?

It's the discussion at the end of this email:
http://lists.xensource.com/archives/html/xen-devel/2010-09/msg00624.html

Tnx. I see it is related to your suggestion to check cr3 against -1 you already mentioned.

It's similar, but different. In particular, checking CR3 against -1 may not fix it. It's possible that I'm just missing the path on vmentry that checks the p2m hasn't been reassigned.

Quoting my two concerns from before:

Quoted 11 lines
Quoted 3 lines

Is this enough? If this p2m might be in host_vmcb->h_cr3 somewhere on another vcpu, then you need to make sure that vcpu gets reset not to use it any more.

There are three possibilities:
An other vcpu is in VMRUN emulation before a nestedp2m is assigned.
In this case, it will get a new nestedp2m as it won't find its 'old' nestedp2m anymore.

An other vcpu is in VMRUN emulation after a nestedp2m is assigned.
It will VMEXIT with a nested page fault.

Why?

Quoted 2 lines

An other vcpu already running l2 guest.
It will VMEXIT with a nested page fault immediately.

Hmm. It will exit for the TLB shootdown IPI, but I think you need to clear vcpu_nestedhvm(v).nh_p2m on the other vcpu to make sure it doesn't re-enter with the p2m you've just recycled.

Cheers,

Tim.

Signature · 3 lines
Tim Deegan <[email protected]>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)
#6 lore raw
Quoted 12 lines · On Wednesday 08 December 2010 11:55:08 Tim Deegan wrote:
Quoted 11 lines · Christoph Egger
Quoted 9 lines · On Wednesday 08 December 2010 11:17:15 Tim Deegan wrote:
Quoted 4 lines · Christoph Egger
Quoted 2 lines

My comments on why p2m_flush_locked() isn't enough to reclaim an in-use p2m for recycling still stand.

Can you point me to the mail in the ML archive you refer to, please?

It's the discussion at the end of this email:
http://lists.xensource.com/archives/html/xen-devel/2010-09/msg00624.htm
l

Quoted 6 lines
Quoted 2 lines

Tnx. I see it is related to your suggestion to check cr3 against -1 you already mentioned.

It's similar, but different. In particular, checking CR3 against -1 may not fix it. It's possible that I'm just missing the path on vmentry that checks the p2m hasn't been reassigned.

My comments are based on the seventh patch series I just sent out.

The assumption is that the p2m is *empty*. So in case it is reassigned the (v)cpu will fall out with a nested page fault since the MMU can't do a page table walk.

Quoted 11 lines

Quoting my two concerns from before:

Quoted 9 lines
Quoted 3 lines

Is this enough? If this p2m might be in host_vmcb->h_cr3 somewhere on another vcpu, then you need to make sure that vcpu gets reset not to use it any more.

There are three possibilities:
An other vcpu is in VMRUN emulation before a nestedp2m is assigned.
In this case, it will get a new nestedp2m as it won't find its 'old' nestedp2m anymore.

Quoted 4 lines
Quoted 2 lines

An other vcpu is in VMRUN emulation after a nestedp2m is assigned.
It will VMEXIT with a nested page fault.

Why?

Because the p2m is empty. The MMU can not do a page table walk.

Quoted 6 lines
Quoted 2 lines

An other vcpu already running l2 guest.
It will VMEXIT with a nested page fault immediately.

Hmm. It will exit for the TLB shootdown IPI, but I think you need to clear vcpu_nestedhvm(v).nh_p2m on the other vcpu to make sure it doesn't re-enter with the p2m you've just recycled.

The p2m is empty so I don't see a problem when it gets recycled.

Christoph

Signature · 6 lines
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#7 lore raw
Quoted 7 lines · Christoph Egger
Quoted 4 lines
Quoted 2 lines

An other vcpu is in VMRUN emulation after a nestedp2m is assigned.
It will VMEXIT with a nested page fault.

Why?

Because the p2m is empty. The MMU can not do a page table walk.

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

An other vcpu already running l2 guest.
It will VMEXIT with a nested page fault immediately.

Hmm. It will exit for the TLB shootdown IPI, but I think you need to clear vcpu_nestedhvm(v).nh_p2m on the other vcpu to make sure it doesn't re-enter with the p2m you've just recycled.

The p2m is empty so I don't see a problem when it gets recycled.

It's only empty very briefly. You've assigned it to a vcpu which is about to take a nested fault and fill it with entries, right?

What happens if the other vcpu is handling an SMI or executing a tight loop of register arithmetic for a few thousand cycles? What stops it seeing the new contents of the p2m?

Tim.

Signature · 3 lines
Tim Deegan <[email protected]>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)