RFC: wait queue usage

diff -r d3859e348951 -r d347a8a36d2e xen/arch/x86/hvm/hvm.c
@@ -4098,7 +4098,7 @@ static int hvm_memory_event_traps(long p
     if ( (p & HVMPME_onchangeonly) && (value == old) )
         return 1;
     
-    rc = mem_event_check_ring(d, &d->mem_access);
+    rc = mem_event_check_ring(d, &d->mem_event->mem_access);
     if ( rc )
         return rc;
     

@@ -4121,7 +4121,7 @@ static int hvm_memory_event_traps(long p
         req.gla_valid = 1;
     }
     
-    mem_event_put_request(d, &d->mem_access, &req);
+    mem_event_put_request(d, &d->mem_event->mem_access, &req);
     
     return 1;
 }

diff -r d3859e348951 -r d347a8a36d2e xen/arch/x86/mm/mem_event.c
@@ -265,7 +265,7 @@ int mem_event_domctl(struct domain *d, x
     {
     case XEN_DOMCTL_MEM_EVENT_OP_PAGING:
     {
-        struct mem_event_domain *med = &d->mem_paging;
+        struct mem_event_domain *med = &d->mem_event->mem_paging;
         rc = -EINVAL;
 
         switch( mec->op )

@@ -310,7 +310,7 @@ int mem_event_domctl(struct domain *d, x
 
     case XEN_DOMCTL_MEM_EVENT_OP_ACCESS: 
     {
-        struct mem_event_domain *med = &d->mem_access;
+        struct mem_event_domain *med = &d->mem_event->mem_access;
         rc = -EINVAL;
 
         switch( mec->op )

@@ -333,7 +333,7 @@ int mem_event_domctl(struct domain *d, x
         case XEN_DOMCTL_MEM_EVENT_OP_ACCESS_DISABLE:
         {
             if ( med->ring_page )
-                rc = mem_event_disable(&d->mem_access);
+                rc = mem_event_disable(med);
         }
         break;
 

diff -r d3859e348951 -r d347a8a36d2e xen/arch/x86/mm/mem_sharing.c
@@ -281,12 +281,12 @@ static struct page_info* mem_sharing_all
     vcpu_pause_nosync(v);
     req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;
 
-    if(mem_event_check_ring(d, &d->mem_share)) return page;
+    if(mem_event_check_ring(d, &d->mem_event->mem_share)) return page;
 
     req.gfn = gfn;
     req.p2mt = p2m_ram_shared;
     req.vcpu_id = v->vcpu_id;
-    mem_event_put_request(d, &d->mem_share, &req);
+    mem_event_put_request(d, &d->mem_event->mem_share, &req);
 
     return page;
 }

@@ -301,7 +301,7 @@ int mem_sharing_sharing_resume(struct do
     mem_event_response_t rsp;
 
     /* Get request off the ring */
-    mem_event_get_response(&d->mem_share, &rsp);
+    mem_event_get_response(&d->mem_event->mem_share, &rsp);
 
     /* Unpause domain/vcpu */
     if( rsp.flags & MEM_EVENT_FLAG_VCPU_PAUSED )

diff -r d3859e348951 -r d347a8a36d2e xen/arch/x86/mm/p2m.c
@@ -885,7 +885,7 @@ void p2m_mem_paging_drop_page(struct dom
     mem_event_request_t req;
 
     /* Check that there's space on the ring for this request */
-    if ( mem_event_check_ring(d, &d->mem_paging) == 0)
+    if ( mem_event_check_ring(d, &d->mem_event->mem_paging) == 0)
     {
         /* Send release notification to pager */
         memset(&req, 0, sizeof(req));

@@ -893,7 +893,7 @@ void p2m_mem_paging_drop_page(struct dom
         req.gfn = gfn;
         req.vcpu_id = v->vcpu_id;
 
-        mem_event_put_request(d, &d->mem_paging, &req);
+        mem_event_put_request(d, &d->mem_event->mem_paging, &req);
     }
 }
 

@@ -928,7 +928,7 @@ void p2m_mem_paging_populate(struct doma
     struct p2m_domain *p2m = p2m_get_hostp2m(d);
 
     /* Check that there's space on the ring for this request */
-    if ( mem_event_check_ring(d, &d->mem_paging) )
+    if ( mem_event_check_ring(d, &d->mem_event->mem_paging) )
         return;
 
     memset(&req, 0, sizeof(req));

@@ -959,7 +959,7 @@ void p2m_mem_paging_populate(struct doma
     else if ( p2mt != p2m_ram_paging_out && p2mt != p2m_ram_paged )
     {
         /* gfn is already on its way back and vcpu is not paused */
-        mem_event_put_req_producers(&d->mem_paging);
+        mem_event_put_req_producers(&d->mem_event->mem_paging);
         return;
     }
 

@@ -968,7 +968,7 @@ void p2m_mem_paging_populate(struct doma
     req.p2mt = p2mt;
     req.vcpu_id = v->vcpu_id;
 
-    mem_event_put_request(d, &d->mem_paging, &req);
+    mem_event_put_request(d, &d->mem_event->mem_paging, &req);
 }
 
 /**

@@ -1048,7 +1048,7 @@ void p2m_mem_paging_resume(struct domain
     mfn_t mfn;
 
     /* Pull the response off the ring */
-    mem_event_get_response(&d->mem_paging, &rsp);
+    mem_event_get_response(&d->mem_event->mem_paging, &rsp);
 
     /* Fix p2m entry if the page was not dropped */
     if ( !(rsp.flags & MEM_EVENT_FLAG_DROP_PAGE) )

@@ -1101,7 +1101,7 @@ void p2m_mem_access_check(unsigned long 
     p2m_unlock(p2m);
 
     /* Otherwise, check if there is a memory event listener, and send the message along */
-    res = mem_event_check_ring(d, &d->mem_access);
+    res = mem_event_check_ring(d, &d->mem_event->mem_access);
     if ( res < 0 ) 
     {
         /* No listener */

@@ -1145,7 +1145,7 @@ void p2m_mem_access_check(unsigned long 
     
     req.vcpu_id = v->vcpu_id;
 
-    mem_event_put_request(d, &d->mem_access, &req);
+    mem_event_put_request(d, &d->mem_event->mem_access, &req);
 
     /* VCPU paused, mem event request sent */
 }

@@ -1155,7 +1155,7 @@ void p2m_mem_access_resume(struct p2m_do
     struct domain *d = p2m->domain;
     mem_event_response_t rsp;
 
-    mem_event_get_response(&d->mem_access, &rsp);
+    mem_event_get_response(&d->mem_event->mem_access, &rsp);
 
     /* Unpause domain */
     if ( rsp.flags & MEM_EVENT_FLAG_VCPU_PAUSED )

diff -r d3859e348951 -r d347a8a36d2e xen/common/domain.c
@@ -304,6 +304,10 @@ struct domain *domain_create(
         init_status |= INIT_gnttab;
 
         poolid = 0;
+
+        d->mem_event = xzalloc(struct mem_event_per_domain);
+        if ( !d->mem_event )
+            goto fail;
     }
 
     if ( arch_domain_create(d, domcr_flags) != 0 )

@@ -335,6 +339,7 @@ struct domain *domain_create(
  fail:
     d->is_dying = DOMDYING_dead;
     atomic_set(&d->refcnt, DOMAIN_DESTROYED);
+    xfree(d->mem_event);
     if ( init_status & INIT_arch )
         arch_domain_destroy(d);
     if ( init_status & INIT_gnttab )

diff -r d3859e348951 -r d347a8a36d2e xen/include/xen/sched.h
@@ -194,6 +194,16 @@ struct mem_event_domain
     int xen_port;
 };
 
+struct mem_event_per_domain
+{
+    /* Memory sharing support */
+    struct mem_event_domain mem_share;
+    /* Memory paging support */
+    struct mem_event_domain mem_paging;
+    /* Memory access support */
+    struct mem_event_domain mem_access;
+};
+
 struct domain
 {
     domid_t          domain_id;

@@ -318,12 +328,8 @@ struct domain
     /* Non-migratable and non-restoreable? */
     bool_t disable_migrate;
 
-    /* Memory sharing support */
-    struct mem_event_domain mem_share;
-    /* Memory paging support */
-    struct mem_event_domain mem_paging;
-    /* Memory access support */
-    struct mem_event_domain mem_access;
+    /* Various mem_events */
+    struct mem_event_per_domain *mem_event;
 
     /* Currently computed from union of all vcpu cpu-affinity masks. */
     nodemask_t node_affinity;

diff -r d347a8a36d2e -r de6860cb9205 xen/arch/x86/hvm/hvm.c
@@ -4099,7 +4099,7 @@ static int hvm_memory_event_traps(long p
         return 1;
     
     rc = mem_event_check_ring(d, &d->mem_event->mem_access);
-    if ( rc )
+    if ( rc < 0 )
         return rc;
     
     memset(&req, 0, sizeof(req));

162 lines not on this page · raw message

diff -r d347a8a36d2e -r de6860cb9205 xen/arch/x86/mm/mem_sharing.c
@@ -253,19 +253,11 @@ static void mem_sharing_audit(void)
 #endif
 
 
-static struct page_info* mem_sharing_alloc_page(struct domain *d, 
-                                                unsigned long gfn)
+static void mem_sharing_notify_helper(struct domain *d, unsigned long gfn)
 {
-    struct page_info* page;
     struct vcpu *v = current;
     mem_event_request_t req;
 
-    page = alloc_domheap_page(d, 0); 
-    if(page != NULL) return page;
-
-    memset(&req, 0, sizeof(req));
-    req.type = MEM_EVENT_TYPE_SHARED;
-
     if ( v->domain != d )
     {
         /* XXX This path needs some attention.  For now, just fail foreign 

@@ -275,20 +267,20 @@ static struct page_info* mem_sharing_all
         gdprintk(XENLOG_ERR, 
                  "Failed alloc on unshare path for foreign (%d) lookup\n",
                  d->domain_id);
-        return page;
+        return;
     }
 
-    vcpu_pause_nosync(v);
-    req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;
+    if (mem_event_check_ring(d, &d->mem_event->mem_share) < 0)
+        return;
 
-    if(mem_event_check_ring(d, &d->mem_event->mem_share)) return page;
-
+    memset(&req, 0, sizeof(req));
+    req.type = MEM_EVENT_TYPE_SHARED;
+    req.flags = MEM_EVENT_FLAG_VCPU_PAUSED;
     req.gfn = gfn;
     req.p2mt = p2m_ram_shared;
     req.vcpu_id = v->vcpu_id;
     mem_event_put_request(d, &d->mem_event->mem_share, &req);
-
-    return page;
+    vcpu_pause_nosync(v);
 }
 
 unsigned int mem_sharing_get_nr_saved_mfns(void)

@@ -653,7 +645,7 @@ gfn_found:
     if(ret == 0) goto private_page_found;
         
     old_page = page;
-    page = mem_sharing_alloc_page(d, gfn);
+    page = alloc_domheap_page(d, 0); 
     if(!page) 
     {
         /* We've failed to obtain memory for private page. Need to re-add the

@@ -661,6 +653,7 @@ gfn_found:
         list_add(&gfn_info->list, &hash_entry->gfns);
         put_gfn(d, gfn);
         shr_unlock();
+        mem_sharing_notify_helper(d, gfn);
         return -ENOMEM;
     }
 

diff -r d347a8a36d2e -r de6860cb9205 xen/arch/x86/mm/p2m.c
@@ -884,17 +884,13 @@ void p2m_mem_paging_drop_page(struct dom
     struct vcpu *v = current;
     mem_event_request_t req;
 
-    /* Check that there's space on the ring for this request */
-    if ( mem_event_check_ring(d, &d->mem_event->mem_paging) == 0)
-    {
-        /* Send release notification to pager */
-        memset(&req, 0, sizeof(req));
-        req.flags |= MEM_EVENT_FLAG_DROP_PAGE;
-        req.gfn = gfn;
-        req.vcpu_id = v->vcpu_id;
+    /* Send release notification to pager */
+    memset(&req, 0, sizeof(req));
+    req.flags = MEM_EVENT_FLAG_DROP_PAGE;
+    req.gfn = gfn;
+    req.vcpu_id = v->vcpu_id;
 
-        mem_event_put_request(d, &d->mem_event->mem_paging, &req);
-    }
+    mem_event_put_request(d, &d->mem_event->mem_paging, &req);
 }
 
 /**

@@ -952,7 +948,6 @@ void p2m_mem_paging_populate(struct doma
     /* Pause domain if request came from guest and gfn has paging type */
     if (  p2m_is_paging(p2mt) && v->domain == d )
     {
-        vcpu_pause_nosync(v);
         req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;
     }
     /* No need to inform pager if the gfn is not in the page-out path */

@@ -969,6 +964,10 @@ void p2m_mem_paging_populate(struct doma
     req.vcpu_id = v->vcpu_id;
 
     mem_event_put_request(d, &d->mem_event->mem_paging, &req);
+
+    /* Pause guest after put_request to allow wake_up after wait_event */
+    if ( req.flags & MEM_EVENT_FLAG_VCPU_PAUSED )
+        vcpu_pause_nosync(v);
 }
 
 /**

@@ -1071,8 +1070,8 @@ void p2m_mem_paging_resume(struct domain
     if ( rsp.flags & MEM_EVENT_FLAG_VCPU_PAUSED )
         vcpu_unpause(d->vcpu[rsp.vcpu_id]);
 
-    /* Unpause any domains that were paused because the ring was full */
-    mem_event_unpause_vcpus(d);
+    /* Unpause all vcpus that were paused because the ring was full */
+    wake_up(&d->mem_event->mem_paging.wq);
 }
 
 void p2m_mem_access_check(unsigned long gpa, bool_t gla_valid, unsigned long gla, 

@@ -1111,7 +1110,7 @@ void p2m_mem_access_check(unsigned long 
                    "Memory access permissions failure, no mem_event listener: pausing VCPU %d, dom %d\n",
                    v->vcpu_id, d->domain_id);
 
-            mem_event_mark_and_pause(v);
+            mem_event_mark_and_pause(v, &d->mem_event->mem_access);
         }
         else
         {

@@ -1131,8 +1130,7 @@ void p2m_mem_access_check(unsigned long 
     req.reason = MEM_EVENT_REASON_VIOLATION;
 
     /* Pause the current VCPU unconditionally */
-    vcpu_pause_nosync(v);
-    req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;    
+    req.flags = MEM_EVENT_FLAG_VCPU_PAUSED;    
 
     /* Send request to mem event */
     req.gfn = gfn;

@@ -1148,6 +1146,7 @@ void p2m_mem_access_check(unsigned long 
     mem_event_put_request(d, &d->mem_event->mem_access, &req);
 
     /* VCPU paused, mem event request sent */
+    vcpu_pause_nosync(v);
 }
 
 void p2m_mem_access_resume(struct p2m_domain *p2m)

@@ -1161,9 +1160,11 @@ void p2m_mem_access_resume(struct p2m_do
     if ( rsp.flags & MEM_EVENT_FLAG_VCPU_PAUSED )
         vcpu_unpause(d->vcpu[rsp.vcpu_id]);
 
-    /* Unpause any domains that were paused because the ring was full or no listener 
-     * was available */
-    mem_event_unpause_vcpus(d);
+    /* Wakeup all vcpus waiting because the ring was full */
+    wake_up(&d->mem_event->mem_access.wq);
+
+    /* Unpause all vcpus that were paused because no listener was available */
+    mem_event_unpause_vcpus(d, &d->mem_event->mem_access);
 }
 
 

diff -r d347a8a36d2e -r de6860cb9205 xen/include/asm-x86/mem_event.h
@@ -25,12 +25,12 @@
 #define __MEM_EVENT_H__
 
 /* Pauses VCPU while marking pause flag for mem event */
-void mem_event_mark_and_pause(struct vcpu *v);
+void mem_event_mark_and_pause(struct vcpu *v, struct mem_event_domain *med);
 int mem_event_check_ring(struct domain *d, struct mem_event_domain *med);
 void mem_event_put_req_producers(struct mem_event_domain *med);
 void mem_event_put_request(struct domain *d, struct mem_event_domain *med, mem_event_request_t *req);
 void mem_event_get_response(struct mem_event_domain *med, mem_event_response_t *rsp);
-void mem_event_unpause_vcpus(struct domain *d);
+void mem_event_unpause_vcpus(struct domain *d, struct mem_event_domain *med);
 
 int mem_event_domctl(struct domain *d, xen_domctl_mem_event_op_t *mec,
                      XEN_GUEST_HANDLE(void) u_domctl);

diff -r d347a8a36d2e -r de6860cb9205 xen/include/xen/sched.h
@@ -14,6 +14,7 @@
 #include <xen/nodemask.h>
 #include <xen/radix-tree.h>
 #include <xen/multicall.h>
+#include <xen/wait.h>
 #include <public/xen.h>
 #include <public/domctl.h>
 #include <public/sysctl.h>

@@ -192,6 +193,10 @@ struct mem_event_domain
     mem_event_front_ring_t front_ring;
     /* event channel port (vcpu0 only) */
     int xen_port;
+    /* mem_event bit for vcpu->pause_flags */
+    int mem_event_bit;
+    /* list of vcpus waiting for room in the ring */
+    struct waitqueue_head wq;
 };
 
 struct mem_event_per_domain

@@ -615,9 +620,12 @@ static inline struct domain *next_domain
  /* VCPU affinity has changed: migrating to a new CPU. */
 #define _VPF_migrating       3
 #define VPF_migrating        (1UL<<_VPF_migrating)
- /* VCPU is blocked on memory-event ring. */
-#define _VPF_mem_event       4
-#define VPF_mem_event        (1UL<<_VPF_mem_event)
+ /* VCPU is blocked on mem_paging ring. */
+#define _VPF_me_mem_paging   4
+#define VPF_me_mem_paging    (1UL<<_VPF_me_mem_paging)
+ /* VCPU is blocked on mem_access ring. */
+#define _VPF_me_mem_access   5
+#define VPF_me_mem_access    (1UL<<_VPF_me_mem_access)
 
 static inline int vcpu_runnable(struct vcpu *v)
 {

diff -r de6860cb9205 -r 9d63ecd3969b xen/arch/x86/mm/p2m-ept.c
@@ -505,7 +505,7 @@ out:
 }
 
 /* Read ept p2m entries */
-static mfn_t ept_get_entry(struct p2m_domain *p2m,
+static unsigned long _ept_get_entry(struct p2m_domain *p2m,
                            unsigned long gfn, p2m_type_t *t, p2m_access_t* a,
                            p2m_query_t q, unsigned int *page_order)
 {

@@ -516,7 +516,7 @@ static mfn_t ept_get_entry(struct p2m_do
     u32 index;
     int i;
     int ret = 0;
-    mfn_t mfn = _mfn(INVALID_MFN);
+    unsigned long mfn = INVALID_MFN;
 
     *t = p2m_mmio_dm;
     *a = p2m_access_n;

@@ -582,17 +582,14 @@ static mfn_t ept_get_entry(struct p2m_do
         *t = ept_entry->sa_p2mt;
         *a = ept_entry->access;
 
-        mfn = _mfn(ept_entry->mfn);
+        mfn = ept_entry->mfn;
         if ( i )
         {
             /* 
              * We may meet super pages, and to split into 4k pages
              * to emulate p2m table
              */
-            unsigned long split_mfn = mfn_x(mfn) +
-                (gfn_remainder &
-                 ((1 << (i * EPT_TABLE_ORDER)) - 1));
-            mfn = _mfn(split_mfn);
+            mfn += (gfn_remainder & ((1 << (i * EPT_TABLE_ORDER)) - 1));
         }
 
         if ( page_order )

@@ -604,6 +601,41 @@ out:
     return mfn;
 }
 
+static int
+ept_get_entry_wait(unsigned long *mfn, int *populated,
+               struct p2m_domain *p2m, unsigned long gfn, 
+               p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
+               unsigned int *page_order)
+{
+    int ret = 1;
+    *mfn = _ept_get_entry(p2m, gfn, t, a, q, page_order);
+
+    /* No further action in case of query */
+    if ( q == p2m_query )
+        goto done;
+
+    /* Populate the page once in case of guest access, then go to sleep */
+    if ( p2m_is_paging(*t) && current->domain == p2m->domain ) {
+        if ( *populated == 0 ) {
+            *populated = 1;
+            p2m_mem_paging_populate(p2m->domain, gfn);
+        }
+        ret = 0;
+    }
+done:
+    return ret;
+}
+static mfn_t
+ept_get_entry(struct p2m_domain *p2m, unsigned long gfn, 
+               p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
+               unsigned int *page_order)
+{
+    unsigned long mfn;
+    int populated = 0;
+    wait_event(p2m->wq, ept_get_entry_wait(&mfn, &populated, p2m, gfn, t, a, q, page_order));
+    return _mfn(mfn);
+}
+
 /* WARNING: Only caller doesn't care about PoD pages.  So this function will
  * always return 0 for PoD pages, not populate them.  If that becomes necessary,
  * pass a p2m_query_t type along to distinguish. */

diff -r de6860cb9205 -r 9d63ecd3969b xen/arch/x86/mm/p2m.c
@@ -76,6 +76,7 @@ static void p2m_initialise(struct domain
     INIT_PAGE_LIST_HEAD(&p2m->pages);
     INIT_PAGE_LIST_HEAD(&p2m->pod.super);
     INIT_PAGE_LIST_HEAD(&p2m->pod.single);
+    init_waitqueue_head(&p2m->wq);
 
     p2m->domain = d;
     p2m->default_access = p2m_access_rwx;

@@ -1072,6 +1073,8 @@ void p2m_mem_paging_resume(struct domain
 
     /* Unpause all vcpus that were paused because the ring was full */
     wake_up(&d->mem_event->mem_paging.wq);
+    /* Unpause all vcpus that were paused because the gfn was paged */
+    wake_up(&p2m->wq);
 }
 
 void p2m_mem_access_check(unsigned long gpa, bool_t gla_valid, unsigned long gla, 

diff -r de6860cb9205 -r 9d63ecd3969b xen/include/asm-x86/p2m.h
@@ -286,6 +286,7 @@ struct p2m_domain {
         unsigned         reclaim_single; /* Last gpfn of a scan */
         unsigned         max_guest;    /* gpfn of max guest demand-populate */
     } pod;
+    struct waitqueue_head wq;
 };
 
 /* get host p2m table */