Skip to contentSkip to conversation

xen/vNUMA: vNUMA support for PV guests.

RFC PATCH In review Elena Ufimtseva 6 participants 14 messages Sep 13, 2013 – Oct 3, 2013

lore
Share & tools
lore
#0 lore raw

RFC PATCH v2 1/7 xen/vNUMA: vNUMA support for PV guests.

Defines XENMEM subop hypercall for PV vNUMA enabled guests and provides vNUMA topology information from per-domain vnuma topology build info.

---
Changes since v1:
* changed type from int to uint and unsigned in vNUMA structures;
* removed unecessary file vnuma.h as requested in review
* added domain_vnuma_destroy;
* fixed usage of rcu_lock_domain_by_any_id;
* removed unecessary guest_handle_cast calls;
* coding style fixes;

---
 xen/common/domain.c         |   25 +++++++++++++++-
 xen/common/domctl.c         |   68 ++++++++++++++++++++++++++++++++++++++++++-
 xen/common/memory.c         |   56 +++++++++++++++++++++++++++++++++++
 xen/include/public/domctl.h |   15 +++++++++-
 xen/include/public/memory.h |    9 +++++-
 xen/include/xen/domain.h    |   11 +++++++
 xen/include/xen/sched.h     |    1 +
 xen/include/xen/vnuma.h     |   27 +++++++++++++++++
 8 files changed, 208 insertions(+), 4 deletions(-)
 create mode 100644 xen/include/xen/vnuma.h
xen/common/domain.c +24 -1
index 9390a22..bb414cf 100644
@@ -227,6 +227,7 @@ struct domain *domain_create(
     spin_lock_init(&d->node_affinity_lock);
     d->node_affinity = NODE_MASK_ALL;
     d->auto_node_affinity = 1;
+    d->vnuma.nr_vnodes = 0;
 
     spin_lock_init(&d->shutdown_lock);
     d->shutdown_code = -1;
@@ -530,8 +531,9 @@ int domain_kill(struct domain *d)
         evtchn_destroy(d);
         gnttab_release_mappings(d);
         tmem_destroy(d->tmem);
-        domain_set_outstanding_pages(d, 0);
         d->tmem = NULL;
+        domain_set_outstanding_pages(d, 0);
+        domain_vnuma_destroy(&d->vnuma);
         /* fallthrough */
     case DOMDYING_dying:
         rc = domain_relinquish_resources(d);
@@ -1279,6 +1281,27 @@ int continue_hypercall_on_cpu(
     return 0;
 }
 
+void domain_vnuma_destroy(struct domain_vnuma_info *v)
+{
+    if (v->vnuma_memblks) {
+        xfree(v->vnuma_memblks);
+        v->vnuma_memblks = NULL;
+    }
+    if (v->vcpu_to_vnode) {
+        xfree(v->vcpu_to_vnode);
+        v->vcpu_to_vnode = NULL;
+    }
+    if (v->vdistance) {
+        xfree(v->vdistance);
+        v->vdistance = NULL;
+    }
+    if (v->vnode_to_pnode) {
+        xfree(v->vnode_to_pnode);
+        v->vnode_to_pnode = NULL;
+    }
+    v->nr_vnodes = 0;
+}
+
 /*
  * Local variables:
  * mode: C
xen/common/domctl.c +67 -1

80 lines not on this page · raw message

xen/common/memory.c +56 -0

69 lines not on this page · raw message

xen/include/public/domctl.h +14 -1
index 4c5b2bb..3574d0a 100644
@@ -35,6 +35,7 @@
 #include "xen.h"
 #include "grant_table.h"
 #include "hvm/save.h"
+#include "memory.h"
 
 #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009
 
@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {
 typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_broken_page_p2m_t);
 
+struct xen_domctl_vnuma {
+    uint16_t nr_vnodes;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(uint) vnode_to_pnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+
+typedef struct xen_domctl_vnuma xen_domctl_vnuma_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_vnuma_t);
+
 struct xen_domctl {
     uint32_t cmd;
 #define XEN_DOMCTL_createdomain                   1
@@ -920,6 +932,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_set_broken_page_p2m           67
 #define XEN_DOMCTL_setnodeaffinity               68
 #define XEN_DOMCTL_getnodeaffinity               69
+#define XEN_DOMCTL_setvnumainfo                  70
 #define XEN_DOMCTL_gdbsx_guestmemio            1000
 #define XEN_DOMCTL_gdbsx_pausevcpu             1001
 #define XEN_DOMCTL_gdbsx_unpausevcpu           1002
@@ -979,6 +992,7 @@ struct xen_domctl {
         struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
         struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
         struct xen_domctl_gdbsx_domstatus   gdbsx_domstatus;
+        struct xen_domctl_vnuma             vnuma;
         uint8_t                             pad[128];
     } u;
 };
@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_t);
 
 #endif /* __XEN_PUBLIC_DOMCTL_H__ */
-
 /*
  * Local variables:
  * mode: C
xen/include/public/memory.h +8 -1
index 7a26dee..28f6aaf 100644
@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);
  * Caller must be privileged or the hypercall fails.
  */
 #define XENMEM_claim_pages                  24
-
 /*
  * XENMEM_claim_pages flags - the are no flags at this time.
  * The zero value is appropiate.
  */
 
+struct vnuma_memblk {
+              uint64_t start, end;
+};
+typedef struct vnuma_memblk vnuma_memblk_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t);
+
+#define XENMEM_get_vnuma_info               25
+
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
 #endif /* __XEN_PUBLIC_MEMORY_H__ */
xen/include/xen/domain.h +11 -0
index a057069..c9d53e3 100644
@@ -4,6 +4,7 @@
 
 #include <public/xen.h>
 #include <asm/domain.h>
+#include <public/memory.h>
 
 typedef union {
     struct vcpu_guest_context *nat;
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;
 
 extern bool_t opt_dom0_vcpus_pin;
 
+struct domain_vnuma_info {
+    uint16_t nr_vnodes;
+    uint *vdistance;
+    uint *vcpu_to_vnode;
+    uint *vnode_to_pnode;
+    vnuma_memblk_t *vnuma_memblks;
+};
+
+void domain_vnuma_destroy(struct domain_vnuma_info *v);
+
 #endif /* __XEN_DOMAIN_H__ */
xen/include/xen/sched.h +1 -0
index ae6a3b8..cb023cf 100644
@@ -377,6 +377,7 @@ struct domain
     nodemask_t node_affinity;
     unsigned int last_alloc_node;
     spinlock_t node_affinity_lock;
+    struct domain_vnuma_info vnuma;
 };
 
 struct domain_setup_info
xen/include/xen/vnuma.h new file +27 -0
new file mode 100644
index 0000000..0b41da0
@@ -0,0 +1,27 @@
+#ifndef _VNUMA_H
+#define _VNUMA_H
+#include <public/memory.h>
+
+/* DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t); */
+
+struct vnuma_topology_info {
+    domid_t domid;
+    uint16_t nr_vnodes;
+    uint32_t _pad;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+typedef struct vnuma_topology_info vnuma_topology_info_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_topology_info_t);
+
+#define __vnode_distance_offset(_dom, _i, _j) \
+        ( ((_j) * ((_dom)->vnuma.nr_vnodes)) + (_i) )
+
+#define __vnode_distance(_dom, _i, _j) \
+        ( (_dom)->vnuma.vdistance[__vnode_distance_offset((_dom), (_i), (_j))] )
+
+#define __vnode_distance_set(_dom, _i, _j, _v) \
+        do { __vnode_distance((_dom), (_i), (_j)) = (_v); } while(0)
+
+#endif
Signature · 1 line
1.7.10.4
#1 lore raw
Quoted 31 lines · Elena Ufimtseva

Defines XENMEM subop hypercall for PV vNUMA enabled guests and provides vNUMA topology information from per-domain vnuma topology build info.

---
Changes since v1:
* changed type from int to uint and unsigned in vNUMA structures;
* removed unecessary file vnuma.h as requested in review
* added domain_vnuma_destroy;
* fixed usage of rcu_lock_domain_by_any_id;
* removed unecessary guest_handle_cast calls;
* coding style fixes;

---
 xen/common/domain.c         |   25 +++++++++++++++-
 xen/common/domctl.c         |   68 ++++++++++++++++++++++++++++++++++++++++++-
 xen/common/memory.c         |   56 +++++++++++++++++++++++++++++++++++
 xen/include/public/domctl.h |   15 +++++++++-
 xen/include/public/memory.h |    9 +++++-
 xen/include/xen/domain.h    |   11 +++++++
 xen/include/xen/sched.h     |    1 +
 xen/include/xen/vnuma.h     |   27 +++++++++++++++++
 8 files changed, 208 insertions(+), 4 deletions(-)
 create mode 100644 xen/include/xen/vnuma.h
xen/common/domain.c +0 -0
index 9390a22..bb414cf 100644

xen/common/domain.c struct domain *domain_create( · line 230

Quoted 4 lines
xen/common/domain.c +1 -0
@@ -227,6 +227,7 @@ struct domain *domain_create(
     spin_lock_init(&d->node_affinity_lock);
     d->node_affinity = NODE_MASK_ALL;
     d->auto_node_affinity = 1;
+    d->vnuma.nr_vnodes = 0;

alloc_domain_struct() clears d, so this assignment of 0 is not needed.

xen/common/domain.c int domain_kill(struct domain *d) · line 533

Quoted 8 lines
xen/common/domain.c +0 -1
     spin_lock_init(&d->shutdown_lock);
     d->shutdown_code = -1;
@@ -530,8 +531,9 @@ int domain_kill(struct domain *d)
         evtchn_destroy(d);
         gnttab_release_mappings(d);
         tmem_destroy(d->tmem);
-        domain_set_outstanding_pages(d, 0);

Any particular reason for this function call to move? I cant see any reason.

Quoted 7 lines
xen/common/domain.c +2 -0
         d->tmem = NULL;
+        domain_set_outstanding_pages(d, 0);
+        domain_vnuma_destroy(&d->vnuma);
         /* fallthrough */
     case DOMDYING_dying:
         rc = domain_relinquish_resources(d);

xen/common/domain.c int continue_hypercall_on_cpu( · line 1289

Quoted 9 lines
xen/common/domain.c +6 -0
@@ -1279,6 +1281,27 @@ int continue_hypercall_on_cpu(
     return 0;
 }
 
+void domain_vnuma_destroy(struct domain_vnuma_info *v)
+{
+    if (v->vnuma_memblks) {
+        xfree(v->vnuma_memblks);
+        v->vnuma_memblks = NULL;
+    }

Coding style (see CODING_STYLE in the root of the tree) so this would become:

if ( v->vnuma_memblks )
{
xfree(v->vnuma_memblks);
v->vnuma_memblks = NULL;
}

However, xfree() (like regular free()) is happy with NULL pointers, so you can drop the if altogether.

Quoted 31 lines
xen/common/domain.c +15 -0
+    if (v->vcpu_to_vnode) {
+        xfree(v->vcpu_to_vnode);
+        v->vcpu_to_vnode = NULL;
+    }
+    if (v->vdistance) {
+        xfree(v->vdistance);
+        v->vdistance = NULL;
+    }
+    if (v->vnode_to_pnode) {
+        xfree(v->vnode_to_pnode);
+        v->vnode_to_pnode = NULL;
+    }
+    v->nr_vnodes = 0;
+}
+
 /*
  * Local variables:
  * mode: C
xen/common/domctl.c +1 -0
index 9760d50..e5d05c7 100644
@@ -29,6 +29,7 @@
 #include <asm/page.h>
 #include <public/domctl.h>
 #include <xsm/xsm.h>
+#include <xen/vnuma.h>
 
 static DEFINE_SPINLOCK(domctl_lock);
 DEFINE_SPINLOCK(vcpu_alloc_lock);

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 865

Quoted 4 lines
xen/common/domctl.c +0 -1
@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
         ret = set_global_virq_handler(d, virq);
     }
     break;
-

Please leave this blank line in here.

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876

Quoted 11 lines
xen/common/domctl.c +11 -0
+    case XEN_DOMCTL_setvnumainfo:
+    {
+        unsigned int i, j, nr_vnodes, dist_size;
+        unsigned int dist, vmap, vntop;
+        vnuma_memblk_t vmemblk;
+        
+        ret = -EINVAL;
+        dist = i = j = 0;
+        nr_vnodes = op->u.vnuma.nr_vnodes; 
+        if (nr_vnodes <= 0 || nr_vnodes > NR_CPUS)
+            break;

nr_vnodes is unsigned. It cannot possibly be less than 0.

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 881

Quoted 5 lines
xen/common/domctl.c +5 -0
+        d->vnuma.nr_vnodes = nr_vnodes;
+        dist_size = nr_vnodes * nr_vnodes;
+        if (
+        (d->vnuma.vdistance = xmalloc_bytes(
+                 sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||

sizeof in an operator not a function. As you are not passing a type, the brackets are not required.

Quoted 4 lines
xen/common/domctl.c +4 -0
+        (d->vnuma.vnuma_memblks = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnuma_memblks) * nr_vnodes)) == NULL ||
+        (d->vnuma.vcpu_to_vnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) == NULL ||

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 895

Quoted 10 lines
xen/common/domctl.c +10 -0
+        (d->vnuma.vnode_to_pnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnode_to_pnode) * nr_vnodes)) == NULL )
+            goto err_dom;
+        for ( i = 0; i < nr_vnodes; i++ )
+            for ( j = 0; j < nr_vnodes; j++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&dist,
+                                        op->u.vnuma.vdistance,
+                                        __vnode_distance_offset(d, i, j), 1)) )
+                    goto err_dom;

This error logic is broken. A failure in __copy_from_guest_offset() should result in -EFAULT, but will result in -EINVAL after following err_dom;

Quoted 26 lines
xen/common/domctl.c +26 -0
+                __vnode_distance_set(d, i, j, dist);
+            }
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmemblk,
+                                    op->u.vnuma.vnuma_memblks, i, 1)) )
+                goto err_dom;
+            d->vnuma.vnuma_memblks[i].start = vmemblk.start;
+            d->vnuma.vnuma_memblks[i].end = vmemblk.end;
+        }
+        for ( i = 0; i < d->max_vcpus; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmap,
+                           op->u.vnuma.vcpu_to_vnode, i, 1)) )
+                goto err_dom;
+            d->vnuma.vcpu_to_vnode[i] = vmap;
+        }
+        if ( !guest_handle_is_null(op->u.vnuma.vnode_to_pnode) )
+        {
+            for ( i = 0; i < nr_vnodes; i++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&vntop,
+                                        op->u.vnuma.vnode_to_pnode, i, 1)) )
+                    goto err_dom;
+                d->vnuma.vnode_to_pnode[i] = vntop;
+            }

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 931

Quoted 10 lines
xen/common/domctl.c +10 -0
+        }
+        else
+            for(i = 0; i < nr_vnodes; i++)
+                d->vnuma.vnode_to_pnode[i] = NUMA_NO_NODE;
+        ret = 0;
+        break;
+err_dom:
+        ret = -EINVAL;
+    }
+    break;

And an extra line in here please.

Quoted 23 lines
xen/common/domctl.c +0 -0
     default:
         ret = arch_do_domctl(op, d, u_domctl);
         break;
xen/common/memory.c +5 -0
index 50b740f..6dc2452 100644
@@ -28,6 +28,7 @@
 #include <public/memory.h>
 #include <xsm/xsm.h>
 #include <xen/trace.h>
+#include <xen/vnuma.h>
 
 struct memop_args {
     /* INPUT */
@@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         rcu_unlock_domain(d);
 
         break;
+    case XENMEM_get_vnuma_info:
+    {
+        int i, j;
+        struct vnuma_topology_info mtopology;

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749

Quoted 10 lines
xen/common/memory.c +9 -0
+        struct vnuma_topology_info touser_topo;
+        struct domain *d;
+        unsigned int max_pages, max_vcpus, nr_vnodes;
+        vnuma_memblk_t *vblks;
 
+        rc = -EINVAL;
+        if ( guest_handle_is_null(arg) )
+            return rc;
+        if( copy_from_guest(&mtopology, arg, 1) )
+            return -EINVAL;

-EFAULT;

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 751

Quoted 2 lines
xen/common/memory.c +2 -0
+        if ( (d = rcu_lock_domain_by_any_id(mtopology.domid)) == NULL )
+            return -EINVAL;

-ESRCH ( I think? )

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 762

Quoted 11 lines
xen/common/memory.c +11 -0
+        touser_topo.nr_vnodes = d->vnuma.nr_vnodes;
+        max_pages = d->max_pages;
+        max_vcpus = d->max_vcpus;
+        rcu_unlock_domain(d);
+    
+        nr_vnodes = touser_topo.nr_vnodes;
+        rc = copy_to_guest(arg, &touser_topo, 1);
+        if ( rc )
+            return -EFAULT;
+        if ( nr_vnodes == 0 || nr_vnodes > max_vcpus )
+            return -EFAULT;

-EINVAL;

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765

Quoted 3 lines
xen/common/memory.c +3 -0
+        vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes);
+        if ( vblks == NULL )
+            return -EFAULT;

-ENOMEM;

Now, if you have an unconditional rc = -EFAULT then all your goto out logic will be correct

Quoted 11 lines
xen/common/memory.c +11 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+                if ( copy_to_guest_offset(mtopology.vnuma_memblks, i,
+                           &d->vnuma.vnuma_memblks[i], 1) < 0 )
+                    goto out;
+        }
+        for ( i = 0; i < touser_topo.nr_vnodes; i++ )
+            for ( j = 0; j < touser_topo.nr_vnodes; j++ )
+            {
+                if ( copy_to_guest_offset(mtopology.vdistance,
+                                            __vnode_distance_offset(d, i, j),

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 786

Quoted 10 lines
xen/common/memory.c +10 -0
+                                            &__vnode_distance(d, i, j), 1) )
+                    goto out;
+            }
+        for ( i = 0; i < d->max_vcpus ; i++ )
+        {
+            if ( copy_to_guest_offset(mtopology.vcpu_to_vnode, i,
+                                        &d->vnuma.vcpu_to_vnode[i], 1) )
+                    goto out;
+        }
+        return rc;

But this would need to turn into rc = 0; to avoid return -EFAULT in the success case, and unconditionally leaking vblks

Quoted 55 lines
xen/common/memory.c +5 -0
+out:
+        if ( vblks ) xfree(vblks);
+        return rc;
+        break;
+    }
     default:
         rc = arch_memory_op(op, arg);
         break;
xen/include/public/domctl.h +14 -0
index 4c5b2bb..3574d0a 100644
@@ -35,6 +35,7 @@
 #include "xen.h"
 #include "grant_table.h"
 #include "hvm/save.h"
+#include "memory.h"
 
 #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009
 
@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {
 typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_broken_page_p2m_t);
 
+struct xen_domctl_vnuma {
+    uint16_t nr_vnodes;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(uint) vnode_to_pnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+
+typedef struct xen_domctl_vnuma xen_domctl_vnuma_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_vnuma_t);
+
 struct xen_domctl {
     uint32_t cmd;
 #define XEN_DOMCTL_createdomain                   1
@@ -920,6 +932,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_set_broken_page_p2m           67
 #define XEN_DOMCTL_setnodeaffinity               68
 #define XEN_DOMCTL_getnodeaffinity               69
+#define XEN_DOMCTL_setvnumainfo                  70
 #define XEN_DOMCTL_gdbsx_guestmemio            1000
 #define XEN_DOMCTL_gdbsx_pausevcpu             1001
 #define XEN_DOMCTL_gdbsx_unpausevcpu           1002
@@ -979,6 +992,7 @@ struct xen_domctl {
         struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
         struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
         struct xen_domctl_gdbsx_domstatus   gdbsx_domstatus;
+        struct xen_domctl_vnuma             vnuma;
         uint8_t                             pad[128];
     } u;
 };

xen/include/public/domctl.h typedef struct xen_domctl xen_domctl_t; · line 989

Quoted 4 lines
xen/include/public/domctl.h +0 -1
@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_t);
 
 #endif /* __XEN_PUBLIC_DOMCTL_H__ */
-

Spurious whitespace change. Please remove

xen/include/public/memory.h DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t); · line 456

Quoted 12 lines
xen/include/public/domctl.h +0 -0
 /*
  * Local variables:
  * mode: C
xen/include/public/memory.h +0 -1
index 7a26dee..28f6aaf 100644
@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);
  * Caller must be privileged or the hypercall fails.
  */
 #define XENMEM_claim_pages                  24
-

And here

Quoted 73 lines
xen/include/public/memory.h +8 -0
 /*
  * XENMEM_claim_pages flags - the are no flags at this time.
  * The zero value is appropiate.
  */
 
+struct vnuma_memblk {
+              uint64_t start, end;
+};
+typedef struct vnuma_memblk vnuma_memblk_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t);
+
+#define XENMEM_get_vnuma_info               25
+
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
 #endif /* __XEN_PUBLIC_MEMORY_H__ */
xen/include/xen/domain.h +11 -0
index a057069..c9d53e3 100644
@@ -4,6 +4,7 @@
 
 #include <public/xen.h>
 #include <asm/domain.h>
+#include <public/memory.h>
 
 typedef union {
     struct vcpu_guest_context *nat;
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;
 
 extern bool_t opt_dom0_vcpus_pin;
 
+struct domain_vnuma_info {
+    uint16_t nr_vnodes;
+    uint *vdistance;
+    uint *vcpu_to_vnode;
+    uint *vnode_to_pnode;
+    vnuma_memblk_t *vnuma_memblks;
+};
+
+void domain_vnuma_destroy(struct domain_vnuma_info *v);
+
 #endif /* __XEN_DOMAIN_H__ */
xen/include/xen/sched.h +1 -0
index ae6a3b8..cb023cf 100644
@@ -377,6 +377,7 @@ struct domain
     nodemask_t node_affinity;
     unsigned int last_alloc_node;
     spinlock_t node_affinity_lock;
+    struct domain_vnuma_info vnuma;
 };
 
 struct domain_setup_info
xen/include/xen/vnuma.h new file +12 -0
new file mode 100644
index 0000000..0b41da0
@@ -0,0 +1,27 @@
+#ifndef _VNUMA_H
+#define _VNUMA_H
+#include <public/memory.h>
+
+/* DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t); */
+
+struct vnuma_topology_info {
+    domid_t domid;
+    uint16_t nr_vnodes;
+    uint32_t _pad;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;

xen/include/xen/vnuma.h line 22

Quoted 10 lines
xen/include/xen/vnuma.h new file +10 -0
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+typedef struct vnuma_topology_info vnuma_topology_info_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_topology_info_t);
+
+#define __vnode_distance_offset(_dom, _i, _j) \
+        ( ((_j) * ((_dom)->vnuma.nr_vnodes)) + (_i) )
+
+#define __vnode_distance(_dom, _i, _j) \
+        ( (_dom)->vnuma.vdistance[__vnode_distance_offset((_dom), (_i), (_j))] )

You expand _dom here twice, which is bad practice for macros.

I suggest static inline functions as a better alternative.

~Andrew

Quoted 5 lines
xen/include/xen/vnuma.h new file +5 -0
+
+#define __vnode_distance_set(_dom, _i, _j, _v) \
+        do { __vnode_distance((_dom), (_i), (_j)) = (_v); } while(0)
+
+#endif
#2 lore raw
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Andrew Cooper

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 881

Quoted 3 lines · Elena Ufimtseva
xen/common/domctl.c +3 -0
+        if (
+        (d->vnuma.vdistance = xmalloc_bytes(
+                 sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||

sizeof in an operator not a function. As you are not passing a type, the brackets are not required.

Actually, I'd request parentheses to be added if I saw a use of sizeof() without them. I think we have quite few (hopefully a majority) of uses similar to the one above in the tree.

Jan

#3 lore raw
In reply to Elena Ufimtseva

xen/common/domain.c struct domain *domain_create( · line 230

Quoted 8 lines
Quoted 1 line
Quoted 1 line · Elena Ufimtseva
xen/common/domain.c +1 -0
@@ -227,6 +227,7 @@ struct domain *domain_create(
     spin_lock_init(&d->node_affinity_lock);
     d->node_affinity = NODE_MASK_ALL;
     d->auto_node_affinity = 1;
+    d->vnuma.nr_vnodes = 0;

Pointless (struct domain gets zero-allocated)?

xen/common/domain.c int domain_kill(struct domain *d) · line 535

Quoted 7 lines
xen/common/domain.c +1 -1
@@ -530,8 +531,9 @@ int domain_kill(struct domain *d)
         evtchn_destroy(d);
         gnttab_release_mappings(d);
         tmem_destroy(d->tmem);
-        domain_set_outstanding_pages(d, 0);
         d->tmem = NULL;
+        domain_set_outstanding_pages(d, 0);

Why is this being moved?

Quoted 9 lines
xen/common/domain.c +9 -0
+void domain_vnuma_destroy(struct domain_vnuma_info *v)
+{
+    if (v->vnuma_memblks) {
+        xfree(v->vnuma_memblks);
+        v->vnuma_memblks = NULL;
+    }
+    if (v->vcpu_to_vnode) {
+        xfree(v->vcpu_to_vnode);
+        v->vcpu_to_vnode = NULL;

xen/common/domain.c int continue_hypercall_on_cpu( · line 1302

Quoted 10 lines
xen/common/domain.c +10 -0
+    }
+    if (v->vdistance) {
+        xfree(v->vdistance);
+        v->vdistance = NULL;
+    }
+    if (v->vnode_to_pnode) {
+        xfree(v->vnode_to_pnode);
+        v->vnode_to_pnode = NULL;
+    }
+    v->nr_vnodes = 0;

All the if()s are pointless - xfree() deals with NULL input quite fine. And for the record, they also all don't match our coding style.

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 865

Quoted 5 lines
xen/common/domain.c +0 -1
@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
         ret = set_global_virq_handler(d, virq);
     }
     break;
-

Please keep that newline, and insert another one after the final break of the code you add.

Quoted 13 lines
xen/common/domain.c +13 -0
+    case XEN_DOMCTL_setvnumainfo:
+    {
+        unsigned int i, j, nr_vnodes, dist_size;
+        unsigned int dist, vmap, vntop;
+        vnuma_memblk_t vmemblk;
+        
+        ret = -EINVAL;
+        dist = i = j = 0;
+        nr_vnodes = op->u.vnuma.nr_vnodes; 
+        if (nr_vnodes <= 0 || nr_vnodes > NR_CPUS)
+            break;
+        d->vnuma.nr_vnodes = nr_vnodes;
+        dist_size = nr_vnodes * nr_vnodes;

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888

Quoted 10 lines
xen/common/domain.c +10 -0
+        if (
+        (d->vnuma.vdistance = xmalloc_bytes(
+                 sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||
+        (d->vnuma.vnuma_memblks = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnuma_memblks) * nr_vnodes)) == NULL ||
+        (d->vnuma.vcpu_to_vnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) == NULL ||
+        (d->vnuma.vnode_to_pnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnode_to_pnode) * nr_vnodes)) == NULL )
+            goto err_dom;

Coding style. Also, you leak all memory that you managed to get hold of in the error case.

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 892

Quoted 4 lines
xen/common/domain.c +4 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+            for ( j = 0; j < nr_vnodes; j++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&dist,

__copy_from_guest_offset() without prior guest_handle_okay()? And the error code should be -EFAULT in that case.

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 898

Quoted 6 lines
xen/common/domain.c +6 -0
+                                        op->u.vnuma.vdistance,
+                                        __vnode_distance_offset(d, i, j), 1)) )
+                    goto err_dom;
+                __vnode_distance_set(d, i, j, dist);
+            }
+        for ( i = 0; i < nr_vnodes; i++ )

If possible please fold this with the earlier loop.

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 911

Quoted 13 lines
xen/common/domain.c +13 -0
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmemblk,
+                                    op->u.vnuma.vnuma_memblks, i, 1)) )
+                goto err_dom;
+            d->vnuma.vnuma_memblks[i].start = vmemblk.start;
+            d->vnuma.vnuma_memblks[i].end = vmemblk.end;
+        }
+        for ( i = 0; i < d->max_vcpus; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmap,
+                           op->u.vnuma.vcpu_to_vnode, i, 1)) )
+                goto err_dom;
+            d->vnuma.vcpu_to_vnode[i] = vmap;

If the types are compatible, things like this don't need doing in a loop - you could copy the whole array in one go.

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 924

Quoted 13 lines
xen/common/domain.c +13 -0
+        }
+        if ( !guest_handle_is_null(op->u.vnuma.vnode_to_pnode) )
+        {
+            for ( i = 0; i < nr_vnodes; i++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&vntop,
+                                        op->u.vnuma.vnode_to_pnode, i, 1)) )
+                    goto err_dom;
+                d->vnuma.vnode_to_pnode[i] = vntop;
+            }
+        }
+        else
+            for(i = 0; i < nr_vnodes; i++)

Coding style.

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925

Quoted 1 line
xen/common/domain.c +1 -0
+                d->vnuma.vnode_to_pnode[i] = NUMA_NO_NODE;

What's the value of using this domctl this way?

xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 929

Quoted 4 lines
xen/common/domain.c +4 -0
+        ret = 0;
+        break;
+err_dom:
+        ret = -EINVAL;

"ret" should not be unconditionally set to a fixed value here, the more that you set it almost first thing in the case statement.

Quoted 8 lines
xen/common/domain.c +4 -0
@@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         rcu_unlock_domain(d);
 
         break;
+    case XENMEM_get_vnuma_info:
+    {
+        int i, j;
+        struct vnuma_topology_info mtopology;

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749

Quoted 10 lines
xen/common/domain.c +9 -0
+        struct vnuma_topology_info touser_topo;
+        struct domain *d;
+        unsigned int max_pages, max_vcpus, nr_vnodes;
+        vnuma_memblk_t *vblks;
 
+        rc = -EINVAL;
+        if ( guest_handle_is_null(arg) )
+            return rc;
+        if( copy_from_guest(&mtopology, arg, 1) )
+            return -EINVAL;

Consistency please, preferably such that you - as long as possible - always return -E rather than rc.

Also the return value here again is -EFAULT.

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 751

Quoted 2 lines
xen/common/domain.c +2 -0
+        if ( (d = rcu_lock_domain_by_any_id(mtopology.domid)) == NULL )
+            return -EINVAL;

And -ESRCH or some such here (please use existing code as reference).

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 762

Quoted 11 lines
xen/common/domain.c +11 -0
+        touser_topo.nr_vnodes = d->vnuma.nr_vnodes;
+        max_pages = d->max_pages;
+        max_vcpus = d->max_vcpus;
+        rcu_unlock_domain(d);
+    
+        nr_vnodes = touser_topo.nr_vnodes;
+        rc = copy_to_guest(arg, &touser_topo, 1);
+        if ( rc )
+            return -EFAULT;
+        if ( nr_vnodes == 0 || nr_vnodes > max_vcpus )
+            return -EFAULT;

-EINVAL

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765

Quoted 3 lines
xen/common/domain.c +3 -0
+        vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes);
+        if ( vblks == NULL )
+            return -EFAULT;

-ENOMEM

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 769

Quoted 4 lines
xen/common/domain.c +4 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+                if ( copy_to_guest_offset(mtopology.vnuma_memblks, i,
+                           &d->vnuma.vnuma_memblks[i], 1) < 0 )

Indentation.

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 772

Quoted 3 lines
xen/common/domain.c +3 -0
+                    goto out;
+        }
+        for ( i = 0; i < touser_topo.nr_vnodes; i++ )

Again, please fold this with the previous loop, and use (preferably) the simple variable as loop boundary (but in any case be consistent).

Quoted 6 lines
xen/common/domain.c +6 -0
+            for ( j = 0; j < touser_topo.nr_vnodes; j++ )
+            {
+                if ( copy_to_guest_offset(mtopology.vdistance,
+                                            __vnode_distance_offset(d, i, j),
+                                            &__vnode_distance(d, i, j), 1) )
+                    goto out;

xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 788

Quoted 10 lines
xen/common/domain.c +10 -0
+            }
+        for ( i = 0; i < d->max_vcpus ; i++ )
+        {
+            if ( copy_to_guest_offset(mtopology.vcpu_to_vnode, i,
+                                        &d->vnuma.vcpu_to_vnode[i], 1) )
+                    goto out;
+        }
+        return rc;
+out:
+        if ( vblks ) xfree(vblks);

Coding style.

Quoted 9 lines
xen/include/xen/vnuma.h new file +6 -0
@@ -0,0 +1,27 @@
+#ifndef _VNUMA_H
+#define _VNUMA_H
+#include <public/memory.h>
+
+/* DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t); */
+

xen/include/xen/vnuma.h line 16

Quoted 10 lines
xen/include/xen/vnuma.h new file +10 -0
+struct vnuma_topology_info {
+    domid_t domid;
+    uint16_t nr_vnodes;
+    uint32_t _pad;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+typedef struct vnuma_topology_info vnuma_topology_info_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_topology_info_t);

At least up to here this appears to belong into the public header.

Jan

Quoted 14 lines
xen/include/xen/vnuma.h new file +6 -0
+
+#define __vnode_distance_offset(_dom, _i, _j) \
+        ( ((_j) * ((_dom)->vnuma.nr_vnodes)) + (_i) )
+
+#define __vnode_distance(_dom, _i, _j) \
+        ( (_dom)->vnuma.vdistance[__vnode_distance_offset((_dom), (_i), 

(_j))] )

xen/include/xen/vnuma.h new file +5 -0
+
+#define __vnode_distance_set(_dom, _i, _j, _v) \
+        do { __vnode_distance((_dom), (_i), (_j)) = (_v); } while(0)
+
+#endif
Signature · 1 line
1.7.10.4
#4 lore raw
In reply to Elena Ufimtseva
Quoted 1 line · Elena Ufimtseva

+void domain_vnuma_destroy(struct domain_vnuma_info *v)

As long as you're respinning, I think I'd prefer this be named something different -- 'v' is almost universally of type "struct vcpu" in the Xen code, so this code is a little confusing. I might use "vinfo", or even just take a domain pointer and dereference it yourself.

Quoted 38 lines
xen/common/domain.c +20 -0
+{
+    if (v->vnuma_memblks) {
+        xfree(v->vnuma_memblks);
+        v->vnuma_memblks = NULL;
+    }
+    if (v->vcpu_to_vnode) {
+        xfree(v->vcpu_to_vnode);
+        v->vcpu_to_vnode = NULL;
+    }
+    if (v->vdistance) {
+        xfree(v->vdistance);
+        v->vdistance = NULL;
+    }
+    if (v->vnode_to_pnode) {
+        xfree(v->vnode_to_pnode);
+        v->vnode_to_pnode = NULL;
+    }
+    v->nr_vnodes = 0;
+}
+
 /*
  * Local variables:
  * mode: C
xen/common/domctl.c +1 -0
index 9760d50..e5d05c7 100644
@@ -29,6 +29,7 @@
 #include <asm/page.h>
 #include <public/domctl.h>
 #include <xsm/xsm.h>
+#include <xen/vnuma.h>
 
 static DEFINE_SPINLOCK(domctl_lock);
 DEFINE_SPINLOCK(vcpu_alloc_lock);
@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
         ret = set_global_virq_handler(d, virq);
     }

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 873

Quoted 10 lines
xen/common/domctl.c +8 -1
     break;
-
+    case XEN_DOMCTL_setvnumainfo:
+    {
+        unsigned int i, j, nr_vnodes, dist_size;
+        unsigned int dist, vmap, vntop;
+        vnuma_memblk_t vmemblk;
+
+        ret = -EINVAL;
+        dist = i = j = 0;

I think the zeroing of these is unnecessary -- you zero them / initialize them before you use them below.

You need to break this code into "paragraphs" of related changes, with a space in between. For example, maybe here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876

Quoted 3 lines
xen/common/domctl.c +3 -0
+        nr_vnodes = op->u.vnuma.nr_vnodes;
+        if (nr_vnodes <= 0 || nr_vnodes > NR_CPUS)
+            break;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 878

Quoted 2 lines
xen/common/domctl.c +2 -0
+        d->vnuma.nr_vnodes = nr_vnodes;
+        dist_size = nr_vnodes * nr_vnodes;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888

Quoted 10 lines
xen/common/domctl.c +10 -0
+        if (
+        (d->vnuma.vdistance = xmalloc_bytes(
+                 sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||
+        (d->vnuma.vnuma_memblks = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnuma_memblks) * nr_vnodes)) == NULL ||
+        (d->vnuma.vcpu_to_vnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) == NULL ||
+        (d->vnuma.vnode_to_pnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnode_to_pnode) * nr_vnodes)) == NULL )
+            goto err_dom;

here...

Also, for an idiomatic way to do this kind of multiple allocation w/ error checking, take a look at xen/arch/x86/hvm/hvm.c:hvm_domain_initialise() (to pick a random function I've worked with recently). Look at the

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 897

Quoted 9 lines
xen/common/domctl.c +9 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+            for ( j = 0; j < nr_vnodes; j++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&dist,
+                                        op->u.vnuma.vdistance,
+                                        __vnode_distance_offset(d, i, j), 1)) )
+                    goto err_dom;
+                __vnode_distance_set(d, i, j, dist);
+            }

here...

Also, in each of these error paths, you leak the structures allocated above.

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 905

Quoted 8 lines
xen/common/domctl.c +8 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmemblk,
+                                    op->u.vnuma.vnuma_memblks, i, 1)) )
+                goto err_dom;
+            d->vnuma.vnuma_memblks[i].start = vmemblk.start;
+            d->vnuma.vnuma_memblks[i].end = vmemblk.end;
+        }

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 912

Quoted 7 lines
xen/common/domctl.c +7 -0
+        for ( i = 0; i < d->max_vcpus; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmap,
+                           op->u.vnuma.vcpu_to_vnode, i, 1)) )
+                goto err_dom;
+            d->vnuma.vcpu_to_vnode[i] = vmap;
+        }

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925

Quoted 13 lines
xen/common/domctl.c +13 -0
+        if ( !guest_handle_is_null(op->u.vnuma.vnode_to_pnode) )
+        {
+            for ( i = 0; i < nr_vnodes; i++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&vntop,
+                                        op->u.vnuma.vnode_to_pnode, i, 1)) )
+                    goto err_dom;
+                d->vnuma.vnode_to_pnode[i] = vntop;
+            }
+        }
+        else
+            for(i = 0; i < nr_vnodes; i++)
+                d->vnuma.vnode_to_pnode[i] = NUMA_NO_NODE;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 926

Quoted 1 line
xen/common/domctl.c +1 -0
+        ret = 0;

and maybe here.

Quoted 24 lines
xen/common/domctl.c +5 -0
+        break;
+err_dom:
+        ret = -EINVAL;
+    }
+    break;
     default:
         ret = arch_do_domctl(op, d, u_domctl);
         break;
xen/common/memory.c +1 -0
index 50b740f..6dc2452 100644
@@ -28,6 +28,7 @@
 #include <public/memory.h>
 #include <xsm/xsm.h>
 #include <xen/trace.h>
+#include <xen/vnuma.h>
 
 struct memop_args {
     /* INPUT */
@@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         rcu_unlock_domain(d);
 
         break;

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 745

Quoted 10 lines
xen/common/memory.c +9 -0
+    case XENMEM_get_vnuma_info:
+    {
+        int i, j;
+        struct vnuma_topology_info mtopology;
+        struct vnuma_topology_info touser_topo;
+        struct domain *d;
+        unsigned int max_pages, max_vcpus, nr_vnodes;
+        vnuma_memblk_t *vblks;
 
+        rc = -EINVAL;

here...

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 747

Quoted 2 lines
xen/common/memory.c +2 -0
+        if ( guest_handle_is_null(arg) )
+            return rc;

here...

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749

Quoted 2 lines
xen/common/memory.c +2 -0
+        if( copy_from_guest(&mtopology, arg, 1) )
+            return -EINVAL;

here... (and so on)

This should be -EFAULT

Quoted 6 lines
xen/common/memory.c +6 -0
+        if ( (d = rcu_lock_domain_by_any_id(mtopology.domid)) == NULL )
+            return -EINVAL;
+        touser_topo.nr_vnodes = d->vnuma.nr_vnodes;
+        max_pages = d->max_pages;
+        max_vcpus = d->max_vcpus;
+        rcu_unlock_domain(d);

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765

Quoted 10 lines
xen/common/memory.c +10 -0
+
+        nr_vnodes = touser_topo.nr_vnodes;
+        rc = copy_to_guest(arg, &touser_topo, 1);
+        if ( rc )
+            return -EFAULT;
+        if ( nr_vnodes == 0 || nr_vnodes > max_vcpus )
+            return -EFAULT;
+        vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes);
+        if ( vblks == NULL )
+            return -EFAULT;

Uum, what is this for? It doesn't appear to be used -- if there's a copy failure it's freed again, if things succeed it's leaked.

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 770

Quoted 5 lines
xen/common/memory.c +5 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+                if ( copy_to_guest_offset(mtopology.vnuma_memblks, i,
+                           &d->vnuma.vnuma_memblks[i], 1) < 0 )
+                    goto out;

4 spaces for indentation, please.

Quoted 116 lines
xen/common/memory.c +21 -0
+        }
+        for ( i = 0; i < touser_topo.nr_vnodes; i++ )
+            for ( j = 0; j < touser_topo.nr_vnodes; j++ )
+            {
+                if ( copy_to_guest_offset(mtopology.vdistance,
+                                            __vnode_distance_offset(d, i, j),
+                                            &__vnode_distance(d, i, j), 1) )
+                    goto out;
+            }
+        for ( i = 0; i < d->max_vcpus ; i++ )
+        {
+            if ( copy_to_guest_offset(mtopology.vcpu_to_vnode, i,
+                                        &d->vnuma.vcpu_to_vnode[i], 1) )
+                    goto out;
+        }
+        return rc;
+out:
+        if ( vblks ) xfree(vblks);
+        return rc;
+        break;
+    }
     default:
         rc = arch_memory_op(op, arg);
         break;
xen/include/public/domctl.h +14 -1
index 4c5b2bb..3574d0a 100644
@@ -35,6 +35,7 @@
 #include "xen.h"
 #include "grant_table.h"
 #include "hvm/save.h"
+#include "memory.h"
 
 #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009
 
@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {
 typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_broken_page_p2m_t);
 
+struct xen_domctl_vnuma {
+    uint16_t nr_vnodes;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(uint) vnode_to_pnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+
+typedef struct xen_domctl_vnuma xen_domctl_vnuma_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_vnuma_t);
+
 struct xen_domctl {
     uint32_t cmd;
 #define XEN_DOMCTL_createdomain                   1
@@ -920,6 +932,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_set_broken_page_p2m           67
 #define XEN_DOMCTL_setnodeaffinity               68
 #define XEN_DOMCTL_getnodeaffinity               69
+#define XEN_DOMCTL_setvnumainfo                  70
 #define XEN_DOMCTL_gdbsx_guestmemio            1000
 #define XEN_DOMCTL_gdbsx_pausevcpu             1001
 #define XEN_DOMCTL_gdbsx_unpausevcpu           1002
@@ -979,6 +992,7 @@ struct xen_domctl {
         struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
         struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
         struct xen_domctl_gdbsx_domstatus   gdbsx_domstatus;
+        struct xen_domctl_vnuma             vnuma;
         uint8_t                             pad[128];
     } u;
 };
@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_t);
 
 #endif /* __XEN_PUBLIC_DOMCTL_H__ */
-
 /*
  * Local variables:
  * mode: C
xen/include/public/memory.h +8 -1
index 7a26dee..28f6aaf 100644
@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);
  * Caller must be privileged or the hypercall fails.
  */
 #define XENMEM_claim_pages                  24
-
 /*
  * XENMEM_claim_pages flags - the are no flags at this time.
  * The zero value is appropiate.
  */
 
+struct vnuma_memblk {
+              uint64_t start, end;
+};
+typedef struct vnuma_memblk vnuma_memblk_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t);
+
+#define XENMEM_get_vnuma_info               25
+
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
 #endif /* __XEN_PUBLIC_MEMORY_H__ */
xen/include/xen/domain.h +1 -0
index a057069..c9d53e3 100644
@@ -4,6 +4,7 @@
 
 #include <public/xen.h>
 #include <asm/domain.h>
+#include <public/memory.h>
 
 typedef union {
     struct vcpu_guest_context *nat;

xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99

Quoted 10 lines
xen/include/xen/domain.h +7 -0
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;
 
 extern bool_t opt_dom0_vcpus_pin;
 
+struct domain_vnuma_info {
+    uint16_t nr_vnodes;
+    uint *vdistance;
+    uint *vcpu_to_vnode;
+    uint *vnode_to_pnode;
+    vnuma_memblk_t *vnuma_memblks;
+};

Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?

-George

#5 lore raw

Hi guys
Thank you for reviewing, I will clean these out.

George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.

Elena

Quoted 356 lines · George Dunlap
Quoted 1 line · Elena Ufimtseva

+void domain_vnuma_destroy(struct domain_vnuma_info *v)

As long as you're respinning, I think I'd prefer this be named something different -- 'v' is almost universally of type "struct vcpu" in the Xen code, so this code is a little confusing. I might use "vinfo", or even just take a domain pointer and dereference it yourself.

Quoted 38 lines
xen/common/domain.c +20 -0
+{
+    if (v->vnuma_memblks) {
+        xfree(v->vnuma_memblks);
+        v->vnuma_memblks = NULL;
+    }
+    if (v->vcpu_to_vnode) {
+        xfree(v->vcpu_to_vnode);
+        v->vcpu_to_vnode = NULL;
+    }
+    if (v->vdistance) {
+        xfree(v->vdistance);
+        v->vdistance = NULL;
+    }
+    if (v->vnode_to_pnode) {
+        xfree(v->vnode_to_pnode);
+        v->vnode_to_pnode = NULL;
+    }
+    v->nr_vnodes = 0;
+}
+
 /*
  * Local variables:
  * mode: C
xen/common/domctl.c +1 -0
index 9760d50..e5d05c7 100644
@@ -29,6 +29,7 @@
 #include <asm/page.h>
 #include <public/domctl.h>
 #include <xsm/xsm.h>
+#include <xen/vnuma.h>
 
 static DEFINE_SPINLOCK(domctl_lock);
 DEFINE_SPINLOCK(vcpu_alloc_lock);
@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
         ret = set_global_virq_handler(d, virq);
     }

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 873

Quoted 10 lines
xen/common/domctl.c +8 -1
     break;
-
+    case XEN_DOMCTL_setvnumainfo:
+    {
+        unsigned int i, j, nr_vnodes, dist_size;
+        unsigned int dist, vmap, vntop;
+        vnuma_memblk_t vmemblk;
+
+        ret = -EINVAL;
+        dist = i = j = 0;

I think the zeroing of these is unnecessary -- you zero them / initialize them before you use them below.

You need to break this code into "paragraphs" of related changes, with a space in between. For example, maybe here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876

Quoted 3 lines
xen/common/domctl.c +3 -0
+        nr_vnodes = op->u.vnuma.nr_vnodes;
+        if (nr_vnodes <= 0 || nr_vnodes > NR_CPUS)
+            break;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 878

Quoted 2 lines
xen/common/domctl.c +2 -0
+        d->vnuma.nr_vnodes = nr_vnodes;
+        dist_size = nr_vnodes * nr_vnodes;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888

Quoted 10 lines
xen/common/domctl.c +10 -0
+        if (
+        (d->vnuma.vdistance = xmalloc_bytes(
+                 sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||
+        (d->vnuma.vnuma_memblks = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnuma_memblks) * nr_vnodes)) == NULL ||
+        (d->vnuma.vcpu_to_vnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) == NULL ||
+        (d->vnuma.vnode_to_pnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnode_to_pnode) * nr_vnodes)) == NULL )
+            goto err_dom;

here...

Also, for an idiomatic way to do this kind of multiple allocation w/ error checking, take a look at xen/arch/x86/hvm/hvm.c:hvm_domain_initialise() (to pick a random function I've worked with recently). Look at the

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 897

Quoted 9 lines
xen/common/domctl.c +9 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+            for ( j = 0; j < nr_vnodes; j++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&dist,
+                                        op->u.vnuma.vdistance,
+                                        __vnode_distance_offset(d, i, j), 1)) )
+                    goto err_dom;
+                __vnode_distance_set(d, i, j, dist);
+            }

here...

Also, in each of these error paths, you leak the structures allocated above.

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 905

Quoted 8 lines
xen/common/domctl.c +8 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmemblk,
+                                    op->u.vnuma.vnuma_memblks, i, 1)) )
+                goto err_dom;
+            d->vnuma.vnuma_memblks[i].start = vmemblk.start;
+            d->vnuma.vnuma_memblks[i].end = vmemblk.end;
+        }

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 912

Quoted 7 lines
xen/common/domctl.c +7 -0
+        for ( i = 0; i < d->max_vcpus; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmap,
+                           op->u.vnuma.vcpu_to_vnode, i, 1)) )
+                goto err_dom;
+            d->vnuma.vcpu_to_vnode[i] = vmap;
+        }

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925

Quoted 13 lines
xen/common/domctl.c +13 -0
+        if ( !guest_handle_is_null(op->u.vnuma.vnode_to_pnode) )
+        {
+            for ( i = 0; i < nr_vnodes; i++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&vntop,
+                                        op->u.vnuma.vnode_to_pnode, i, 1)) )
+                    goto err_dom;
+                d->vnuma.vnode_to_pnode[i] = vntop;
+            }
+        }
+        else
+            for(i = 0; i < nr_vnodes; i++)
+                d->vnuma.vnode_to_pnode[i] = NUMA_NO_NODE;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 926

Quoted 1 line
xen/common/domctl.c +1 -0
+        ret = 0;

and maybe here.

Quoted 24 lines
xen/common/domctl.c +5 -0
+        break;
+err_dom:
+        ret = -EINVAL;
+    }
+    break;
     default:
         ret = arch_do_domctl(op, d, u_domctl);
         break;
xen/common/memory.c +1 -0
index 50b740f..6dc2452 100644
@@ -28,6 +28,7 @@
 #include <public/memory.h>
 #include <xsm/xsm.h>
 #include <xen/trace.h>
+#include <xen/vnuma.h>
 
 struct memop_args {
     /* INPUT */
@@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         rcu_unlock_domain(d);
 
         break;

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 745

Quoted 10 lines
xen/common/memory.c +9 -0
+    case XENMEM_get_vnuma_info:
+    {
+        int i, j;
+        struct vnuma_topology_info mtopology;
+        struct vnuma_topology_info touser_topo;
+        struct domain *d;
+        unsigned int max_pages, max_vcpus, nr_vnodes;
+        vnuma_memblk_t *vblks;
 
+        rc = -EINVAL;

here...

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 747

Quoted 2 lines
xen/common/memory.c +2 -0
+        if ( guest_handle_is_null(arg) )
+            return rc;

here...

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749

Quoted 2 lines
xen/common/memory.c +2 -0
+        if( copy_from_guest(&mtopology, arg, 1) )
+            return -EINVAL;

here... (and so on)

This should be -EFAULT

Quoted 6 lines
xen/common/memory.c +6 -0
+        if ( (d = rcu_lock_domain_by_any_id(mtopology.domid)) == NULL )
+            return -EINVAL;
+        touser_topo.nr_vnodes = d->vnuma.nr_vnodes;
+        max_pages = d->max_pages;
+        max_vcpus = d->max_vcpus;
+        rcu_unlock_domain(d);

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765

Quoted 10 lines
xen/common/memory.c +10 -0
+
+        nr_vnodes = touser_topo.nr_vnodes;
+        rc = copy_to_guest(arg, &touser_topo, 1);
+        if ( rc )
+            return -EFAULT;
+        if ( nr_vnodes == 0 || nr_vnodes > max_vcpus )
+            return -EFAULT;
+        vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes);
+        if ( vblks == NULL )
+            return -EFAULT;

Uum, what is this for? It doesn't appear to be used -- if there's a copy failure it's freed again, if things succeed it's leaked.

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 770

Quoted 5 lines
xen/common/memory.c +5 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+                if ( copy_to_guest_offset(mtopology.vnuma_memblks, i,
+                           &d->vnuma.vnuma_memblks[i], 1) < 0 )
+                    goto out;

4 spaces for indentation, please.

Quoted 116 lines
xen/common/memory.c +21 -0
+        }
+        for ( i = 0; i < touser_topo.nr_vnodes; i++ )
+            for ( j = 0; j < touser_topo.nr_vnodes; j++ )
+            {
+                if ( copy_to_guest_offset(mtopology.vdistance,
+                                            __vnode_distance_offset(d, i, j),
+                                            &__vnode_distance(d, i, j), 1) )
+                    goto out;
+            }
+        for ( i = 0; i < d->max_vcpus ; i++ )
+        {
+            if ( copy_to_guest_offset(mtopology.vcpu_to_vnode, i,
+                                        &d->vnuma.vcpu_to_vnode[i], 1) )
+                    goto out;
+        }
+        return rc;
+out:
+        if ( vblks ) xfree(vblks);
+        return rc;
+        break;
+    }
     default:
         rc = arch_memory_op(op, arg);
         break;
xen/include/public/domctl.h +14 -1
index 4c5b2bb..3574d0a 100644
@@ -35,6 +35,7 @@
 #include "xen.h"
 #include "grant_table.h"
 #include "hvm/save.h"
+#include "memory.h"
 
 #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009
 
@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {
 typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_broken_page_p2m_t);
 
+struct xen_domctl_vnuma {
+    uint16_t nr_vnodes;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(uint) vnode_to_pnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+
+typedef struct xen_domctl_vnuma xen_domctl_vnuma_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_vnuma_t);
+
 struct xen_domctl {
     uint32_t cmd;
 #define XEN_DOMCTL_createdomain                   1
@@ -920,6 +932,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_set_broken_page_p2m           67
 #define XEN_DOMCTL_setnodeaffinity               68
 #define XEN_DOMCTL_getnodeaffinity               69
+#define XEN_DOMCTL_setvnumainfo                  70
 #define XEN_DOMCTL_gdbsx_guestmemio            1000
 #define XEN_DOMCTL_gdbsx_pausevcpu             1001
 #define XEN_DOMCTL_gdbsx_unpausevcpu           1002
@@ -979,6 +992,7 @@ struct xen_domctl {
         struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
         struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
         struct xen_domctl_gdbsx_domstatus   gdbsx_domstatus;
+        struct xen_domctl_vnuma             vnuma;
         uint8_t                             pad[128];
     } u;
 };
@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_t);
 
 #endif /* __XEN_PUBLIC_DOMCTL_H__ */
-
 /*
  * Local variables:
  * mode: C
xen/include/public/memory.h +8 -1
index 7a26dee..28f6aaf 100644
@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);
  * Caller must be privileged or the hypercall fails.
  */
 #define XENMEM_claim_pages                  24
-
 /*
  * XENMEM_claim_pages flags - the are no flags at this time.
  * The zero value is appropiate.
  */
 
+struct vnuma_memblk {
+              uint64_t start, end;
+};
+typedef struct vnuma_memblk vnuma_memblk_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t);
+
+#define XENMEM_get_vnuma_info               25
+
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
 #endif /* __XEN_PUBLIC_MEMORY_H__ */
xen/include/xen/domain.h +1 -0
index a057069..c9d53e3 100644
@@ -4,6 +4,7 @@
 
 #include <public/xen.h>
 #include <asm/domain.h>
+#include <public/memory.h>
 
 typedef union {
     struct vcpu_guest_context *nat;

xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99

Quoted 10 lines
xen/include/xen/domain.h +7 -0
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;
 
 extern bool_t opt_dom0_vcpus_pin;
 
+struct domain_vnuma_info {
+    uint16_t nr_vnodes;
+    uint *vdistance;
+    uint *vcpu_to_vnode;
+    uint *vnode_to_pnode;
+    vnuma_memblk_t *vnuma_memblks;
+};

Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?

-George

Signature · 1 line
Elena
#6 lore raw

Looks like I have.. Linux type for number of nodes is int, so Id rather keep it this way.

Quoted 373 lines · Elena Ufimtseva

Hi guys
Thank you for reviewing, I will clean these out.

George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.

Elena

Quoted 355 lines · George Dunlap
Quoted 1 line · Elena Ufimtseva

+void domain_vnuma_destroy(struct domain_vnuma_info *v)

As long as you're respinning, I think I'd prefer this be named something different -- 'v' is almost universally of type "struct vcpu" in the Xen code, so this code is a little confusing. I might use "vinfo", or even just take a domain pointer and dereference it yourself.

Quoted 38 lines
xen/common/domain.c +20 -0
+{
+    if (v->vnuma_memblks) {
+        xfree(v->vnuma_memblks);
+        v->vnuma_memblks = NULL;
+    }
+    if (v->vcpu_to_vnode) {
+        xfree(v->vcpu_to_vnode);
+        v->vcpu_to_vnode = NULL;
+    }
+    if (v->vdistance) {
+        xfree(v->vdistance);
+        v->vdistance = NULL;
+    }
+    if (v->vnode_to_pnode) {
+        xfree(v->vnode_to_pnode);
+        v->vnode_to_pnode = NULL;
+    }
+    v->nr_vnodes = 0;
+}
+
 /*
  * Local variables:
  * mode: C
xen/common/domctl.c +1 -0
index 9760d50..e5d05c7 100644
@@ -29,6 +29,7 @@
 #include <asm/page.h>
 #include <public/domctl.h>
 #include <xsm/xsm.h>
+#include <xen/vnuma.h>
 
 static DEFINE_SPINLOCK(domctl_lock);
 DEFINE_SPINLOCK(vcpu_alloc_lock);
@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
         ret = set_global_virq_handler(d, virq);
     }

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 873

Quoted 10 lines
xen/common/domctl.c +8 -1
     break;
-
+    case XEN_DOMCTL_setvnumainfo:
+    {
+        unsigned int i, j, nr_vnodes, dist_size;
+        unsigned int dist, vmap, vntop;
+        vnuma_memblk_t vmemblk;
+
+        ret = -EINVAL;
+        dist = i = j = 0;

I think the zeroing of these is unnecessary -- you zero them / initialize them before you use them below.

You need to break this code into "paragraphs" of related changes, with a space in between. For example, maybe here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876

Quoted 3 lines
xen/common/domctl.c +3 -0
+        nr_vnodes = op->u.vnuma.nr_vnodes;
+        if (nr_vnodes <= 0 || nr_vnodes > NR_CPUS)
+            break;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 878

Quoted 2 lines
xen/common/domctl.c +2 -0
+        d->vnuma.nr_vnodes = nr_vnodes;
+        dist_size = nr_vnodes * nr_vnodes;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888

Quoted 10 lines
xen/common/domctl.c +10 -0
+        if (
+        (d->vnuma.vdistance = xmalloc_bytes(
+                 sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||
+        (d->vnuma.vnuma_memblks = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnuma_memblks) * nr_vnodes)) == NULL ||
+        (d->vnuma.vcpu_to_vnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) == NULL ||
+        (d->vnuma.vnode_to_pnode = xmalloc_bytes(
+                 sizeof(*d->vnuma.vnode_to_pnode) * nr_vnodes)) == NULL )
+            goto err_dom;

here...

Also, for an idiomatic way to do this kind of multiple allocation w/ error checking, take a look at xen/arch/x86/hvm/hvm.c:hvm_domain_initialise() (to pick a random function I've worked with recently). Look at the

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 897

Quoted 9 lines
xen/common/domctl.c +9 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+            for ( j = 0; j < nr_vnodes; j++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&dist,
+                                        op->u.vnuma.vdistance,
+                                        __vnode_distance_offset(d, i, j), 1)) )
+                    goto err_dom;
+                __vnode_distance_set(d, i, j, dist);
+            }

here...

Also, in each of these error paths, you leak the structures allocated above.

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 905

Quoted 8 lines
xen/common/domctl.c +8 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmemblk,
+                                    op->u.vnuma.vnuma_memblks, i, 1)) )
+                goto err_dom;
+            d->vnuma.vnuma_memblks[i].start = vmemblk.start;
+            d->vnuma.vnuma_memblks[i].end = vmemblk.end;
+        }

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 912

Quoted 7 lines
xen/common/domctl.c +7 -0
+        for ( i = 0; i < d->max_vcpus; i++ )
+        {
+            if ( unlikely(__copy_from_guest_offset(&vmap,
+                           op->u.vnuma.vcpu_to_vnode, i, 1)) )
+                goto err_dom;
+            d->vnuma.vcpu_to_vnode[i] = vmap;
+        }

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925

Quoted 13 lines
xen/common/domctl.c +13 -0
+        if ( !guest_handle_is_null(op->u.vnuma.vnode_to_pnode) )
+        {
+            for ( i = 0; i < nr_vnodes; i++ )
+            {
+                if ( unlikely(__copy_from_guest_offset(&vntop,
+                                        op->u.vnuma.vnode_to_pnode, i, 1)) )
+                    goto err_dom;
+                d->vnuma.vnode_to_pnode[i] = vntop;
+            }
+        }
+        else
+            for(i = 0; i < nr_vnodes; i++)
+                d->vnuma.vnode_to_pnode[i] = NUMA_NO_NODE;

here...

xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 926

Quoted 1 line
xen/common/domctl.c +1 -0
+        ret = 0;

and maybe here.

Quoted 24 lines
xen/common/domctl.c +5 -0
+        break;
+err_dom:
+        ret = -EINVAL;
+    }
+    break;
     default:
         ret = arch_do_domctl(op, d, u_domctl);
         break;
xen/common/memory.c +1 -0
index 50b740f..6dc2452 100644
@@ -28,6 +28,7 @@
 #include <public/memory.h>
 #include <xsm/xsm.h>
 #include <xen/trace.h>
+#include <xen/vnuma.h>
 
 struct memop_args {
     /* INPUT */
@@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
         rcu_unlock_domain(d);
 
         break;

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 745

Quoted 10 lines
xen/common/memory.c +9 -0
+    case XENMEM_get_vnuma_info:
+    {
+        int i, j;
+        struct vnuma_topology_info mtopology;
+        struct vnuma_topology_info touser_topo;
+        struct domain *d;
+        unsigned int max_pages, max_vcpus, nr_vnodes;
+        vnuma_memblk_t *vblks;
 
+        rc = -EINVAL;

here...

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 747

Quoted 2 lines
xen/common/memory.c +2 -0
+        if ( guest_handle_is_null(arg) )
+            return rc;

here...

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749

Quoted 2 lines
xen/common/memory.c +2 -0
+        if( copy_from_guest(&mtopology, arg, 1) )
+            return -EINVAL;

here... (and so on)

This should be -EFAULT

Quoted 6 lines
xen/common/memory.c +6 -0
+        if ( (d = rcu_lock_domain_by_any_id(mtopology.domid)) == NULL )
+            return -EINVAL;
+        touser_topo.nr_vnodes = d->vnuma.nr_vnodes;
+        max_pages = d->max_pages;
+        max_vcpus = d->max_vcpus;
+        rcu_unlock_domain(d);

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765

Quoted 10 lines
xen/common/memory.c +10 -0
+
+        nr_vnodes = touser_topo.nr_vnodes;
+        rc = copy_to_guest(arg, &touser_topo, 1);
+        if ( rc )
+            return -EFAULT;
+        if ( nr_vnodes == 0 || nr_vnodes > max_vcpus )
+            return -EFAULT;
+        vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes);
+        if ( vblks == NULL )
+            return -EFAULT;

Uum, what is this for? It doesn't appear to be used -- if there's a copy failure it's freed again, if things succeed it's leaked.

xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 770

Quoted 5 lines
xen/common/memory.c +5 -0
+        for ( i = 0; i < nr_vnodes; i++ )
+        {
+                if ( copy_to_guest_offset(mtopology.vnuma_memblks, i,
+                           &d->vnuma.vnuma_memblks[i], 1) < 0 )
+                    goto out;

4 spaces for indentation, please.

Quoted 116 lines
xen/common/memory.c +21 -0
+        }
+        for ( i = 0; i < touser_topo.nr_vnodes; i++ )
+            for ( j = 0; j < touser_topo.nr_vnodes; j++ )
+            {
+                if ( copy_to_guest_offset(mtopology.vdistance,
+                                            __vnode_distance_offset(d, i, j),
+                                            &__vnode_distance(d, i, j), 1) )
+                    goto out;
+            }
+        for ( i = 0; i < d->max_vcpus ; i++ )
+        {
+            if ( copy_to_guest_offset(mtopology.vcpu_to_vnode, i,
+                                        &d->vnuma.vcpu_to_vnode[i], 1) )
+                    goto out;
+        }
+        return rc;
+out:
+        if ( vblks ) xfree(vblks);
+        return rc;
+        break;
+    }
     default:
         rc = arch_memory_op(op, arg);
         break;
xen/include/public/domctl.h +14 -1
index 4c5b2bb..3574d0a 100644
@@ -35,6 +35,7 @@
 #include "xen.h"
 #include "grant_table.h"
 #include "hvm/save.h"
+#include "memory.h"
 
 #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009
 
@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {
 typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_broken_page_p2m_t);
 
+struct xen_domctl_vnuma {
+    uint16_t nr_vnodes;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(uint) vnode_to_pnode;
+    XEN_GUEST_HANDLE_64(vnuma_memblk_t) vnuma_memblks;
+};
+
+typedef struct xen_domctl_vnuma xen_domctl_vnuma_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_vnuma_t);
+
 struct xen_domctl {
     uint32_t cmd;
 #define XEN_DOMCTL_createdomain                   1
@@ -920,6 +932,7 @@ struct xen_domctl {
 #define XEN_DOMCTL_set_broken_page_p2m           67
 #define XEN_DOMCTL_setnodeaffinity               68
 #define XEN_DOMCTL_getnodeaffinity               69
+#define XEN_DOMCTL_setvnumainfo                  70
 #define XEN_DOMCTL_gdbsx_guestmemio            1000
 #define XEN_DOMCTL_gdbsx_pausevcpu             1001
 #define XEN_DOMCTL_gdbsx_unpausevcpu           1002
@@ -979,6 +992,7 @@ struct xen_domctl {
         struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;
         struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;
         struct xen_domctl_gdbsx_domstatus   gdbsx_domstatus;
+        struct xen_domctl_vnuma             vnuma;
         uint8_t                             pad[128];
     } u;
 };
@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_t);
 
 #endif /* __XEN_PUBLIC_DOMCTL_H__ */
-
 /*
  * Local variables:
  * mode: C
xen/include/public/memory.h +8 -1
index 7a26dee..28f6aaf 100644
@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);
  * Caller must be privileged or the hypercall fails.
  */
 #define XENMEM_claim_pages                  24
-
 /*
  * XENMEM_claim_pages flags - the are no flags at this time.
  * The zero value is appropiate.
  */
 
+struct vnuma_memblk {
+              uint64_t start, end;
+};
+typedef struct vnuma_memblk vnuma_memblk_t;
+DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t);
+
+#define XENMEM_get_vnuma_info               25
+
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
 #endif /* __XEN_PUBLIC_MEMORY_H__ */
xen/include/xen/domain.h +1 -0
index a057069..c9d53e3 100644
@@ -4,6 +4,7 @@
 
 #include <public/xen.h>
 #include <asm/domain.h>
+#include <public/memory.h>
 
 typedef union {
     struct vcpu_guest_context *nat;

xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99

Quoted 10 lines
xen/include/xen/domain.h +7 -0
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;
 
 extern bool_t opt_dom0_vcpus_pin;
 
+struct domain_vnuma_info {
+    uint16_t nr_vnodes;
+    uint *vdistance;
+    uint *vcpu_to_vnode;
+    uint *vnode_to_pnode;
+    vnuma_memblk_t *vnuma_memblks;
+};

Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?

Quoted 1 line

-George

--
Elena

Signature · 1 line
Elena
#7 lore raw
In reply to Elena Ufimtseva
Quoted 1 line
Quoted 1 line
Quoted 1 line · Elena Ufimtseva

Please don't top post.

Quoted 4 lines

George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.

With nr_vnodes being uint16_t, the vnode numbers should be too. Limiting them to u8 would possibly be even better, but then nr_vnodes would better be unsigned int (perhaps that was the case from the beginning, regardless of the types used for the arrays).

The pnode array surely can also be uint8_t for the time being, considering that there are other places where node IDs are limited to 8 bits.

And with struct acpi_table_slit having just 8-bit distances, there's no apparent reason why the virtual distances can't be 8 bits too.

But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.

Jan

Quoted 14 lines
Quoted 14 lines

xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99

Quoted 11 lines
xen/include/xen/domain.h +7 -0
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;
 
 extern bool_t opt_dom0_vcpus_pin;
 
+struct domain_vnuma_info {
+    uint16_t nr_vnodes;
+    uint *vdistance;
+    uint *vcpu_to_vnode;
+    uint *vnode_to_pnode;
+    vnuma_memblk_t *vnuma_memblks;
+};

Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?

#8 lore raw
Quoted 15 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · Elena Ufimtseva

Please don't top post.

Quoted 4 lines

George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.

With nr_vnodes being uint16_t, the vnode numbers should be too. Limiting them to u8 would possibly be even better, but then nr_vnodes would better be unsigned int (perhaps that was the case from the beginning, regardless of the types used for the arrays).

Quoted 4 lines

The pnode array surely can also be uint8_t for the time being, considering that there are other places where node IDs are limited to 8 bits.

All agreed.

Quoted 7 lines

And with struct acpi_table_slit having just 8-bit distances, there's no apparent reason why the virtual distances can't be 8 bits too.

But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.

And, in fact, 'node_to_node_distance' in xen/include/public/sysctl.h
(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h)
is uint32.

Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#9 lore raw
Quoted 25 lines · Dario Faggioli
Quoted 15 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · Elena Ufimtseva

Please don't top post.

Quoted 4 lines

George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.

With nr_vnodes being uint16_t, the vnode numbers should be too. Limiting them to u8 would possibly be even better, but then nr_vnodes would better be unsigned int (perhaps that was the case from the beginning, regardless of the types used for the arrays).

Quoted 4 lines

The pnode array surely can also be uint8_t for the time being, considering that there are other places where node IDs are limited to 8 bits.

All agreed.

Quoted 3 lines

And with struct acpi_table_slit having just 8-bit distances, there's no apparent reason why the virtual distances can't be 8 bits too.

Quoted 8 lines
Quoted 4 lines

But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.

And, in fact, 'node_to_node_distance' in xen/include/public/sysctl.h
(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h)
is uint32.

Linux has u8 for distance. Ok, thank you for pointing that out.

Elena

Quoted 8 lines

Dario

--
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

Signature · 1 line
Elena
#10 lore raw
Quoted 7 lines · Elena Ufimtseva
Quoted 5 lines · Dario Faggioli
Quoted 4 lines · Jan Beulich

But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.

Quoted 7 lines
Quoted 4 lines

And, in fact, 'node_to_node_distance' in xen/include/public/sysctl.h
(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h)
is uint32.

Linux has u8 for distance. Ok, thank you for pointing that out.

EhEh... So, very hard to be consistent with every actor in the play! :-)

Anyway, I think the point here is, as Jan was saying, to distinguish internal representation from interface. Within Xen, we should store everything in the smallest and nicest possible way (which, BTW, is what Linux does by using u8 for distances).

OTOH, when it comes to exported interfaces, we should be much more cautious, since changing the way Xen stores distances internally is trivial, changing the interface (either API or ABI) could be a nightmare!

In this case, what we (Xen) tell Linux is the interface, it is up to him (Linux) to convert that in a way that suits its own internals. In fact, despite Linux being the only one OS using this interface for now, it's not wise to design the interface itself specifically for Linux, since many OSes may want to support it in the future.

Regards,
Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#11 lore raw

Hi Elena,
Thank you for your great work first :)

In your patch, guest can use a hypercall name: "XENMEM_get_vnuma_info" to get the structure below:
+struct vnuma_topology_info {

+    domid_t domid;
+    uint16_t nr_vnodes;
+    uint32_t _pad;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(vnuma_

memblk_t) vnuma_memblks;
+};

I see that Xen have save a vnode_to_pnode[] array for mapping However, this hypercall won't give this mapping to Guest VM. Is that right?

Quoted 46 lines · Dario Faggioli
Quoted 9 lines · Elena Ufimtseva
Quoted 7 lines · Dario Faggioli
Quoted 4 lines · Jan Beulich

But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.

And, in fact, 'node_to_node_distance' in xen/include/public/sysctl.h (

http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h
)

Quoted 5 lines
Quoted 2 lines

is uint32.

Linux has u8 for distance. Ok, thank you for pointing that out.

EhEh... So, very hard to be consistent with every actor in the play! :-)

Anyway, I think the point here is, as Jan was saying, to distinguish internal representation from interface. Within Xen, we should store everything in the smallest and nicest possible way (which, BTW, is what Linux does by using u8 for distances).

OTOH, when it comes to exported interfaces, we should be much more cautious, since changing the way Xen stores distances internally is trivial, changing the interface (either API or ABI) could be a nightmare!

In this case, what we (Xen) tell Linux is the interface, it is up to him (Linux) to convert that in a way that suits its own internals. In fact, despite Linux being the only one OS using this interface for now, it's not wise to design the interface itself specifically for Linux, since many OSes may want to support it in the future.

Regards,
Dario

--
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

Signature · 8 lines
Yechen Li

Team of System Virtualization and Cloud Computing
School of Electronic Engineering  and Computer Science,
Peking University, China

Nothing is impossible because impossible itself  says: " I'm possible "
lccycc From PKU

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#12 lore raw
Quoted 16 lines · Li Yechen

Hi Elena,
Thank you for your great work first :)

In your patch, guest can use a hypercall name: "XENMEM_get_vnuma_info" to get the structure below:

(quoted diff) +7 -0
+struct vnuma_topology_info {
+    domid_t domid;
+    uint16_t nr_vnodes;
+    uint32_t _pad;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(vnuma_

memblk_t) vnuma_memblks;
+};

Quoted 2 lines

I see that Xen have save a vnode_to_pnode[] array for mapping

Yes it does. We spoke about it quite a bit, and Xen seems to be the best place where to store such info, considering how easy it is to keep it there, and that it may turn out useful in future (and ballooning is a lcear example of that).

Quoted 2 lines

However, this hypercall won't give this mapping to Guest VM.

Not with this hypercall, no. Actually, did most of the people, when reviewing your RFC patches, said that they prefer the mapping not exposed at all to the guest?

Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#13 lore raw

Hello Li, Dario! :)

Quoted 25 lines · Dario Faggioli
Quoted 17 lines · Li Yechen

Hi Elena,
Thank you for your great work first :)

In your patch, guest can use a hypercall name: "XENMEM_get_vnuma_info" to get the structure below:

(quoted diff) +7 -0
+struct vnuma_topology_info {
+    domid_t domid;
+    uint16_t nr_vnodes;
+    uint32_t _pad;
+    XEN_GUEST_HANDLE_64(uint) vdistance;
+    XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
+    XEN_GUEST_HANDLE_64(vnuma_

memblk_t) vnuma_memblks;
+};

Quoted 2 lines

I see that Xen have save a vnode_to_pnode[] array for mapping

Yes it does. We spoke about it quite a bit, and Xen seems to be the best place where to store such info, considering how easy it is to keep it there, and that it may turn out useful in future (and ballooning is a lcear example of that).

Quoted 5 lines
Quoted 2 lines

However, this hypercall won't give this mapping to Guest VM.

Not with this hypercall, no. Actually, did most of the people, when reviewing your RFC patches, said that they prefer the mapping not exposed at all to the guest?

Yes Dario, correct. I think someone, possibly Konrad, was saying that it would be an interesting possibility.
But I dont think there was a change in this.
So fo Li I think I can provide hypercall that will return the mappings. Li, do you think it work for you?

Quoted 9 lines

Dario

--
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

Signature · 1 line
Elena