xen/vNUMA: vNUMA support for PV guests.
RFC PATCH v2 1/7 xen/vNUMA: vNUMA support for PV guests.
Defines XENMEM subop hypercall for PV vNUMA enabled guests and provides vNUMA topology information from per-domain vnuma topology build info.
---
Changes since v1:
* changed type from int to uint and unsigned in vNUMA structures;
* removed unecessary file vnuma.h as requested in review
* added domain_vnuma_destroy;
* fixed usage of rcu_lock_domain_by_any_id;
* removed unecessary guest_handle_cast calls;
* coding style fixes;
--- xen/common/domain.c | 25 +++++++++++++++- xen/common/domctl.c | 68 ++++++++++++++++++++++++++++++++++++++++++- xen/common/memory.c | 56 +++++++++++++++++++++++++++++++++++ xen/include/public/domctl.h | 15 +++++++++- xen/include/public/memory.h | 9 +++++- xen/include/xen/domain.h | 11 +++++++ xen/include/xen/sched.h | 1 + xen/include/xen/vnuma.h | 27 +++++++++++++++++ 8 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 xen/include/xen/vnuma.h
xen/common/domain.c +24 -1
@@ -227,6 +227,7 @@ struct domain *domain_create(spin_lock_init (&d->node_affinity_lock); d->node_affinity =NODE_MASK_ALL ; d->auto_node_affinity =1 ; + d->vnuma.nr_vnodes =0 ;spin_lock_init (&d->shutdown_lock); d->shutdown_code =-1 ;
@@ -530,8 +531,9 @@ int domain_kill(struct domain *d)evtchn_destroy (d);gnttab_release_mappings (d);tmem_destroy (d->tmem); -domain_set_outstanding_pages (d,0 ); d->tmem =NULL ; +domain_set_outstanding_pages (d,0 ); +domain_vnuma_destroy (&d->vnuma);/* fallthrough */ case DOMDYING_dying: rc =domain_relinquish_resources (d);
@@ -1279,6 +1281,27 @@ int continue_hypercall_on_cpu(return 0 ; } +void domain_vnuma_destroy (struct domain_vnuma_info *v) +{ +if (v->vnuma_memblks) { +xfree (v->vnuma_memblks); + v->vnuma_memblks =NULL ; + } +if (v->vcpu_to_vnode) { +xfree (v->vcpu_to_vnode); + v->vcpu_to_vnode =NULL ; + } +if (v->vdistance) { +xfree (v->vdistance); + v->vdistance =NULL ; + } +if (v->vnode_to_pnode) { +xfree (v->vnode_to_pnode); + v->vnode_to_pnode =NULL ; + } + v->nr_vnodes =0 ; +} +/* * Local variables: * mode: C
xen/common/domctl.c +67 -1
80 lines not on this page · raw message
xen/common/memory.c +56 -0
69 lines not on this page · raw message
xen/include/public/domctl.h +14 -1
@@ -35,6 +35,7 @@#include "xen.h" #include "grant_table.h" #include "hvm/save.h" +#include "memory.h" #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009
@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t ;DEFINE_XEN_GUEST_HANDLE (xen_domctl_set_broken_page_p2m_t); +struct xen_domctl_vnuma { +uint16_t nr_vnodes; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (uint) vnode_to_pnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; + +typedef struct xen_domctl_vnuma xen_domctl_vnuma_t ; +DEFINE_XEN_GUEST_HANDLE (xen_domctl_vnuma_t); +struct xen_domctl {uint32_t cmd;#define XEN_DOMCTL_createdomain 1
@@ -920,6 +932,7 @@ struct xen_domctl {#define XEN_DOMCTL_set_broken_page_p2m 67#define XEN_DOMCTL_setnodeaffinity 68#define XEN_DOMCTL_getnodeaffinity 69 +#define XEN_DOMCTL_setvnumainfo 70#define XEN_DOMCTL_gdbsx_guestmemio 1000#define XEN_DOMCTL_gdbsx_pausevcpu 1001#define XEN_DOMCTL_gdbsx_unpausevcpu 1002
@@ -979,6 +992,7 @@ struct xen_domctl {struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; +struct xen_domctl_vnuma vnuma;uint8_t pad[128 ]; } u; };
@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;DEFINE_XEN_GUEST_HANDLE (xen_domctl_t);#endif /* __XEN_PUBLIC_DOMCTL_H__ */ -/* * Local variables: * mode: C
xen/include/public/memory.h +8 -1
@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);* Caller must be privileged or the hypercall fails. */ #define XENMEM_claim_pages 24 -/* * XENMEM_claim_pages flags - the are no flags at this time. * The zero value is appropiate. */ +struct vnuma_memblk { +uint64_t start, end; +}; +typedef struct vnuma_memblk vnuma_memblk_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_memblk_t); + +#define XENMEM_get_vnuma_info 25 +#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */ #endif /* __XEN_PUBLIC_MEMORY_H__ */
xen/include/xen/domain.h +11 -0
@@ -4,6 +4,7 @@#include <public/xen.h> #include <asm/domain.h> +#include <public/memory.h> typedef union { struct vcpu_guest_context *nat;
@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;extern bool_t opt_dom0_vcpus_pin; +struct domain_vnuma_info { +uint16_t nr_vnodes; +uint *vdistance; +uint *vcpu_to_vnode; +uint *vnode_to_pnode; +vnuma_memblk_t *vnuma_memblks; +}; + +void domain_vnuma_destroy (struct domain_vnuma_info *v); +#endif /* __XEN_DOMAIN_H__ */
xen/include/xen/sched.h +1 -0
@@ -377,6 +377,7 @@ struct domainnodemask_t node_affinity;unsigned int last_alloc_node;spinlock_t node_affinity_lock; +struct domain_vnuma_info vnuma; };struct domain_setup_info
xen/include/xen/vnuma.h new file +27 -0
@@ -0,0 +1,27 @@ +#ifndef _VNUMA_H +#define _VNUMA_H +#include <public/memory.h> + +/* DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t); */ + +struct vnuma_topology_info { +domid_t domid; +uint16_t nr_vnodes; +uint32_t _pad; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; +typedef struct vnuma_topology_info vnuma_topology_info_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_topology_info_t); + +#define __vnode_distance_offset (_dom, _i, _j) \ + ( ((_j) * ((_dom)->vnuma.nr_vnodes)) + (_i) ) + +#define __vnode_distance (_dom, _i, _j) \ + ( (_dom)->vnuma.vdistance[__vnode_distance_offset((_dom), (_i), (_j))] ) + +#define __vnode_distance_set (_dom, _i, _j, _v) \ + do { __vnode_distance((_dom), (_i), (_j)) = (_v); } while(0) + +#endif
Signature · 1 line
1.7.10.4
Quoted 31 lines · Elena Ufimtseva
Defines XENMEM subop hypercall for PV vNUMA enabled guests and provides vNUMA topology information from per-domain vnuma topology build info.
Signed-off-by Elena Ufimtseva <[email protected]>---
Changes since v1:
* changed type from int to uint and unsigned in vNUMA structures;
* removed unecessary filevnuma.has requested in review
* addeddomain_vnuma_destroy;
* fixed usage ofrcu_lock_domain_by_any_id;
* removed unecessaryguest_handle_castcalls;
* coding style fixes;--- xen/common/domain.c | 25 +++++++++++++++- xen/common/domctl.c | 68 ++++++++++++++++++++++++++++++++++++++++++- xen/common/memory.c | 56 +++++++++++++++++++++++++++++++++++ xen/include/public/domctl.h | 15 +++++++++- xen/include/public/memory.h | 9 +++++- xen/include/xen/domain.h | 11 +++++++ xen/include/xen/sched.h | 1 + xen/include/xen/vnuma.h | 27 +++++++++++++++++ 8 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 xen/include/xen/vnuma.hxen/common/domain.c +0
-0
xen/common/domain.c struct domain *domain_create( · line 230
Quoted 4 lines
xen/common/domain.c +1
-0@@ -227,6 +227,7 @@ struct domain *domain_create(spin_lock_init (&d->node_affinity_lock); d->node_affinity =NODE_MASK_ALL ; d->auto_node_affinity =1 ; + d->vnuma.nr_vnodes =0 ;
alloc_domain_struct() clears d, so this assignment of 0 is not needed.
xen/common/domain.c int domain_kill(struct domain *d) · line 533
Quoted 8 lines
xen/common/domain.c +0
-1spin_lock_init (&d->shutdown_lock); d->shutdown_code =-1 ;@@ -530,8 +531,9 @@ int domain_kill(struct domain *d)evtchn_destroy (d);gnttab_release_mappings (d);tmem_destroy (d->tmem); -domain_set_outstanding_pages (d,0 );
Any particular reason for this function call to move? I cant see any reason.
Quoted 7 lines
xen/common/domain.c +2
-0d->tmem =NULL ; +domain_set_outstanding_pages (d,0 ); +domain_vnuma_destroy (&d->vnuma);/* fallthrough */ case DOMDYING_dying: rc =domain_relinquish_resources (d);
xen/common/domain.c int continue_hypercall_on_cpu( · line 1289
Quoted 9 lines
xen/common/domain.c +6
-0@@ -1279,6 +1281,27 @@ int continue_hypercall_on_cpu(return 0 ; } +void domain_vnuma_destroy (struct domain_vnuma_info *v) +{ +if (v->vnuma_memblks) { +xfree (v->vnuma_memblks); + v->vnuma_memblks =NULL ; + }
Coding style (see CODING_STYLE in the root of the tree) so this would become:
if ( v->vnuma_memblks )
{
xfree(v->vnuma_memblks);
v->vnuma_memblks = NULL;
}
However, xfree() (like regular free()) is happy with NULL pointers, so you can drop the if altogether.
Quoted 31 lines
xen/common/domain.c +15
-0+if (v->vcpu_to_vnode) { +xfree (v->vcpu_to_vnode); + v->vcpu_to_vnode =NULL ; + } +if (v->vdistance) { +xfree (v->vdistance); + v->vdistance =NULL ; + } +if (v->vnode_to_pnode) { +xfree (v->vnode_to_pnode); + v->vnode_to_pnode =NULL ; + } + v->nr_vnodes =0 ; +} +/* * Local variables: * mode: C xen/common/domctl.c +1
-0@@ -29,6 +29,7 @@#include <asm/page.h> #include <public/domctl.h> #include <xsm/xsm.h> +#include <xen/vnuma.h> static DEFINE_SPINLOCK (domctl_lock);DEFINE_SPINLOCK (vcpu_alloc_lock);
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 865
Quoted 4 lines
xen/common/domctl.c +0
-1@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) ret =set_global_virq_handler (d, virq); }break ; -
Please leave this blank line in here.
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876
Quoted 11 lines
xen/common/domctl.c +11
-0+case XEN_DOMCTL_setvnumainfo: + { +unsigned int i, j, nr_vnodes, dist_size; +unsigned int dist, vmap, vntop; +vnuma_memblk_t vmemblk; + + ret = -EINVAL ; + dist = i = j =0 ; + nr_vnodes = op->u.vnuma.nr_vnodes; +if (nr_vnodes <=0 || nr_vnodes >NR_CPUS ) +break ;
nr_vnodes is unsigned. It cannot possibly be less than 0.
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 881
Quoted 5 lines
xen/common/domctl.c +5
-0+ d->vnuma.nr_vnodes = nr_vnodes; + dist_size = nr_vnodes * nr_vnodes; + if ( + (d->vnuma.vdistance = xmalloc_bytes( + sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||
sizeof in an operator not a function. As you are not passing a type, the brackets are not required.
Quoted 4 lines
xen/common/domctl.c +4
-0+ (d->vnuma.vnuma_memblks =xmalloc_bytes ( +sizeof (*d->vnuma.vnuma_memblks) * nr_vnodes)) ==NULL || + (d->vnuma.vcpu_to_vnode =xmalloc_bytes ( + sizeof(*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) == NULL ||
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 895
Quoted 10 lines
xen/common/domctl.c +10
-0+ (d->vnuma.vnode_to_pnode = xmalloc_bytes( + sizeof(*d->vnuma.vnode_to_pnode) * nr_vnodes)) == NULL ) + goto err_dom; +for ( i =0 ; i < nr_vnodes; i++ ) +for ( j =0 ; j < nr_vnodes; j++ ) + { +if (unlikely (__copy_from_guest_offset (&dist, + op->u.vnuma.vdistance, +__vnode_distance_offset (d, i, j),1 )) ) + goto err_dom;
This error logic is broken. A failure in __copy_from_guest_offset() should result in -EFAULT, but will result in -EINVAL after following err_dom;
Quoted 26 lines
xen/common/domctl.c +26
-0+__vnode_distance_set (d, i, j, dist); + } +for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmemblk, + op->u.vnuma.vnuma_memblks, i,1 )) ) + goto err_dom; + d->vnuma.vnuma_memblks[i].start = vmemblk.start; + d->vnuma.vnuma_memblks[i].end = vmemblk.end; + } +for ( i =0 ; i < d->max_vcpus; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmap, + op->u.vnuma.vcpu_to_vnode, i,1 )) ) + goto err_dom; + d->vnuma.vcpu_to_vnode[i] = vmap; + } +if ( !guest_handle_is_null (op->u.vnuma.vnode_to_pnode) ) + { +for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vntop, + op->u.vnuma.vnode_to_pnode, i,1 )) ) + goto err_dom; + d->vnuma.vnode_to_pnode[i] = vntop; + }
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 931
Quoted 10 lines
xen/common/domctl.c +10
-0+ } +else +for (i =0 ; i < nr_vnodes; i++) + d->vnuma.vnode_to_pnode[i] =NUMA_NO_NODE ; + ret =0 ; +break ; +err_dom: + ret = -EINVAL ; + } +break ;
And an extra line in here please.
Quoted 23 lines
xen/common/domctl.c +0
-0default : ret =arch_do_domctl (op, d, u_domctl);break ;xen/common/memory.c +5
-0@@ -28,6 +28,7 @@#include <public/memory.h> #include <xsm/xsm.h> #include <xen/trace.h> +#include <xen/vnuma.h> struct memop_args {/* INPUT */ @@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)rcu_unlock_domain (d);break ; +case XENMEM_get_vnuma_info: + { +int i, j; +struct vnuma_topology_info mtopology;
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749
Quoted 10 lines
xen/common/memory.c +9
-0+struct vnuma_topology_info touser_topo; +struct domain *d; +unsigned int max_pages, max_vcpus, nr_vnodes; +vnuma_memblk_t *vblks; + rc = -EINVAL ; +if (guest_handle_is_null (arg) ) +return rc; +if (copy_from_guest (&mtopology, arg,1 ) ) +return -EINVAL ;
-EFAULT;
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 751
Quoted 2 lines
xen/common/memory.c +2
-0+if ( (d =rcu_lock_domain_by_any_id (mtopology.domid)) ==NULL ) +return -EINVAL ;
-ESRCH ( I think? )
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 762
Quoted 11 lines
xen/common/memory.c +11
-0+ touser_topo.nr_vnodes = d->vnuma.nr_vnodes; + max_pages = d->max_pages; + max_vcpus = d->max_vcpus; +rcu_unlock_domain (d); + + nr_vnodes = touser_topo.nr_vnodes; + rc =copy_to_guest (arg, &touser_topo,1 ); +if ( rc ) +return -EFAULT ; +if ( nr_vnodes ==0 || nr_vnodes > max_vcpus ) +return -EFAULT ;
-EINVAL;
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765
Quoted 3 lines
xen/common/memory.c +3
-0+ vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes); +if ( vblks ==NULL ) +return -EFAULT ;
-ENOMEM;
Now, if you have an unconditional rc = -EFAULT then all your goto out logic will be correct
Quoted 11 lines
xen/common/memory.c +11
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (copy_to_guest_offset (mtopology.vnuma_memblks, i, + &d->vnuma.vnuma_memblks[i],1 ) <0 ) + goto out; + } +for ( i =0 ; i < touser_topo.nr_vnodes; i++ ) +for ( j =0 ; j < touser_topo.nr_vnodes; j++ ) + { + if ( copy_to_guest_offset(mtopology.vdistance, + __vnode_distance_offset(d, i, j),
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 786
Quoted 10 lines
xen/common/memory.c +10
-0+ &__vnode_distance(d, i, j), 1) ) + goto out; + } +for ( i =0 ; i < d->max_vcpus ; i++ ) + { +if (copy_to_guest_offset (mtopology.vcpu_to_vnode, i, + &d->vnuma.vcpu_to_vnode[i],1 ) ) + goto out; + } +return rc;
But this would need to turn into rc = 0; to avoid return -EFAULT in the success case, and unconditionally leaking vblks
Quoted 55 lines
xen/common/memory.c +5
-0+out: +if ( vblks )xfree (vblks); +return rc; +break ; + }default : rc =arch_memory_op (op, arg);break ;xen/include/public/domctl.h +14
-0@@ -35,6 +35,7 @@#include "xen.h" #include "grant_table.h" #include "hvm/save.h" +#include "memory.h" #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t ;DEFINE_XEN_GUEST_HANDLE (xen_domctl_set_broken_page_p2m_t); +struct xen_domctl_vnuma { +uint16_t nr_vnodes; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (uint) vnode_to_pnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; + +typedef struct xen_domctl_vnuma xen_domctl_vnuma_t ; +DEFINE_XEN_GUEST_HANDLE (xen_domctl_vnuma_t); +struct xen_domctl {uint32_t cmd;#define XEN_DOMCTL_createdomain 1@@ -920,6 +932,7 @@ struct xen_domctl {#define XEN_DOMCTL_set_broken_page_p2m 67#define XEN_DOMCTL_setnodeaffinity 68#define XEN_DOMCTL_getnodeaffinity 69 +#define XEN_DOMCTL_setvnumainfo 70#define XEN_DOMCTL_gdbsx_guestmemio 1000#define XEN_DOMCTL_gdbsx_pausevcpu 1001#define XEN_DOMCTL_gdbsx_unpausevcpu 1002@@ -979,6 +992,7 @@ struct xen_domctl {struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; +struct xen_domctl_vnuma vnuma;uint8_t pad[128 ]; } u; };
xen/include/public/domctl.h typedef struct xen_domctl xen_domctl_t; · line 989
Quoted 4 lines
xen/include/public/domctl.h +0
-1@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;DEFINE_XEN_GUEST_HANDLE (xen_domctl_t);#endif /* __XEN_PUBLIC_DOMCTL_H__ */ -
Spurious whitespace change. Please remove
xen/include/public/memory.h DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t); · line 456
Quoted 12 lines
xen/include/public/domctl.h +0
-0/* * Local variables: * mode: C xen/include/public/memory.h +0
-1@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);* Caller must be privileged or the hypercall fails. */ #define XENMEM_claim_pages 24 -
And here
Quoted 73 lines
xen/include/public/memory.h +8
-0/* * XENMEM_claim_pages flags - the are no flags at this time. * The zero value is appropiate. */ +struct vnuma_memblk { +uint64_t start, end; +}; +typedef struct vnuma_memblk vnuma_memblk_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_memblk_t); + +#define XENMEM_get_vnuma_info 25 +#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */ #endif /* __XEN_PUBLIC_MEMORY_H__ */ xen/include/xen/domain.h +11
-0@@ -4,6 +4,7 @@#include <public/xen.h> #include <asm/domain.h> +#include <public/memory.h> typedef union { struct vcpu_guest_context *nat;@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;extern bool_t opt_dom0_vcpus_pin; +struct domain_vnuma_info { +uint16_t nr_vnodes; +uint *vdistance; +uint *vcpu_to_vnode; +uint *vnode_to_pnode; +vnuma_memblk_t *vnuma_memblks; +}; + +void domain_vnuma_destroy (struct domain_vnuma_info *v); +#endif /* __XEN_DOMAIN_H__ */ xen/include/xen/sched.h +1
-0@@ -377,6 +377,7 @@ struct domainnodemask_t node_affinity;unsigned int last_alloc_node;spinlock_t node_affinity_lock; +struct domain_vnuma_info vnuma; };struct domain_setup_info xen/include/xen/vnuma.h new file +12
-0@@ -0,0 +1,27 @@ +#ifndef _VNUMA_H +#define _VNUMA_H +#include <public/memory.h> + +/* DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t); */ + +struct vnuma_topology_info { + domid_t domid; + uint16_t nr_vnodes; + uint32_t _pad; + XEN_GUEST_HANDLE_64(uint) vdistance; + XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode;
xen/include/xen/vnuma.h line 22
Quoted 10 lines
xen/include/xen/vnuma.h new file +10
-0+XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; +typedef struct vnuma_topology_info vnuma_topology_info_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_topology_info_t); + +#define __vnode_distance_offset (_dom, _i, _j) \ + ( ((_j) * ((_dom)->vnuma.nr_vnodes)) + (_i) ) + +#define __vnode_distance (_dom, _i, _j) \ + ( (_dom)->vnuma.vdistance[__vnode_distance_offset((_dom), (_i), (_j))] )
You expand _dom here twice, which is bad practice for macros.
I suggest static inline functions as a better alternative.
~Andrew
Quoted 5 lines
xen/include/xen/vnuma.h new file +5
-0+ +#define __vnode_distance_set (_dom, _i, _j, _v) \ + do { __vnode_distance((_dom), (_i), (_j)) = (_v); } while(0) + +#endif
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Andrew Cooper
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 881
Quoted 3 lines · Elena Ufimtseva
xen/common/domctl.c +3
-0+ if ( + (d->vnuma.vdistance = xmalloc_bytes( + sizeof(*d->vnuma.vdistance) * dist_size)) == NULL ||sizeof in an operator not a function. As you are not passing a type, the brackets are not required.
Actually, I'd request parentheses to be added if I saw a use of sizeof() without them. I think we have quite few (hopefully a majority) of uses similar to the one above in the tree.
Jan
xen/common/domain.c struct domain *domain_create( · line 230
Quoted 8 lines
Quoted 1 line
Quoted 1 line · Elena Ufimtseva
xen/common/domain.c +1
-0@@ -227,6 +227,7 @@ struct domain *domain_create(spin_lock_init (&d->node_affinity_lock); d->node_affinity =NODE_MASK_ALL ; d->auto_node_affinity =1 ; + d->vnuma.nr_vnodes =0 ;
Pointless (struct domain gets zero-allocated)?
xen/common/domain.c int domain_kill(struct domain *d) · line 535
Quoted 7 lines
xen/common/domain.c +1
-1@@ -530,8 +531,9 @@ int domain_kill(struct domain *d)evtchn_destroy (d);gnttab_release_mappings (d);tmem_destroy (d->tmem); -domain_set_outstanding_pages (d,0 ); d->tmem =NULL ; +domain_set_outstanding_pages (d,0 );
Why is this being moved?
Quoted 9 lines
xen/common/domain.c +9
-0+void domain_vnuma_destroy (struct domain_vnuma_info *v) +{ +if (v->vnuma_memblks) { +xfree (v->vnuma_memblks); + v->vnuma_memblks =NULL ; + } +if (v->vcpu_to_vnode) { +xfree (v->vcpu_to_vnode); + v->vcpu_to_vnode =NULL ;
xen/common/domain.c int continue_hypercall_on_cpu( · line 1302
Quoted 10 lines
xen/common/domain.c +10
-0+ } +if (v->vdistance) { +xfree (v->vdistance); + v->vdistance =NULL ; + } +if (v->vnode_to_pnode) { +xfree (v->vnode_to_pnode); + v->vnode_to_pnode =NULL ; + } + v->nr_vnodes =0 ;
All the if()s are pointless - xfree() deals with NULL input quite fine. And for the record, they also all don't match our coding style.
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 865
Quoted 5 lines
xen/common/domain.c +0
-1@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) ret =set_global_virq_handler (d, virq); }break ; -
Please keep that newline, and insert another one after the final break of the code you add.
Quoted 13 lines
xen/common/domain.c +13
-0+case XEN_DOMCTL_setvnumainfo: + { +unsigned int i, j, nr_vnodes, dist_size; +unsigned int dist, vmap, vntop; +vnuma_memblk_t vmemblk; + + ret = -EINVAL ; + dist = i = j =0 ; + nr_vnodes = op->u.vnuma.nr_vnodes; +if (nr_vnodes <=0 || nr_vnodes >NR_CPUS ) +break ; + d->vnuma.nr_vnodes = nr_vnodes; + dist_size = nr_vnodes * nr_vnodes;
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888
Quoted 10 lines
xen/common/domain.c +10
-0+if ( + (d->vnuma.vdistance =xmalloc_bytes ( +sizeof (*d->vnuma.vdistance) * dist_size)) ==NULL || + (d->vnuma.vnuma_memblks =xmalloc_bytes ( +sizeof (*d->vnuma.vnuma_memblks) * nr_vnodes)) ==NULL || + (d->vnuma.vcpu_to_vnode =xmalloc_bytes ( +sizeof (*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) ==NULL || + (d->vnuma.vnode_to_pnode =xmalloc_bytes ( +sizeof (*d->vnuma.vnode_to_pnode) * nr_vnodes)) ==NULL ) + goto err_dom;
Coding style. Also, you leak all memory that you managed to get hold of in the error case.
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 892
Quoted 4 lines
xen/common/domain.c +4
-0+for ( i =0 ; i < nr_vnodes; i++ ) +for ( j =0 ; j < nr_vnodes; j++ ) + { + if ( unlikely(__copy_from_guest_offset(&dist,
__copy_from_guest_offset() without prior guest_handle_okay()? And the error code should be -EFAULT in that case.
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 898
Quoted 6 lines
xen/common/domain.c +6
-0+ op->u.vnuma.vdistance, + __vnode_distance_offset(d, i, j), 1)) ) + goto err_dom; +__vnode_distance_set (d, i, j, dist); + } +for ( i =0 ; i < nr_vnodes; i++ )
If possible please fold this with the earlier loop.
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 911
Quoted 13 lines
xen/common/domain.c +13
-0+ { +if (unlikely (__copy_from_guest_offset (&vmemblk, + op->u.vnuma.vnuma_memblks, i,1 )) ) + goto err_dom; + d->vnuma.vnuma_memblks[i].start = vmemblk.start; + d->vnuma.vnuma_memblks[i].end = vmemblk.end; + } +for ( i =0 ; i < d->max_vcpus; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmap, + op->u.vnuma.vcpu_to_vnode, i,1 )) ) + goto err_dom; + d->vnuma.vcpu_to_vnode[i] = vmap;
If the types are compatible, things like this don't need doing in a loop - you could copy the whole array in one go.
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 924
Quoted 13 lines
xen/common/domain.c +13
-0+ } +if ( !guest_handle_is_null (op->u.vnuma.vnode_to_pnode) ) + { +for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vntop, + op->u.vnuma.vnode_to_pnode, i,1 )) ) + goto err_dom; + d->vnuma.vnode_to_pnode[i] = vntop; + } + } +else +for (i =0 ; i < nr_vnodes; i++)
Coding style.
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925
Quoted 1 line
xen/common/domain.c +1
-0+ d->vnuma.vnode_to_pnode[i] =NUMA_NO_NODE ;
What's the value of using this domctl this way?
xen/common/domain.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 929
Quoted 4 lines
xen/common/domain.c +4
-0+ ret =0 ; +break ; +err_dom: + ret = -EINVAL ;
"ret" should not be unconditionally set to a fixed value here, the more that you set it almost first thing in the case statement.
Quoted 8 lines
xen/common/domain.c +4
-0@@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)rcu_unlock_domain (d);break ; +case XENMEM_get_vnuma_info: + { +int i, j; +struct vnuma_topology_info mtopology;
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749
Quoted 10 lines
xen/common/domain.c +9
-0+struct vnuma_topology_info touser_topo; +struct domain *d; +unsigned int max_pages, max_vcpus, nr_vnodes; +vnuma_memblk_t *vblks; + rc = -EINVAL ; +if (guest_handle_is_null (arg) ) +return rc; +if (copy_from_guest (&mtopology, arg,1 ) ) +return -EINVAL ;
Consistency please, preferably such that you - as long as possible - always return -E rather than rc.
Also the return value here again is -EFAULT.
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 751
Quoted 2 lines
xen/common/domain.c +2
-0+if ( (d =rcu_lock_domain_by_any_id (mtopology.domid)) ==NULL ) +return -EINVAL ;
And -ESRCH or some such here (please use existing code as reference).
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 762
Quoted 11 lines
xen/common/domain.c +11
-0+ touser_topo.nr_vnodes = d->vnuma.nr_vnodes; + max_pages = d->max_pages; + max_vcpus = d->max_vcpus; +rcu_unlock_domain (d); + + nr_vnodes = touser_topo.nr_vnodes; + rc =copy_to_guest (arg, &touser_topo,1 ); +if ( rc ) +return -EFAULT ; +if ( nr_vnodes ==0 || nr_vnodes > max_vcpus ) +return -EFAULT ;
-EINVAL
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765
Quoted 3 lines
xen/common/domain.c +3
-0+ vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes); +if ( vblks ==NULL ) +return -EFAULT ;
-ENOMEM
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 769
Quoted 4 lines
xen/common/domain.c +4
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (copy_to_guest_offset (mtopology.vnuma_memblks, i, + &d->vnuma.vnuma_memblks[i],1 ) <0 )
Indentation.
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 772
Quoted 3 lines
xen/common/domain.c +3
-0+ goto out; + } +for ( i =0 ; i < touser_topo.nr_vnodes; i++ )
Again, please fold this with the previous loop, and use (preferably) the simple variable as loop boundary (but in any case be consistent).
Quoted 6 lines
xen/common/domain.c +6
-0+for ( j =0 ; j < touser_topo.nr_vnodes; j++ ) + { +if (copy_to_guest_offset (mtopology.vdistance, +__vnode_distance_offset (d, i, j), + &__vnode_distance (d, i, j),1 ) ) + goto out;
xen/common/domain.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 788
Quoted 10 lines
xen/common/domain.c +10
-0+ } +for ( i =0 ; i < d->max_vcpus ; i++ ) + { +if (copy_to_guest_offset (mtopology.vcpu_to_vnode, i, + &d->vnuma.vcpu_to_vnode[i],1 ) ) + goto out; + } +return rc; +out: +if ( vblks )xfree (vblks);
Coding style.
Quoted 9 lines
xen/include/xen/vnuma.h new file +6
-0@@ -0,0 +1,27 @@ +#ifndef _VNUMA_H +#define _VNUMA_H +#include <public/memory.h> + +/* DEFINE_XEN_GUEST_HANDLE(vnuma_memblk_t); */ +
xen/include/xen/vnuma.h line 16
Quoted 10 lines
xen/include/xen/vnuma.h new file +10
-0+struct vnuma_topology_info { +domid_t domid; +uint16_t nr_vnodes; +uint32_t _pad; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; +typedef struct vnuma_topology_info vnuma_topology_info_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_topology_info_t);
At least up to here this appears to belong into the public header.
Jan
Quoted 14 lines
xen/include/xen/vnuma.h new file +6
-0+ +#define __vnode_distance_offset (_dom, _i, _j) \ + ( ((_j) * ((_dom)->vnuma.nr_vnodes)) + (_i) ) + +#define __vnode_distance (_dom, _i, _j) \ + ( (_dom)->vnuma.vdistance[__vnode_distance_offset((_dom), (_i),(_j))] )
xen/include/xen/vnuma.h new file +5
-0+ +#define __vnode_distance_set (_dom, _i, _j, _v) \ + do { __vnode_distance((_dom), (_i), (_j)) = (_v); } while(0) + +#endif Signature · 1 line
1.7.10.4
Quoted 1 line · Elena Ufimtseva
+void
domain_vnuma_destroy(struct domain_vnuma_info *v)
As long as you're respinning, I think I'd prefer this be named something different -- 'v' is almost universally of type "struct vcpu" in the Xen code, so this code is a little confusing. I might use "vinfo", or even just take a domain pointer and dereference it yourself.
Quoted 38 lines
xen/common/domain.c +20
-0+{ +if (v->vnuma_memblks) { +xfree (v->vnuma_memblks); + v->vnuma_memblks =NULL ; + } +if (v->vcpu_to_vnode) { +xfree (v->vcpu_to_vnode); + v->vcpu_to_vnode =NULL ; + } +if (v->vdistance) { +xfree (v->vdistance); + v->vdistance =NULL ; + } +if (v->vnode_to_pnode) { +xfree (v->vnode_to_pnode); + v->vnode_to_pnode =NULL ; + } + v->nr_vnodes =0 ; +} +/* * Local variables: * mode: C xen/common/domctl.c +1
-0@@ -29,6 +29,7 @@#include <asm/page.h> #include <public/domctl.h> #include <xsm/xsm.h> +#include <xen/vnuma.h> static DEFINE_SPINLOCK (domctl_lock);DEFINE_SPINLOCK (vcpu_alloc_lock);@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) ret =set_global_virq_handler (d, virq); }
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 873
Quoted 10 lines
xen/common/domctl.c +8
-1break ; - +case XEN_DOMCTL_setvnumainfo: + { +unsigned int i, j, nr_vnodes, dist_size; +unsigned int dist, vmap, vntop; +vnuma_memblk_t vmemblk; + + ret = -EINVAL ; + dist = i = j =0 ;
I think the zeroing of these is unnecessary -- you zero them / initialize them before you use them below.
You need to break this code into "paragraphs" of related changes, with a space in between. For example, maybe here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876
Quoted 3 lines
xen/common/domctl.c +3
-0+ nr_vnodes = op->u.vnuma.nr_vnodes; +if (nr_vnodes <=0 || nr_vnodes >NR_CPUS ) +break ;
here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 878
Quoted 2 lines
xen/common/domctl.c +2
-0+ d->vnuma.nr_vnodes = nr_vnodes; + dist_size = nr_vnodes * nr_vnodes;
here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888
Quoted 10 lines
xen/common/domctl.c +10
-0+if ( + (d->vnuma.vdistance =xmalloc_bytes ( +sizeof (*d->vnuma.vdistance) * dist_size)) ==NULL || + (d->vnuma.vnuma_memblks =xmalloc_bytes ( +sizeof (*d->vnuma.vnuma_memblks) * nr_vnodes)) ==NULL || + (d->vnuma.vcpu_to_vnode =xmalloc_bytes ( +sizeof (*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) ==NULL || + (d->vnuma.vnode_to_pnode =xmalloc_bytes ( +sizeof (*d->vnuma.vnode_to_pnode) * nr_vnodes)) ==NULL ) + goto err_dom;
here...
Also, for an idiomatic way to do this kind of multiple allocation w/ error checking, take a look at xen/arch/x86/hvm/hvm.c:hvm_domain_initialise() (to pick a random function I've worked with recently). Look at the
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 897
Quoted 9 lines
xen/common/domctl.c +9
-0+for ( i =0 ; i < nr_vnodes; i++ ) +for ( j =0 ; j < nr_vnodes; j++ ) + { +if (unlikely (__copy_from_guest_offset (&dist, + op->u.vnuma.vdistance, +__vnode_distance_offset (d, i, j),1 )) ) + goto err_dom; +__vnode_distance_set (d, i, j, dist); + }
here...
Also, in each of these error paths, you leak the structures allocated above.
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 905
Quoted 8 lines
xen/common/domctl.c +8
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmemblk, + op->u.vnuma.vnuma_memblks, i,1 )) ) + goto err_dom; + d->vnuma.vnuma_memblks[i].start = vmemblk.start; + d->vnuma.vnuma_memblks[i].end = vmemblk.end; + }
here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 912
Quoted 7 lines
xen/common/domctl.c +7
-0+for ( i =0 ; i < d->max_vcpus; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmap, + op->u.vnuma.vcpu_to_vnode, i,1 )) ) + goto err_dom; + d->vnuma.vcpu_to_vnode[i] = vmap; + }
here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925
Quoted 13 lines
xen/common/domctl.c +13
-0+if ( !guest_handle_is_null (op->u.vnuma.vnode_to_pnode) ) + { +for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vntop, + op->u.vnuma.vnode_to_pnode, i,1 )) ) + goto err_dom; + d->vnuma.vnode_to_pnode[i] = vntop; + } + } +else +for (i =0 ; i < nr_vnodes; i++) + d->vnuma.vnode_to_pnode[i] =NUMA_NO_NODE ;
here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 926
Quoted 1 line
xen/common/domctl.c +1
-0+ ret =0 ;
and maybe here.
Quoted 24 lines
xen/common/domctl.c +5
-0+break ; +err_dom: + ret = -EINVAL ; + } +break ;default : ret =arch_do_domctl (op, d, u_domctl);break ;xen/common/memory.c +1
-0@@ -28,6 +28,7 @@#include <public/memory.h> #include <xsm/xsm.h> #include <xen/trace.h> +#include <xen/vnuma.h> struct memop_args {/* INPUT */ @@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)rcu_unlock_domain (d);break ;
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 745
Quoted 10 lines
xen/common/memory.c +9
-0+case XENMEM_get_vnuma_info: + { +int i, j; +struct vnuma_topology_info mtopology; +struct vnuma_topology_info touser_topo; +struct domain *d; +unsigned int max_pages, max_vcpus, nr_vnodes; +vnuma_memblk_t *vblks; + rc = -EINVAL ;
here...
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 747
Quoted 2 lines
xen/common/memory.c +2
-0+if (guest_handle_is_null (arg) ) +return rc;
here...
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749
Quoted 2 lines
xen/common/memory.c +2
-0+if (copy_from_guest (&mtopology, arg,1 ) ) +return -EINVAL ;
here... (and so on)
This should be -EFAULT
Quoted 6 lines
xen/common/memory.c +6
-0+if ( (d =rcu_lock_domain_by_any_id (mtopology.domid)) ==NULL ) +return -EINVAL ; + touser_topo.nr_vnodes = d->vnuma.nr_vnodes; + max_pages = d->max_pages; + max_vcpus = d->max_vcpus; +rcu_unlock_domain (d);
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765
Quoted 10 lines
xen/common/memory.c +10
-0+ + nr_vnodes = touser_topo.nr_vnodes; + rc =copy_to_guest (arg, &touser_topo,1 ); +if ( rc ) +return -EFAULT ; +if ( nr_vnodes ==0 || nr_vnodes > max_vcpus ) +return -EFAULT ; + vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes); +if ( vblks ==NULL ) +return -EFAULT ;
Uum, what is this for? It doesn't appear to be used -- if there's a copy failure it's freed again, if things succeed it's leaked.
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 770
Quoted 5 lines
xen/common/memory.c +5
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (copy_to_guest_offset (mtopology.vnuma_memblks, i, + &d->vnuma.vnuma_memblks[i],1 ) <0 ) + goto out;
4 spaces for indentation, please.
Quoted 116 lines
xen/common/memory.c +21
-0+ } +for ( i =0 ; i < touser_topo.nr_vnodes; i++ ) +for ( j =0 ; j < touser_topo.nr_vnodes; j++ ) + { +if (copy_to_guest_offset (mtopology.vdistance, +__vnode_distance_offset (d, i, j), + &__vnode_distance (d, i, j),1 ) ) + goto out; + } +for ( i =0 ; i < d->max_vcpus ; i++ ) + { +if (copy_to_guest_offset (mtopology.vcpu_to_vnode, i, + &d->vnuma.vcpu_to_vnode[i],1 ) ) + goto out; + } +return rc; +out: +if ( vblks )xfree (vblks); +return rc; +break ; + }default : rc =arch_memory_op (op, arg);break ;xen/include/public/domctl.h +14
-1@@ -35,6 +35,7 @@#include "xen.h" #include "grant_table.h" #include "hvm/save.h" +#include "memory.h" #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t ;DEFINE_XEN_GUEST_HANDLE (xen_domctl_set_broken_page_p2m_t); +struct xen_domctl_vnuma { +uint16_t nr_vnodes; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (uint) vnode_to_pnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; + +typedef struct xen_domctl_vnuma xen_domctl_vnuma_t ; +DEFINE_XEN_GUEST_HANDLE (xen_domctl_vnuma_t); +struct xen_domctl {uint32_t cmd;#define XEN_DOMCTL_createdomain 1@@ -920,6 +932,7 @@ struct xen_domctl {#define XEN_DOMCTL_set_broken_page_p2m 67#define XEN_DOMCTL_setnodeaffinity 68#define XEN_DOMCTL_getnodeaffinity 69 +#define XEN_DOMCTL_setvnumainfo 70#define XEN_DOMCTL_gdbsx_guestmemio 1000#define XEN_DOMCTL_gdbsx_pausevcpu 1001#define XEN_DOMCTL_gdbsx_unpausevcpu 1002@@ -979,6 +992,7 @@ struct xen_domctl {struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; +struct xen_domctl_vnuma vnuma;uint8_t pad[128 ]; } u; };@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;DEFINE_XEN_GUEST_HANDLE (xen_domctl_t);#endif /* __XEN_PUBLIC_DOMCTL_H__ */ -/* * Local variables: * mode: C xen/include/public/memory.h +8
-1@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);* Caller must be privileged or the hypercall fails. */ #define XENMEM_claim_pages 24 -/* * XENMEM_claim_pages flags - the are no flags at this time. * The zero value is appropiate. */ +struct vnuma_memblk { +uint64_t start, end; +}; +typedef struct vnuma_memblk vnuma_memblk_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_memblk_t); + +#define XENMEM_get_vnuma_info 25 +#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */ #endif /* __XEN_PUBLIC_MEMORY_H__ */ xen/include/xen/domain.h +1
-0@@ -4,6 +4,7 @@#include <public/xen.h> #include <asm/domain.h> +#include <public/memory.h> typedef union { struct vcpu_guest_context *nat;
xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99
Quoted 10 lines
xen/include/xen/domain.h +7
-0@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;extern bool_t opt_dom0_vcpus_pin; +struct domain_vnuma_info { +uint16_t nr_vnodes; +uint *vdistance; +uint *vcpu_to_vnode; +uint *vnode_to_pnode; +vnuma_memblk_t *vnuma_memblks; +};
Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?
-George
Hi guys
Thank you for reviewing, I will clean these out.
George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.
Elena
Quoted 356 lines · George Dunlap
Quoted 1 line · Elena Ufimtseva
+void
domain_vnuma_destroy(struct domain_vnuma_info *v)As long as you're respinning, I think I'd prefer this be named something different -- 'v' is almost universally of type "struct vcpu" in the Xen code, so this code is a little confusing. I might use "vinfo", or even just take a domain pointer and dereference it yourself.
Quoted 38 lines
xen/common/domain.c +20
-0+{ +if (v->vnuma_memblks) { +xfree (v->vnuma_memblks); + v->vnuma_memblks =NULL ; + } +if (v->vcpu_to_vnode) { +xfree (v->vcpu_to_vnode); + v->vcpu_to_vnode =NULL ; + } +if (v->vdistance) { +xfree (v->vdistance); + v->vdistance =NULL ; + } +if (v->vnode_to_pnode) { +xfree (v->vnode_to_pnode); + v->vnode_to_pnode =NULL ; + } + v->nr_vnodes =0 ; +} +/* * Local variables: * mode: C xen/common/domctl.c +1
-0@@ -29,6 +29,7 @@#include <asm/page.h> #include <public/domctl.h> #include <xsm/xsm.h> +#include <xen/vnuma.h> static DEFINE_SPINLOCK (domctl_lock);DEFINE_SPINLOCK (vcpu_alloc_lock);@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) ret =set_global_virq_handler (d, virq); }xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 873
Quoted 10 lines
xen/common/domctl.c +8
-1break ; - +case XEN_DOMCTL_setvnumainfo: + { +unsigned int i, j, nr_vnodes, dist_size; +unsigned int dist, vmap, vntop; +vnuma_memblk_t vmemblk; + + ret = -EINVAL ; + dist = i = j =0 ;I think the zeroing of these is unnecessary -- you zero them / initialize them before you use them below.
You need to break this code into "paragraphs" of related changes, with a space in between. For example, maybe here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876
Quoted 3 lines
xen/common/domctl.c +3
-0+ nr_vnodes = op->u.vnuma.nr_vnodes; +if (nr_vnodes <=0 || nr_vnodes >NR_CPUS ) +break ;here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 878
Quoted 2 lines
xen/common/domctl.c +2
-0+ d->vnuma.nr_vnodes = nr_vnodes; + dist_size = nr_vnodes * nr_vnodes;here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888
Quoted 10 lines
xen/common/domctl.c +10
-0+if ( + (d->vnuma.vdistance =xmalloc_bytes ( +sizeof (*d->vnuma.vdistance) * dist_size)) ==NULL || + (d->vnuma.vnuma_memblks =xmalloc_bytes ( +sizeof (*d->vnuma.vnuma_memblks) * nr_vnodes)) ==NULL || + (d->vnuma.vcpu_to_vnode =xmalloc_bytes ( +sizeof (*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) ==NULL || + (d->vnuma.vnode_to_pnode =xmalloc_bytes ( +sizeof (*d->vnuma.vnode_to_pnode) * nr_vnodes)) ==NULL ) + goto err_dom;here...
Also, for an idiomatic way to do this kind of multiple allocation w/ error checking, take a look at xen/arch/x86/hvm/hvm.c:hvm_domain_initialise() (to pick a random function I've worked with recently). Look at the
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 897
Quoted 9 lines
xen/common/domctl.c +9
-0+for ( i =0 ; i < nr_vnodes; i++ ) +for ( j =0 ; j < nr_vnodes; j++ ) + { +if (unlikely (__copy_from_guest_offset (&dist, + op->u.vnuma.vdistance, +__vnode_distance_offset (d, i, j),1 )) ) + goto err_dom; +__vnode_distance_set (d, i, j, dist); + }here...
Also, in each of these error paths, you leak the structures allocated above.
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 905
Quoted 8 lines
xen/common/domctl.c +8
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmemblk, + op->u.vnuma.vnuma_memblks, i,1 )) ) + goto err_dom; + d->vnuma.vnuma_memblks[i].start = vmemblk.start; + d->vnuma.vnuma_memblks[i].end = vmemblk.end; + }here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 912
Quoted 7 lines
xen/common/domctl.c +7
-0+for ( i =0 ; i < d->max_vcpus; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmap, + op->u.vnuma.vcpu_to_vnode, i,1 )) ) + goto err_dom; + d->vnuma.vcpu_to_vnode[i] = vmap; + }here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925
Quoted 13 lines
xen/common/domctl.c +13
-0+if ( !guest_handle_is_null (op->u.vnuma.vnode_to_pnode) ) + { +for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vntop, + op->u.vnuma.vnode_to_pnode, i,1 )) ) + goto err_dom; + d->vnuma.vnode_to_pnode[i] = vntop; + } + } +else +for (i =0 ; i < nr_vnodes; i++) + d->vnuma.vnode_to_pnode[i] =NUMA_NO_NODE ;here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 926
Quoted 1 line
xen/common/domctl.c +1
-0+ ret =0 ;and maybe here.
Quoted 24 lines
xen/common/domctl.c +5
-0+break ; +err_dom: + ret = -EINVAL ; + } +break ;default : ret =arch_do_domctl (op, d, u_domctl);break ;xen/common/memory.c +1
-0@@ -28,6 +28,7 @@#include <public/memory.h> #include <xsm/xsm.h> #include <xen/trace.h> +#include <xen/vnuma.h> struct memop_args {/* INPUT */ @@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)rcu_unlock_domain (d);break ;xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 745
Quoted 10 lines
xen/common/memory.c +9
-0+case XENMEM_get_vnuma_info: + { +int i, j; +struct vnuma_topology_info mtopology; +struct vnuma_topology_info touser_topo; +struct domain *d; +unsigned int max_pages, max_vcpus, nr_vnodes; +vnuma_memblk_t *vblks; + rc = -EINVAL ;here...
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 747
Quoted 2 lines
xen/common/memory.c +2
-0+if (guest_handle_is_null (arg) ) +return rc;here...
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749
Quoted 2 lines
xen/common/memory.c +2
-0+if (copy_from_guest (&mtopology, arg,1 ) ) +return -EINVAL ;here... (and so on)
This should be
-EFAULTQuoted 6 lines
xen/common/memory.c +6
-0+if ( (d =rcu_lock_domain_by_any_id (mtopology.domid)) ==NULL ) +return -EINVAL ; + touser_topo.nr_vnodes = d->vnuma.nr_vnodes; + max_pages = d->max_pages; + max_vcpus = d->max_vcpus; +rcu_unlock_domain (d);xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765
Quoted 10 lines
xen/common/memory.c +10
-0+ + nr_vnodes = touser_topo.nr_vnodes; + rc =copy_to_guest (arg, &touser_topo,1 ); +if ( rc ) +return -EFAULT ; +if ( nr_vnodes ==0 || nr_vnodes > max_vcpus ) +return -EFAULT ; + vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes); +if ( vblks ==NULL ) +return -EFAULT ;Uum, what is this for? It doesn't appear to be used -- if there's a copy failure it's freed again, if things succeed it's leaked.
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 770
Quoted 5 lines
xen/common/memory.c +5
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (copy_to_guest_offset (mtopology.vnuma_memblks, i, + &d->vnuma.vnuma_memblks[i],1 ) <0 ) + goto out;4 spaces for indentation, please.
Quoted 116 lines
xen/common/memory.c +21
-0+ } +for ( i =0 ; i < touser_topo.nr_vnodes; i++ ) +for ( j =0 ; j < touser_topo.nr_vnodes; j++ ) + { +if (copy_to_guest_offset (mtopology.vdistance, +__vnode_distance_offset (d, i, j), + &__vnode_distance (d, i, j),1 ) ) + goto out; + } +for ( i =0 ; i < d->max_vcpus ; i++ ) + { +if (copy_to_guest_offset (mtopology.vcpu_to_vnode, i, + &d->vnuma.vcpu_to_vnode[i],1 ) ) + goto out; + } +return rc; +out: +if ( vblks )xfree (vblks); +return rc; +break ; + }default : rc =arch_memory_op (op, arg);break ;xen/include/public/domctl.h +14
-1@@ -35,6 +35,7 @@#include "xen.h" #include "grant_table.h" #include "hvm/save.h" +#include "memory.h" #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t ;DEFINE_XEN_GUEST_HANDLE (xen_domctl_set_broken_page_p2m_t); +struct xen_domctl_vnuma { +uint16_t nr_vnodes; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (uint) vnode_to_pnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; + +typedef struct xen_domctl_vnuma xen_domctl_vnuma_t ; +DEFINE_XEN_GUEST_HANDLE (xen_domctl_vnuma_t); +struct xen_domctl {uint32_t cmd;#define XEN_DOMCTL_createdomain 1@@ -920,6 +932,7 @@ struct xen_domctl {#define XEN_DOMCTL_set_broken_page_p2m 67#define XEN_DOMCTL_setnodeaffinity 68#define XEN_DOMCTL_getnodeaffinity 69 +#define XEN_DOMCTL_setvnumainfo 70#define XEN_DOMCTL_gdbsx_guestmemio 1000#define XEN_DOMCTL_gdbsx_pausevcpu 1001#define XEN_DOMCTL_gdbsx_unpausevcpu 1002@@ -979,6 +992,7 @@ struct xen_domctl {struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; +struct xen_domctl_vnuma vnuma;uint8_t pad[128 ]; } u; };@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;DEFINE_XEN_GUEST_HANDLE (xen_domctl_t);#endif /* __XEN_PUBLIC_DOMCTL_H__ */ -/* * Local variables: * mode: C xen/include/public/memory.h +8
-1@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);* Caller must be privileged or the hypercall fails. */ #define XENMEM_claim_pages 24 -/* * XENMEM_claim_pages flags - the are no flags at this time. * The zero value is appropiate. */ +struct vnuma_memblk { +uint64_t start, end; +}; +typedef struct vnuma_memblk vnuma_memblk_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_memblk_t); + +#define XENMEM_get_vnuma_info 25 +#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */ #endif /* __XEN_PUBLIC_MEMORY_H__ */ xen/include/xen/domain.h +1
-0@@ -4,6 +4,7 @@#include <public/xen.h> #include <asm/domain.h> +#include <public/memory.h> typedef union { struct vcpu_guest_context *nat;xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99
Quoted 10 lines
xen/include/xen/domain.h +7
-0@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;extern bool_t opt_dom0_vcpus_pin; +struct domain_vnuma_info { +uint16_t nr_vnodes; +uint *vdistance; +uint *vcpu_to_vnode; +uint *vnode_to_pnode; +vnuma_memblk_t *vnuma_memblks; +};Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?
-George
Signature · 1 line
Elena
Looks like I have.. Linux type for number of nodes is int, so Id rather keep it this way.
Quoted 373 lines · Elena Ufimtseva
Hi guys
Thank you for reviewing, I will clean these out.George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.
Elena
Quoted 355 lines · George Dunlap
Quoted 1 line · Elena Ufimtseva
+void
domain_vnuma_destroy(struct domain_vnuma_info *v)As long as you're respinning, I think I'd prefer this be named something different -- 'v' is almost universally of type "struct vcpu" in the Xen code, so this code is a little confusing. I might use "vinfo", or even just take a domain pointer and dereference it yourself.
Quoted 38 lines
xen/common/domain.c +20
-0+{ +if (v->vnuma_memblks) { +xfree (v->vnuma_memblks); + v->vnuma_memblks =NULL ; + } +if (v->vcpu_to_vnode) { +xfree (v->vcpu_to_vnode); + v->vcpu_to_vnode =NULL ; + } +if (v->vdistance) { +xfree (v->vdistance); + v->vdistance =NULL ; + } +if (v->vnode_to_pnode) { +xfree (v->vnode_to_pnode); + v->vnode_to_pnode =NULL ; + } + v->nr_vnodes =0 ; +} +/* * Local variables: * mode: C xen/common/domctl.c +1
-0@@ -29,6 +29,7 @@#include <asm/page.h> #include <public/domctl.h> #include <xsm/xsm.h> +#include <xen/vnuma.h> static DEFINE_SPINLOCK (domctl_lock);DEFINE_SPINLOCK (vcpu_alloc_lock);@@ -862,7 +863,72 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) ret =set_global_virq_handler (d, virq); }xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 873
Quoted 10 lines
xen/common/domctl.c +8
-1break ; - +case XEN_DOMCTL_setvnumainfo: + { +unsigned int i, j, nr_vnodes, dist_size; +unsigned int dist, vmap, vntop; +vnuma_memblk_t vmemblk; + + ret = -EINVAL ; + dist = i = j =0 ;I think the zeroing of these is unnecessary -- you zero them / initialize them before you use them below.
You need to break this code into "paragraphs" of related changes, with a space in between. For example, maybe here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 876
Quoted 3 lines
xen/common/domctl.c +3
-0+ nr_vnodes = op->u.vnuma.nr_vnodes; +if (nr_vnodes <=0 || nr_vnodes >NR_CPUS ) +break ;here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 878
Quoted 2 lines
xen/common/domctl.c +2
-0+ d->vnuma.nr_vnodes = nr_vnodes; + dist_size = nr_vnodes * nr_vnodes;here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 888
Quoted 10 lines
xen/common/domctl.c +10
-0+if ( + (d->vnuma.vdistance =xmalloc_bytes ( +sizeof (*d->vnuma.vdistance) * dist_size)) ==NULL || + (d->vnuma.vnuma_memblks =xmalloc_bytes ( +sizeof (*d->vnuma.vnuma_memblks) * nr_vnodes)) ==NULL || + (d->vnuma.vcpu_to_vnode =xmalloc_bytes ( +sizeof (*d->vnuma.vcpu_to_vnode) * d->max_vcpus)) ==NULL || + (d->vnuma.vnode_to_pnode =xmalloc_bytes ( +sizeof (*d->vnuma.vnode_to_pnode) * nr_vnodes)) ==NULL ) + goto err_dom;here...
Also, for an idiomatic way to do this kind of multiple allocation w/ error checking, take a look at xen/arch/x86/hvm/hvm.c:hvm_domain_initialise() (to pick a random function I've worked with recently). Look at the
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 897
Quoted 9 lines
xen/common/domctl.c +9
-0+for ( i =0 ; i < nr_vnodes; i++ ) +for ( j =0 ; j < nr_vnodes; j++ ) + { +if (unlikely (__copy_from_guest_offset (&dist, + op->u.vnuma.vdistance, +__vnode_distance_offset (d, i, j),1 )) ) + goto err_dom; +__vnode_distance_set (d, i, j, dist); + }here...
Also, in each of these error paths, you leak the structures allocated above.
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 905
Quoted 8 lines
xen/common/domctl.c +8
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmemblk, + op->u.vnuma.vnuma_memblks, i,1 )) ) + goto err_dom; + d->vnuma.vnuma_memblks[i].start = vmemblk.start; + d->vnuma.vnuma_memblks[i].end = vmemblk.end; + }here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 912
Quoted 7 lines
xen/common/domctl.c +7
-0+for ( i =0 ; i < d->max_vcpus; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vmap, + op->u.vnuma.vcpu_to_vnode, i,1 )) ) + goto err_dom; + d->vnuma.vcpu_to_vnode[i] = vmap; + }here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 925
Quoted 13 lines
xen/common/domctl.c +13
-0+if ( !guest_handle_is_null (op->u.vnuma.vnode_to_pnode) ) + { +for ( i =0 ; i < nr_vnodes; i++ ) + { +if (unlikely (__copy_from_guest_offset (&vntop, + op->u.vnuma.vnode_to_pnode, i,1 )) ) + goto err_dom; + d->vnuma.vnode_to_pnode[i] = vntop; + } + } +else +for (i =0 ; i < nr_vnodes; i++) + d->vnuma.vnode_to_pnode[i] =NUMA_NO_NODE ;here...
xen/common/domctl.c long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) · line 926
Quoted 1 line
xen/common/domctl.c +1
-0+ ret =0 ;and maybe here.
Quoted 24 lines
xen/common/domctl.c +5
-0+break ; +err_dom: + ret = -EINVAL ; + } +break ;default : ret =arch_do_domctl (op, d, u_domctl);break ;xen/common/memory.c +1
-0@@ -28,6 +28,7 @@#include <public/memory.h> #include <xsm/xsm.h> #include <xen/trace.h> +#include <xen/vnuma.h> struct memop_args {/* INPUT */ @@ -732,7 +733,62 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)rcu_unlock_domain (d);break ;xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 745
Quoted 10 lines
xen/common/memory.c +9
-0+case XENMEM_get_vnuma_info: + { +int i, j; +struct vnuma_topology_info mtopology; +struct vnuma_topology_info touser_topo; +struct domain *d; +unsigned int max_pages, max_vcpus, nr_vnodes; +vnuma_memblk_t *vblks; + rc = -EINVAL ;here...
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 747
Quoted 2 lines
xen/common/memory.c +2
-0+if (guest_handle_is_null (arg) ) +return rc;here...
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 749
Quoted 2 lines
xen/common/memory.c +2
-0+if (copy_from_guest (&mtopology, arg,1 ) ) +return -EINVAL ;here... (and so on)
This should be
-EFAULTQuoted 6 lines
xen/common/memory.c +6
-0+if ( (d =rcu_lock_domain_by_any_id (mtopology.domid)) ==NULL ) +return -EINVAL ; + touser_topo.nr_vnodes = d->vnuma.nr_vnodes; + max_pages = d->max_pages; + max_vcpus = d->max_vcpus; +rcu_unlock_domain (d);xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 765
Quoted 10 lines
xen/common/memory.c +10
-0+ + nr_vnodes = touser_topo.nr_vnodes; + rc =copy_to_guest (arg, &touser_topo,1 ); +if ( rc ) +return -EFAULT ; +if ( nr_vnodes ==0 || nr_vnodes > max_vcpus ) +return -EFAULT ; + vblks = xmalloc_array(struct vnuma_memblk, nr_vnodes); +if ( vblks ==NULL ) +return -EFAULT ;Uum, what is this for? It doesn't appear to be used -- if there's a copy failure it's freed again, if things succeed it's leaked.
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 770
Quoted 5 lines
xen/common/memory.c +5
-0+for ( i =0 ; i < nr_vnodes; i++ ) + { +if (copy_to_guest_offset (mtopology.vnuma_memblks, i, + &d->vnuma.vnuma_memblks[i],1 ) <0 ) + goto out;4 spaces for indentation, please.
Quoted 116 lines
xen/common/memory.c +21
-0+ } +for ( i =0 ; i < touser_topo.nr_vnodes; i++ ) +for ( j =0 ; j < touser_topo.nr_vnodes; j++ ) + { +if (copy_to_guest_offset (mtopology.vdistance, +__vnode_distance_offset (d, i, j), + &__vnode_distance (d, i, j),1 ) ) + goto out; + } +for ( i =0 ; i < d->max_vcpus ; i++ ) + { +if (copy_to_guest_offset (mtopology.vcpu_to_vnode, i, + &d->vnuma.vcpu_to_vnode[i],1 ) ) + goto out; + } +return rc; +out: +if ( vblks )xfree (vblks); +return rc; +break ; + }default : rc =arch_memory_op (op, arg);break ;xen/include/public/domctl.h +14
-1@@ -35,6 +35,7 @@#include "xen.h" #include "grant_table.h" #include "hvm/save.h" +#include "memory.h" #define XEN_DOMCTL_INTERFACE_VERSION 0x00000009@@ -852,6 +853,17 @@ struct xen_domctl_set_broken_page_p2m {typedef struct xen_domctl_set_broken_page_p2m xen_domctl_set_broken_page_p2m_t ;DEFINE_XEN_GUEST_HANDLE (xen_domctl_set_broken_page_p2m_t); +struct xen_domctl_vnuma { +uint16_t nr_vnodes; +XEN_GUEST_HANDLE_64 (uint) vdistance; +XEN_GUEST_HANDLE_64 (uint) vcpu_to_vnode; +XEN_GUEST_HANDLE_64 (uint) vnode_to_pnode; +XEN_GUEST_HANDLE_64 (vnuma_memblk_t) vnuma_memblks; +}; + +typedef struct xen_domctl_vnuma xen_domctl_vnuma_t ; +DEFINE_XEN_GUEST_HANDLE (xen_domctl_vnuma_t); +struct xen_domctl {uint32_t cmd;#define XEN_DOMCTL_createdomain 1@@ -920,6 +932,7 @@ struct xen_domctl {#define XEN_DOMCTL_set_broken_page_p2m 67#define XEN_DOMCTL_setnodeaffinity 68#define XEN_DOMCTL_getnodeaffinity 69 +#define XEN_DOMCTL_setvnumainfo 70#define XEN_DOMCTL_gdbsx_guestmemio 1000#define XEN_DOMCTL_gdbsx_pausevcpu 1001#define XEN_DOMCTL_gdbsx_unpausevcpu 1002@@ -979,6 +992,7 @@ struct xen_domctl {struct xen_domctl_set_broken_page_p2m set_broken_page_p2m;struct xen_domctl_gdbsx_pauseunp_vcpu gdbsx_pauseunp_vcpu;struct xen_domctl_gdbsx_domstatus gdbsx_domstatus; +struct xen_domctl_vnuma vnuma;uint8_t pad[128 ]; } u; };@@ -986,7 +1000,6 @@ typedef struct xen_domctl xen_domctl_t;DEFINE_XEN_GUEST_HANDLE (xen_domctl_t);#endif /* __XEN_PUBLIC_DOMCTL_H__ */ -/* * Local variables: * mode: C xen/include/public/memory.h +8
-1@@ -453,12 +453,19 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_sharing_op_t);* Caller must be privileged or the hypercall fails. */ #define XENMEM_claim_pages 24 -/* * XENMEM_claim_pages flags - the are no flags at this time. * The zero value is appropiate. */ +struct vnuma_memblk { +uint64_t start, end; +}; +typedef struct vnuma_memblk vnuma_memblk_t ; +DEFINE_XEN_GUEST_HANDLE (vnuma_memblk_t); + +#define XENMEM_get_vnuma_info 25 +#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */ #endif /* __XEN_PUBLIC_MEMORY_H__ */ xen/include/xen/domain.h +1
-0@@ -4,6 +4,7 @@#include <public/xen.h> #include <asm/domain.h> +#include <public/memory.h> typedef union { struct vcpu_guest_context *nat;xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99
Quoted 10 lines
xen/include/xen/domain.h +7
-0@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;extern bool_t opt_dom0_vcpus_pin; +struct domain_vnuma_info { +uint16_t nr_vnodes; +uint *vdistance; +uint *vcpu_to_vnode; +uint *vnode_to_pnode; +vnuma_memblk_t *vnuma_memblks; +};Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?
Quoted 1 line
-George
--
Elena
Signature · 1 line
Elena
Quoted 1 line
Quoted 1 line
Quoted 1 line · Elena Ufimtseva
Please don't top post.
Quoted 4 lines
George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.
With nr_vnodes being uint16_t, the vnode numbers should be too. Limiting them to u8 would possibly be even better, but then nr_vnodes would better be unsigned int (perhaps that was the case from the beginning, regardless of the types used for the arrays).
The pnode array surely can also be uint8_t for the time being, considering that there are other places where node IDs are limited to 8 bits.
And with struct acpi_table_slit having just 8-bit distances, there's no apparent reason why the virtual distances can't be 8 bits too.
But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.
Jan
Quoted 14 lines
Quoted 14 lines
xen/include/xen/domain.h extern unsigned int xen_processor_pmbits; · line 99
Quoted 11 lines
xen/include/xen/domain.h +7
-0@@ -89,4 +90,14 @@ extern unsigned int xen_processor_pmbits;extern bool_t opt_dom0_vcpus_pin; +struct domain_vnuma_info { +uint16_t nr_vnodes; +uint *vdistance; +uint *vcpu_to_vnode; +uint *vnode_to_pnode; +vnuma_memblk_t *vnuma_memblks; +};Are these the right sizes for things to allocate? Do we really need 32 bits to represent distance, vnode and pnode? Can we use u16 or u8?
Quoted 15 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · Elena Ufimtseva
Please don't top post.
Quoted 4 lines
George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.
With
nr_vnodesbeinguint16_t, the vnode numbers should be too. Limiting them to u8 would possibly be even better, but thennr_vnodeswould better be unsigned int (perhaps that was the case from the beginning, regardless of the types used for the arrays).
Quoted 4 lines
The pnode array surely can also be
uint8_tfor the time being, considering that there are other places where node IDs are limited to 8 bits.
All agreed.
Quoted 7 lines
And with struct
acpi_table_slithaving just 8-bit distances, there's no apparent reason why the virtual distances can't be 8 bits too.But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.
And, in fact, 'node_to_node_distance' in xen/include/public/sysctl.h
(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h)
is uint32.
Dario
Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere) ----------------------------------------------------------------- Dario Faggioli, Ph.D, http://about.me/dario.faggioli Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Quoted 25 lines · Dario Faggioli
Quoted 15 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · Elena Ufimtseva
Please don't top post.
Quoted 4 lines
George, after talking to Dario, I think the max number of physical nodes will not exceed 256. Dario's automatic NUMA placement work with this number and I think it can be easily u8. Unless anyone has other thoughts.
With
nr_vnodesbeinguint16_t, the vnode numbers should be too. Limiting them to u8 would possibly be even better, but thennr_vnodeswould better be unsigned int (perhaps that was the case from the beginning, regardless of the types used for the arrays).Quoted 4 lines
The pnode array surely can also be
uint8_tfor the time being, considering that there are other places where node IDs are limited to 8 bits.All agreed.
Quoted 3 lines
And with struct
acpi_table_slithaving just 8-bit distances, there's no apparent reason why the virtual distances can't be 8 bits too.
Quoted 8 lines
Quoted 4 lines
But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.
And, in fact, '
node_to_node_distance' inxen/include/public/sysctl.h
(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h)
is uint32.
Linux has u8 for distance. Ok, thank you for pointing that out.
Elena
Quoted 8 lines
Dario
--
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
Signature · 1 line
Elena
Quoted 7 lines · Elena Ufimtseva
Quoted 5 lines · Dario Faggioli
Quoted 4 lines · Jan Beulich
But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.
Quoted 7 lines
Quoted 4 lines
And, in fact, '
node_to_node_distance' inxen/include/public/sysctl.h
(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h)
is uint32.Linux has u8 for distance. Ok, thank you for pointing that out.
EhEh... So, very hard to be consistent with every actor in the play! :-)
Anyway, I think the point here is, as Jan was saying, to distinguish internal representation from interface. Within Xen, we should store everything in the smallest and nicest possible way (which, BTW, is what Linux does by using u8 for distances).
OTOH, when it comes to exported interfaces, we should be much more cautious, since changing the way Xen stores distances internally is trivial, changing the interface (either API or ABI) could be a nightmare!
In this case, what we (Xen) tell Linux is the interface, it is up to him (Linux) to convert that in a way that suits its own internals. In fact, despite Linux being the only one OS using this interface for now, it's not wise to design the interface itself specifically for Linux, since many OSes may want to support it in the future.
Regards,
Dario
Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere) ----------------------------------------------------------------- Dario Faggioli, Ph.D, http://about.me/dario.faggioli Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Hi Elena,
Thank you for your great work first :)
In your patch, guest can use a hypercall name: "XENMEM_get_vnuma_info" to get the structure below:
+struct vnuma_topology_info {
+ domid_t domid; + uint16_t nr_vnodes; + uint32_t _pad; + XEN_GUEST_HANDLE_64(uint) vdistance; + XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode; + XEN_GUEST_HANDLE_64(vnuma_
memblk_t) vnuma_memblks;
+};
I see that Xen have save a vnode_to_pnode[] array for mapping However, this hypercall won't give this mapping to Guest VM. Is that right?
Quoted 46 lines · Dario Faggioli
Quoted 9 lines · Elena Ufimtseva
Quoted 7 lines · Dario Faggioli
Quoted 4 lines · Jan Beulich
But - all this is only for the internal representations. Anything in the public interface should be wide enough to allow future extension.
And, in fact, '
node_to_node_distance' inxen/include/public/sysctl.h(http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/sysctl.h
)Quoted 5 lines
Quoted 2 lines
is uint32.
Linux has u8 for distance. Ok, thank you for pointing that out.
EhEh... So, very hard to be consistent with every actor in the play! :-)
Anyway, I think the point here is, as Jan was saying, to distinguish internal representation from interface. Within Xen, we should store everything in the smallest and nicest possible way (which, BTW, is what Linux does by using u8 for distances).
OTOH, when it comes to exported interfaces, we should be much more cautious, since changing the way Xen stores distances internally is trivial, changing the interface (either API or ABI) could be a nightmare!
In this case, what we (Xen) tell Linux is the interface, it is up to him (Linux) to convert that in a way that suits its own internals. In fact, despite Linux being the only one OS using this interface for now, it's not wise to design the interface itself specifically for Linux, since many OSes may want to support it in the future.
Regards,
Dario--
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
Signature · 8 lines
Yechen Li Team of System Virtualization and Cloud Computing School of Electronic Engineering and Computer Science, Peking University, China Nothing is impossible because impossible itself says: " I'm possible " lccycc From PKU
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Quoted 16 lines · Li Yechen
Hi Elena,
Thank you for your great work first :)In your patch, guest can use a hypercall name: "
XENMEM_get_vnuma_info" to get the structure below:(quoted diff) +7
-0+struct vnuma_topology_info { + domid_t domid; + uint16_t nr_vnodes; + uint32_t _pad; + XEN_GUEST_HANDLE_64(uint) vdistance; + XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode; + XEN_GUEST_HANDLE_64(vnuma_
memblk_t)vnuma_memblks;
+};
Quoted 2 lines
I see that Xen have save a vnode_to_pnode[] array for mapping
Yes it does. We spoke about it quite a bit, and Xen seems to be the best place where to store such info, considering how easy it is to keep it there, and that it may turn out useful in future (and ballooning is a lcear example of that).
Quoted 2 lines
However, this hypercall won't give this mapping to Guest VM.
Not with this hypercall, no. Actually, did most of the people, when reviewing your RFC patches, said that they prefer the mapping not exposed at all to the guest?
Dario
Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere) ----------------------------------------------------------------- Dario Faggioli, Ph.D, http://about.me/dario.faggioli Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Hello Li, Dario! :)
Quoted 25 lines · Dario Faggioli
Quoted 17 lines · Li Yechen
Hi Elena,
Thank you for your great work first :)In your patch, guest can use a hypercall name: "
XENMEM_get_vnuma_info" to get the structure below:(quoted diff) +7
-0+struct vnuma_topology_info { + domid_t domid; + uint16_t nr_vnodes; + uint32_t _pad; + XEN_GUEST_HANDLE_64(uint) vdistance; + XEN_GUEST_HANDLE_64(uint) vcpu_to_vnode; + XEN_GUEST_HANDLE_64(vnuma_
memblk_t)vnuma_memblks;
+};Quoted 2 lines
I see that Xen have save a vnode_to_pnode[] array for mapping
Yes it does. We spoke about it quite a bit, and Xen seems to be the best place where to store such info, considering how easy it is to keep it there, and that it may turn out useful in future (and ballooning is a lcear example of that).
Quoted 5 lines
Quoted 2 lines
However, this hypercall won't give this mapping to Guest VM.
Not with this hypercall, no. Actually, did most of the people, when reviewing your RFC patches, said that they prefer the mapping not exposed at all to the guest?
Yes Dario, correct. I think someone, possibly Konrad, was saying that it would be an interesting possibility.
But I dont think there was a change in this.
So fo Li I think I can provide hypercall that will return the mappings. Li, do you think it work for you?
Quoted 9 lines
Dario
--
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
Signature · 1 line
Elena