Automatically derive soft affinity from vnuma information
v2 cover Automatically derive soft affinity from vnuma information
Round 2. All patches have Wei's Ack.
Wei, I modified the comment of libxl__vnuma_config_check() and added your ack on patch 1, as you said during review.
This is available as a git branch here:
git://xenbits.xen.org/people/dariof/xen.git rel/numa/vnuma/softaffinity-v2
Regards,
Dario
---
Dario Faggioli (3):
libxl: check whether vcpu affinity and vnuma info match
libxl: automatically set soft affinity after vnuma info
libxl: cleanup some misuse of 'cpumap' as parameter
tools/libxl/libxl_dom.c | 46 ++++++++++++++++++++++++++++++++++- tools/libxl/libxl_utils.h | 6 ++--- tools/libxl/libxl_vnuma.c | 60 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-)
PATCH v2 1/3 libxl: check whether vcpu affinity and vnuma info match
More specifically, vcpus are assigned to a vnode, which in turn is associated with a pnode. If a vcpu also has, in its (hard or soft) affinity, some pcpus that are not part of the said pnode, print a warning to the user.
---
Changes from v1:
* updated the doc comment of libxl__vnuma_config_check(), as requested during review.
--- tools/libxl/libxl_vnuma.c | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+)
tools/libxl/libxl_vnuma.c +60 -0
@@ -33,11 +33,54 @@ static int compare_vmemrange(const void *a, const void *b)return 0 ; } +/* Check if a vcpu has an hard (or soft) affinity set in such +* a way that it does not match the pnode to which the vcpu itself +* is assigned to. +*/ +static int check_vnuma_affinity (libxl__gc *gc, +unsigned int vcpu, +unsigned int pnode, +unsigned int num_affinity, +const libxl_bitmap *affinity, +const char *kind) +{ +libxl_bitmap nodemap; +int rc =0 ; + +libxl_bitmap_init (&nodemap); + + rc =libxl_node_bitmap_alloc (CTX , &nodemap,0 ); +if (rc) { +LOG (ERROR ,"Can't allocate nodemap" ); + goto out; + } + + rc =libxl_cpumap_to_nodemap (CTX , affinity, &nodemap); +if (rc) { +LOG (ERROR ,"Can't convert Vcpu %d affinity to nodemap" , vcpu); + goto out; + } + +if (libxl_bitmap_count_set (&nodemap) !=1 || + !libxl_bitmap_test (&nodemap, pnode)) +LOG (WARN ,"Vcpu %d %s affinity and vnuma info mismatch" , vcpu, kind); + +out: +libxl_bitmap_dispose (&nodemap); +return rc; +} +/* Check if vNUMA configuration is valid: * 1. all pnodes inside vnode_to_pnode array are valid * 2. each vcpu belongs to one and only one vnode * 3. each vmemrange is valid and doesn't overlap with any other * 4. local distance cannot be larger than remote distance +* +* Check also, if any hard or soft affinity is specified, whether +* they match with the vNUMA related bits (namely vcpus to vnodes +* mappings and vnodes to pnodes association). If that does not +* hold, however, just print a warning, as that has "only" +* performance implications. */ int libxl__vnuma_config_check(libxl__gc *gc, const libxl_domain_build_info *b_info,
@@ -101,6 +144,23 @@ int libxl__vnuma_config_check(libxl__gc *gc, } } +/* Check whether vcpu affinity (if any) matches vnuma configuration */ +for (i =0 ; i < b_info->num_vnuma_nodes; i++) { + v = &b_info->vnuma_nodes[i]; +libxl_for_each_set_bit (j, v->vcpus) { +if (b_info->num_vcpu_hard_affinity > j) +check_vnuma_affinity (gc, j, v->pnode, + b_info->num_vcpu_hard_affinity, + &b_info->vcpu_hard_affinity[j], +"hard" ); +if (b_info->num_vcpu_soft_affinity > j) +check_vnuma_affinity (gc, j, v->pnode, + b_info->num_vcpu_soft_affinity, + &b_info->vcpu_soft_affinity[j], +"soft" ); + } + } +/* Check vmemranges */ qsort (state->vmemranges, state->num_vmemranges,sizeof (xen_vmemrange_t), compare_vmemrange);
PATCH v2 2/3 libxl: automatically set soft affinity after vnuma info
In reply to Dario FaggioliMore specifically, vcpus are assigned to a vnode, which in turn is associated with a pnode. If a vcpu does not have any soft affinity, automatically build up one, matching the pcpus of the said pnode.
--- tools/libxl/libxl_dom.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+)
tools/libxl/libxl_dom.c +44 -0
@@ -439,6 +439,44 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,return rc; } +static int set_vnuma_affinity (libxl__gc *gc,uint32_t domid, +libxl_domain_build_info *info) +{ +libxl_bitmap cpumap; +libxl_vnode_info *v; +unsigned int i, j; +int rc =0 ; + +libxl_bitmap_init (&cpumap); + + rc =libxl_cpu_bitmap_alloc (CTX , &cpumap,0 ); +if (rc) { +LOG (ERROR ,"Can't allocate nodemap" ); + goto out; + } + +/* +* For each vcpu in each vnode, set its soft affinity to +* the pcpus belonging to the pnode the vnode is on +*/ +for (i =0 ; i < info->num_vnuma_nodes; i++) { + v = &info->vnuma_nodes[i]; + + rc =libxl_node_to_cpumap (CTX , v->pnode, &cpumap); +if (rc) { +LOG (ERROR ,"Can't get cpumap for vnode %d" , i); + goto out; + } + +libxl_for_each_set_bit (j, v->vcpus) +libxl_set_vcpuaffinity (CTX , domid, j,NULL , &cpumap); + } + +out: +libxl_bitmap_dispose (&cpumap); +return rc; +} + int libxl__build_post(libxl__gc *gc, uint32_t domid, libxl_domain_build_info *info, libxl__domain_build_state *state,
@@ -450,6 +488,12 @@ int libxl__build_post(libxl__gc *gc, uint32_t domid,char **ents;int i, rc; +if (info->num_vnuma_nodes && !info->num_vcpu_soft_affinity) { + rc =set_vnuma_affinity (gc, domid, info); +if (rc) +return rc; + } + rc =libxl_domain_sched_params_set (CTX , domid, &info->sched_params);if (rc)return rc;
PATCH v2 3/3 libxl: cleanup some misuse of 'cpumap' as parameter
In reply to Dario Faggioliin favour of the more generic 'bitmap', which is better since these are generic libxl_bitmap_* functions.
Also fix a typo, and remove a stale (and wrong) comment.
No functional change intended.
--- tools/libxl/libxl_dom.c | 2 +- tools/libxl/libxl_utils.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-)
tools/libxl/libxl_dom.c +1 -1
@@ -376,7 +376,7 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid, }if (info->nodemap.size)libxl_domain_set_nodeaffinity (ctx, domid, &info->nodemap); -/* As mentioned in libxl.h, vcpu_hard_array takes precedence */ +if (info->num_vcpu_hard_affinity || info->num_vcpu_soft_affinity) {libxl_bitmap *hard_affinity, *soft_affinity;int i, n_vcpus;
tools/libxl/libxl_utils.h +3 -3
@@ -89,8 +89,8 @@ int libxl_bitmap_is_empty(const libxl_bitmap *bitmap);int libxl_bitmap_test (const libxl_bitmap *bitmap,int bit);void libxl_bitmap_set (libxl_bitmap *bitmap,int bit);void libxl_bitmap_reset (libxl_bitmap *bitmap,int bit); -int libxl_bitmap_count_set (const libxl_bitmap *cpumap); -char *libxl_bitmap_to_hex_string (libxl_ctx *ctx,const libxl_bitmap *cpumap); +int libxl_bitmap_count_set (const libxl_bitmap *bitmap); +char *libxl_bitmap_to_hex_string (libxl_ctx *ctx,const libxl_bitmap *bitmap);static inline void libxl_bitmap_set_any (libxl_bitmap *bitmap) {memset (bitmap->map,-1 , bitmap->size);
@@ -145,7 +145,7 @@ int libxl_node_to_cpumap(libxl_ctx *ctx, int node, libxl_bitmap *cpumap);/* Populate nodemap with the nodes of the cpus in cpumap */ int libxl_cpumap_to_nodemap (libxl_ctx *ctx, -const libxl_bitmap *cpuemap, +const libxl_bitmap *cpumap,libxl_bitmap *nodemap);static inline uint32_t libxl__sizekb_to_mb (uint32_t s) {
Ian Campbell
AppliedQuoted 1 line · Dario Faggioli
Round 2. All patches have Wei's Ack.
Applied.
Ian.