libxl: Extend CPU affinity syntax and enable for specifying it in config file.

Allow for "^<cpuid>" syntax while specifying the pCPUs list during a vcpu-pin. This enables doing the following:

xl vcpu-pin 1 1 0-4,^2

and achieving:

xl vcpu-list
Name ID VCPU CPU State Time(s) CPU Affinity
...
Squeeze_pv 1 1 3 -b- 2.4 0-1,3-4
...

Negative ranges are also supported, such as "0-4,^1-2" to mean "0,3-4"

Signed-off-by Dario Faggioli <[email protected]>

Enable CPU affinity specification in a VM's config file with the exact syntax `xl vcpu-pin' provides.

Signed-off-by Dario Faggioli <[email protected]>

diff -r e4fd5305381e tools/libxl/libxl.c

tools/libxl/libxl.c +15 -0

@@ -2663,6 +2663,21 @@ int libxl_set_vcpuaffinity(libxl_ctx *ct
     return 0;
 }
 
+int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid,
+                               unsigned int max_vcpus, libxl_cpumap *cpumap)
+{
+    int i, rc = 0;
+
+    for (i = 0; i < max_vcpus; i++) {
+        if (libxl_set_vcpuaffinity(ctx, domid, i, cpumap)) {
+            LIBXL__LOG(ctx, LIBXL__LOG_WARNING,
+                       "failed to set affinity for %d", i);
+            rc = ERROR_FAIL;
+        }
+    }
+    return rc;
+}
+
 int libxl_set_vcpuonline(libxl_ctx *ctx, uint32_t domid, libxl_cpumap *cpumap)
 {
     GC_INIT(ctx);

diff -r e4fd5305381e tools/libxl/libxl.h

tools/libxl/libxl.h +2 -0

@@ -560,6 +560,8 @@ libxl_vcpuinfo *libxl_list_vcpu(libxl_ct
                                        int *nb_vcpu, int *nrcpus);
 int libxl_set_vcpuaffinity(libxl_ctx *ctx, uint32_t domid, uint32_t vcpuid,
                            libxl_cpumap *cpumap);
+int libxl_set_vcpuaffinity_all(libxl_ctx *ctx, uint32_t domid,
+                               unsigned int max_vcpus, libxl_cpumap *cpumap);
 int libxl_set_vcpuonline(libxl_ctx *ctx, uint32_t domid, libxl_cpumap *cpumap);
 
 int libxl_get_sched_id(libxl_ctx *ctx);

diff -r e4fd5305381e tools/libxl/libxl_create.c

tools/libxl/libxl_create.c +3 -0

@@ -71,6 +71,9 @@ int libxl_init_build_info(libxl_ctx *ctx
     memset(b_info, '\0', sizeof(*b_info));
     b_info->max_vcpus = 1;
     b_info->cur_vcpus = 1;
+    if (libxl_cpumap_alloc(ctx, &b_info->cpumap))
+        return ERROR_NOMEM;
+    libxl_cpumap_set_any(&b_info->cpumap);
     b_info->max_memkb = 32 * 1024;
     b_info->target_memkb = b_info->max_memkb;
     b_info->disable_migrate = 0;

diff -r e4fd5305381e tools/libxl/libxl_dom.c

tools/libxl/libxl_dom.c +1 -0

@@ -65,6 +65,7 @@ int libxl__build_pre(libxl__gc *gc, uint
     libxl_ctx *ctx = libxl__gc_owner(gc);
     int tsc_mode;
     xc_domain_max_vcpus(ctx->xch, domid, info->max_vcpus);
+    libxl_set_vcpuaffinity_all(ctx, domid, info->max_vcpus, &info->cpumap);
     xc_domain_setmaxmem(ctx->xch, domid, info->target_memkb + LIBXL_MAXMEM_CONSTANT);
     if (info->type == LIBXL_DOMAIN_TYPE_PV)
         xc_domain_set_memmap_limit(ctx->xch, domid,

diff -r e4fd5305381e tools/libxl/libxl_types.idl

tools/libxl/libxl_types.idl +1 -0

@@ -162,6 +162,7 @@ libxl_domain_create_info = Struct("domai
 libxl_domain_build_info = Struct("domain_build_info",[
     ("max_vcpus",       integer),
     ("cur_vcpus",       integer),
+    ("cpumap",          libxl_cpumap),
     ("tsc_mode",        libxl_tsc_mode),
     ("max_memkb",       uint32),
     ("target_memkb",    uint32),

diff -r e4fd5305381e tools/libxl/libxl_utils.h

tools/libxl/libxl_utils.h +12 -0

@@ -70,6 +70,18 @@ int libxl_cpumap_alloc(libxl_ctx *ctx, l
 int libxl_cpumap_test(libxl_cpumap *cpumap, int cpu);
 void libxl_cpumap_set(libxl_cpumap *cpumap, int cpu);
 void libxl_cpumap_reset(libxl_cpumap *cpumap, int cpu);
+static inline void libxl_cpumap_set_any(libxl_cpumap *cpumap)
+{
+    memset(cpumap->map, -1, cpumap->size);
+}
+static inline void libxl_cpumap_set_none(libxl_cpumap *cpumap)
+{
+    memset(cpumap->map, 0, cpumap->size);
+}
+static inline int libxl_cpumap_cpu_valid(libxl_cpumap *cpumap, int cpu)
+{
+    return cpu >= 0 && cpu < (cpumap->size * 8);
+}
 #define libxl_for_each_cpu(var, map) for (var = 0; var < (map).size * 8; var++)
 #define libxl_for_each_set_cpu(v, m) for (v = 0; v < (m).size * 8; v++) \
                                              if (libxl_cpumap_test(&(m), v))

diff -r e4fd5305381e tools/libxl/xl_cmdimpl.c

tools/libxl/xl_cmdimpl.c +83 -60

@@ -569,6 +569,65 @@ static void split_string_into_string_lis
     free(s);
 }
 
+static int vcpupin_parse(char *cpu, libxl_cpumap *cpumap)
+{
+    libxl_cpumap exclude_cpumap;
+    uint32_t cpuida, cpuidb;
+    char *endptr, *toka, *tokb, *saveptr = NULL;
+    int i, rc = 0, rmcpu;
+
+    if (!strcmp(cpu, "all")) {
+        libxl_cpumap_set_any(cpumap);
+        return 0;
+    }
+
+    if (libxl_cpumap_alloc(ctx, &exclude_cpumap)) {
+        fprintf(stderr, "Error: Failed to allocate cpumap.\n");
+        return ENOMEM;
+    }
+
+    for (toka = strtok_r(cpu, ",", &saveptr); toka;
+         toka = strtok_r(NULL, ",", &saveptr)) {
+        rmcpu = 0;
+        if (*toka == '^') {
+            /* This (These) Cpu(s) will be removed from the map */
+            toka++;
+            rmcpu = 1;
+        }
+        /* Extract a valid (range of) cpu(s) */
+        cpuida = cpuidb = strtoul(toka, &endptr, 10);
+        if (endptr == toka) {
+            fprintf(stderr, "Error: Invalid argument.\n");
+            rc = EINVAL;
+            goto vcpp_out;
+        }
+        if (*endptr == '-') {
+            tokb = endptr + 1;
+            cpuidb = strtoul(tokb, &endptr, 10);
+            if (endptr == tokb || cpuida > cpuidb) {
+                fprintf(stderr, "Error: Invalid argument.\n");
+                rc = EINVAL;
+                goto vcpp_out;
+            }
+        }
+        while (cpuida <= cpuidb) {
+            rmcpu == 0 ? libxl_cpumap_set(cpumap, cpuida) :
+                         libxl_cpumap_set(&exclude_cpumap, cpuida);
+            cpuida++;
+        }
+    }
+
+    /* Clear all the cpus from the removal list */
+    libxl_for_each_set_cpu(i, exclude_cpumap) {
+        libxl_cpumap_reset(cpumap, i);
+    }
+
+vcpp_out:
+    libxl_cpumap_dispose(&exclude_cpumap);
+
+    return rc;
+}
+
 static void parse_config_data(const char *configfile_filename_report,
                               const char *configfile_data,
                               int configfile_len,

@@ -578,7 +637,7 @@ static void parse_config_data(const char
     const char *buf;
     long l;
     XLU_Config *config;
-    XLU_ConfigList *vbds, *nics, *pcis, *cvfbs, *cpuids;
+    XLU_ConfigList *cpus, *vbds, *nics, *pcis, *cvfbs, *cpuids;
     int pci_power_mgmt = 0;
     int pci_msitranslate = 1;
     int e;

@@ -661,6 +720,29 @@ static void parse_config_data(const char
     if (!xlu_cfg_get_long (config, "maxvcpus", &l, 0))
         b_info->max_vcpus = l;
 
+    if (!xlu_cfg_get_list (config, "cpus", &cpus, 0, 1)) {
+        int i, n_cpus = 0;
+
+        libxl_cpumap_set_none(&b_info->cpumap);
+        while ((buf = xlu_cfg_get_listitem(cpus, n_cpus)) != NULL) {
+            i = atoi(buf);
+            if (!libxl_cpumap_cpu_valid(&b_info->cpumap, i)) {
+                fprintf(stderr, "cpu %d illegal\n", i);
+                exit(1);
+            }
+            libxl_cpumap_set(&b_info->cpumap, i);
+            n_cpus++;
+        }
+    }
+    else if (!xlu_cfg_get_string (config, "cpus", &buf, 0)) {
+        char *buf2 = strdup(buf);
+
+        libxl_cpumap_set_none(&b_info->cpumap);
+        if (vcpupin_parse(buf2, &b_info->cpumap))
+            exit(1);
+        free(buf2);
+    }
+
     if (!xlu_cfg_get_long (config, "memory", &l, 0)) {
         b_info->max_memkb = l * 1024;
         b_info->target_memkb = b_info->max_memkb;

@@ -3503,65 +3585,6 @@ int main_vcpulist(int argc, char **argv)
     return 0;
 }
 
-static int vcpupin_parse(char *cpu, libxl_cpumap *cpumap)
-{
-    libxl_cpumap exclude_cpumap;
-    uint32_t cpuida, cpuidb;
-    char *endptr, *toka, *tokb, *saveptr = NULL;
-    int i, rc = 0, rmcpu;
-
-    if (!strcmp(cpu, "all")) {
-        memset(cpumap->map, -1, cpumap->size);
-        return 0;
-    }
-
-    if (libxl_cpumap_alloc(ctx, &exclude_cpumap)) {
-        fprintf(stderr, "Error: Failed to allocate cpumap.\n");
-        return ENOMEM;
-    }
-
-    for (toka = strtok_r(cpu, ",", &saveptr); toka;
-         toka = strtok_r(NULL, ",", &saveptr)) {
-        rmcpu = 0;
-        if (*toka == '^') {
-            /* This (These) Cpu(s) will be removed from the map */
-            toka++;
-            rmcpu = 1;
-        }
-        /* Extract a valid (range of) cpu(s) */
-        cpuida = cpuidb = strtoul(toka, &endptr, 10);
-        if (endptr == toka) {
-            fprintf(stderr, "Error: Invalid argument.\n");
-            rc = EINVAL;
-            goto vcpp_out;
-        }
-        if (*endptr == '-') {
-            tokb = endptr + 1;
-            cpuidb = strtoul(tokb, &endptr, 10);
-            if (endptr == tokb || cpuida > cpuidb) {
-                fprintf(stderr, "Error: Invalid argument.\n");
-                rc = EINVAL;
-                goto vcpp_out;
-            }
-        }
-        while (cpuida <= cpuidb) {
-            rmcpu == 0 ? libxl_cpumap_set(cpumap, cpuida) :
-                         libxl_cpumap_set(&exclude_cpumap, cpuida);
-            cpuida++;
-        }
-    }
-
-    /* Clear all the cpus from the removal list */
-    libxl_for_each_set_cpu(i, exclude_cpumap) {
-        libxl_cpumap_reset(cpumap, i);
-    }
-
-vcpp_out:
-    libxl_cpumap_dispose(&exclude_cpumap);
-
-    return rc;
-}
-
 static void vcpupin(const char *d, const char *vcpu, char *cpu)
 {
     libxl_vcpuinfo *vcpuinfo;

This series slightly extends the current support for specifying CPU affinity, basically adding the support for "^<cpuid>" kind of entries (i.e., "^6", meaning "not on CPU#6), and enables doing so in a VM's config file, like it (probably?) was possible with `xm'.