Skip to contentSkip to conversation

adjust the cpu-affinity to more than 64 cpus

PATCH In review James (song wei) 4 participants 14 messages Mar 17, 2010 – Mar 31, 2010 Arm/actions semi architecture

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 adjust the cpu-affinity to more than 64 cpus

Hi,
There are more than 64 cpus on new intel platform especially on NUMA
system, so that we need break the pcpu limit (that is 64) when set affinity of a VCPU.

-James (Song Wei)

tools/libxc/xc_domain.c +26 -11
diff -r c1f272c3a441 tools/libxc/xc_domain.c
@@ -98,23 +98,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
     DECLARE_DOMCTL;
     int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize); 
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
     domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
     domctl.domain = (domid_t)domid;
     domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
     set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
     
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
     {
         PERROR("Could not lock memory for Xen hypercall");
         goto out;
@@ -122,9 +127,10 @@
 
     ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
  out:
+    free(local);
     return ret;
 }
 
@@ -132,18 +138,26 @@
 int xc_vcpu_getaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap,
+                        int cpusize)
 {
     DECLARE_DOMCTL;
     int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
     domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
     domctl.domain = (domid_t)domid;
     domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
     set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
     
     if ( lock_pages(local, sizeof(local)) != 0 )
     {
@@ -154,8 +168,9 @@
     ret = do_domctl(xc_handle, &domctl);
 
     unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
     return ret;
 }
 
tools/libxc/xenctrl.h +4 -2
diff -r c1f272c3a441 tools/libxc/xenctrl.h
@@ -309,11 +309,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
  * This function will return information about one or more domains. It is
tools/python/xen/lowlevel/xc/xc.c +24 -9
diff -r c1f272c3a441 tools/python/xen/lowlevel/xc/xc.c
@@ -215,35 +215,54 @@
 {
     uint32_t dom;
     int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
     PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(cpumap); 
 
     static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
+    
 
     if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|iO", kwd_list, 
                                       &dom, &vcpu, &cpulist) )
         return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+  
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+    
+
     if ( (cpulist != NULL) && PyList_Check(cpulist) )
     {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
         for ( i = 0; i < PyList_Size(cpulist); i++ ) 
         {
             long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            *(cpumap + cpu / (cpumap_size * 8)) |= (uint64_t)1 << (cpu %
(cpumap_size * 8));
         }
     }

- if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+ if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
cpumap_size) != 0 )
+ {

+        free(cpumap);
         return pyxc_error_to_exception();
-    
+    }
+
     Py_INCREF(zero);
+    free(cpumap); 
     return zero;

}

@@ -362,7 +381,11 @@

     uint32_t dom, vcpu = 0;
     xc_vcpuinfo_t info;
     int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(cpumap);
     static char *kwd_list[] = { "domid", "vcpu", NULL };

@@ -370,12 +393,25 @@

                                       &dom, &vcpu) )
         return NULL;
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 ) 
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
     rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
     if ( rc < 0 )
         return pyxc_error_to_exception();

- rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+ size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+
+ if((cpumap = malloc(cpumap_size * size)) == NULL)
+ return pyxc_error_to_exception();
+
+ rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,

cpumap_size * size);
     if ( rc < 0 )
+    {
+        free(cpumap);
         return pyxc_error_to_exception();

+ }

     info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",
                               "online",   info.online,
@@ -385,17 +421,18 @@
                               "cpu",      info.cpu);
 
     cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < size * cpumap_size * 8; i++ )
     {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
             PyObject *pyint = PyInt_FromLong(i);
             PyList_Append(cpulist, pyint);
             Py_DECREF(pyint);
         }
-        cpumap >>= 1;
+        *(cpumap + i / (cpumap_size * 8)) >>= 1;
     }
     PyDict_SetItemString(info_dict, "cpumap", cpulist);
     Py_DECREF(cpulist);
+    free(cpumap);
     return info_dict;

}

http://old.nabble.com/file/p27928229/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

Signature · 2 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-affinity-to-more-than-64-cpus-tp27928229p27928229.html
Sent from the Xen - Dev mailing list archive at Nabble.com.
#1 lore raw
Quoted 4 lines
Quoted 1 line
Quoted 1 line

"James (song wei)" <[email protected]> 17.03.10 09:56 >>>

tools/python/xen/lowlevel/xc/xc.c line 223

Quoted 10 lines
tools/python/xen/lowlevel/xc/xc.c +5 -1
@@ -215,35 +215,54 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(cpumap); 

Perhaps sizeof(*cpumap)?

Quoted 2 lines

...
+ *(cpumap + cpu / (cpumap_size * 8)) |= (uint64_t)1 << (cpu %

(cpumap_size * 8));

Using [] here and in similar places further down would likely make these constructs a little bit more legible.

tools/python/xen/lowlevel/xc/xc.c line 388

Quoted 10 lines
tools/python/xen/lowlevel/xc/xc.c +5 -1
@@ -362,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(cpumap);

Same as above.

tools/python/xen/lowlevel/xc/xc.c line 424

Quoted 6 lines
tools/python/xen/lowlevel/xc/xc.c +1 -1
@@ -385,17 +421,18 @@
                              "cpu",      info.cpu);
 
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < size * cpumap_size * 8; i++ )

Why not simply use nr_cpus here?

Jan

#2 lore raw
Quoted 62 lines · Jan Beulich
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines

"James (song wei)" <[email protected]> 17.03.10 09:56 >>>

tools/python/xen/lowlevel/xc/xc.c line 223

Quoted 10 lines
tools/python/xen/lowlevel/xc/xc.c +5 -1
@@ -215,35 +215,54 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(cpumap); 

Perhaps sizeof(*cpumap)?

-- Yeah,you are right.

Quoted 2 lines

...
+ *(cpumap + cpu / (cpumap_size * 8)) |= (uint64_t)1 << (cpu %

(cpumap_size * 8));

Using [] here and in similar places further down would likely make these constructs a little bit more legible.
--yes.

tools/python/xen/lowlevel/xc/xc.c line 388

Quoted 10 lines
tools/python/xen/lowlevel/xc/xc.c +5 -1
@@ -362,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(cpumap);

Same as above.

tools/python/xen/lowlevel/xc/xc.c line 424

Quoted 6 lines
tools/python/xen/lowlevel/xc/xc.c +1 -1
@@ -385,17 +421,18 @@
                              "cpu",      info.cpu);
 
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < size * cpumap_size * 8; i++ )

Why not simply use nr_cpus here?
--Yes, copy amount of nr_cpus bitsare enough here.

Jan, thank you very much! I'll post the new patch here soon.

-Jame (Song Wei)

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Signature · 2 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-affinity-to-more-than-64-cpus-tp27928229p27941020.html
Sent from the Xen - Dev mailing list archive at Nabble.com.
#3 lore raw

Keir, could you take a look at this issue.
New Patch for this issue:

tools/libxc/xc_domain.c +25 -11
diff -r 8b269215464b tools/libxc/xc_domain.c
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
     DECLARE_DOMCTL;
     int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize); 
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
     domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
     domctl.domain = (domid_t)domid;
     domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
     set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
     
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
     {
         PERROR("Could not lock memory for Xen hypercall");
         goto out;
@@ -129,9 +134,10 @@
 
     ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
  out:
+    free(local);
     return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
     DECLARE_DOMCTL;
     int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
     domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
     domctl.domain = (domid_t)domid;
     domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
     set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
     
     if ( lock_pages(local, sizeof(local)) != 0 )
     {
@@ -161,8 +174,9 @@
     ret = do_domctl(xc_handle, &domctl);
 
     unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
     return ret;
 }
 
tools/libxc/xenctrl.h +4 -2
diff -r 8b269215464b tools/libxc/xenctrl.h
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                         uint32_t domid,
                         int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
  * This function will return information about one or more domains. It is
tools/python/xen/lowlevel/xc/xc.c +22 -9
diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c
@@ -217,8 +217,12 @@
 {
     uint32_t dom;
     int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
     PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap); 
 
     static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                       &dom, &vcpu, &cpulist) )
         return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+  
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
     if ( (cpulist != NULL) && PyList_Check(cpulist) )
     {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
         for ( i = 0; i < PyList_Size(cpulist); i++ ) 
         {
             long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
(cpumap_size * 8));
         }
     }

- if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+ if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
cpumap_size) != 0 )
+ {

+        free(cpumap);
         return pyxc_error_to_exception();
-    
+    }
     Py_INCREF(zero);
+    free(cpumap); 
     return zero;

}

@@ -365,7 +381,11 @@

     uint32_t dom, vcpu = 0;
     xc_vcpuinfo_t info;
     int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
     static char *kwd_list[] = { "domid", "vcpu", NULL };

@@ -373,12 +393,25 @@

                                       &dom, &vcpu) )
         return NULL;
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 ) 
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
     rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
     if ( rc < 0 )
         return pyxc_error_to_exception();

- rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+ size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+
+ if((cpumap = malloc(cpumap_size * size)) == NULL)
+ return pyxc_error_to_exception();
+
+ rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,

cpumap_size * size);
     if ( rc < 0 )
+    {
+        free(cpumap);
         return pyxc_error_to_exception();

+ }

     info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",
                               "online",   info.online,
@@ -386,19 +419,19 @@
                               "running",  info.running,
                               "cpu_time", info.cpu_time,
                               "cpu",      info.cpu);
-
     cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
     {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
             PyObject *pyint = PyInt_FromLong(i);
             PyList_Append(cpulist, pyint);
             Py_DECREF(pyint);
         }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
     }
     PyDict_SetItemString(info_dict, "cpumap", cpulist);
     Py_DECREF(cpulist);
+    free(cpumap);
     return info_dict;

}

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

Signature · 2 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.
#4 lore raw

Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     116.5 any cpu
Domain-0                             0     1     1   r--     116.5 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     117.0 any cpu
Domain-0                             0     1     1   r--     117.0 any cpu

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      60.8 any cpu
Domain-0                             0     1     1   -b-      42.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      61.6 0
Domain-0                             0     1     1   -b-      43.2 any cpu

Best regards,
Kan

Quoted 262 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.c +25 -11
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize); 
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");
        goto out;
@@ -129,9 +134,10 @@
 
    ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.h +4 -2
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
 * This function will return information about one or more domains. It is

diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c

tools/python/xen/lowlevel/xc/xc.c +27 -11
@@ -217,8 +217,12 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap); 
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )
        return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+  
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )
    {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ ) 
        {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
 (cpumap_size * 8));
        }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
 cpumap_size) != 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
-    
+    }
    Py_INCREF(zero);
+    free(cpumap); 
    return zero;

}

tools/python/xen/lowlevel/xc/xc.c +19 -2
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )
        return NULL;
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 ) 
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
    if ( rc < 0 )
        return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8); 
+
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception(); 
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
 cpumap_size * size);
    if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",

"online", info.online,

tools/python/xen/lowlevel/xc/xc.c +4 -4
@@ -386,19 +419,19 @@
                              "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu);
-
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);
    Py_DECREF(cpulist);
+    free(cpumap);
    return info_dict;
 }
 

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

Signature · 9 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.


_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
author
#5 lore raw

I update upstream code to c/s 20145. This patch works fine and I haven't meet this problem.

linux-2j72:/home # xm vcpu-pin 0 1 0
linux-2j72:/home # xm vcpu-list 0
Name ID VCPU CPU State Time(s) CPU Affinity

Domain-0                             0     0     0   ---      23.4 0
Domain-0                             0     1     0   r--      19.2 0 

B.T.W, there are something wrong when compile c/s 21046!

-James

Quoted 1 line
Quoted 1 line
Quoted 1 line

Masaki Kanno <[email protected]> 2010-3-19 11:14 >>>

Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     116.5 any cpu
Domain-0                             0     1     1   r--     116.5 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     117.0 any cpu
Domain-0                             0     1     1   r--     117.0 any cpu

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      60.8 any cpu
Domain-0                             0     1     1   -b-      42.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      61.6 0
Domain-0                             0     1     1   -b-      43.2 any cpu

Best regards,
Kan

Quoted 247 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +25 -11
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize); 
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");
        goto out;
@@ -129,9 +134,10 @@
 
    ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +4 -2
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
 * This function will return information about one or more domains. It is

diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +27 -11
@@ -217,8 +217,12 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap); 
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )
        return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+  
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )
    {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ ) 
        {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
 (cpumap_size * 8));
        }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
 cpumap_size) != 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
-    
+    }
    Py_INCREF(zero);
+    free(cpumap); 
    return zero;

}

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +19 -2
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )
        return NULL;
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 ) 
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
    if ( rc < 0 )
        return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8); 
+
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception(); 
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
 cpumap_size * size);
    if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",

"online", info.online,

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +4 -4
@@ -386,19 +419,19 @@
                              "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu);
-
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);
    Py_DECREF(cpulist);
+    free(cpumap);
    return info_dict;
 }
 
Quoted 15 lines

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

Signature · 9 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.


_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#6 lore raw

issue with c/s 21046 (was Re: [Patch] adjust the cpu-affinity to more than 64 cpus)

Quoted 2 lines
Quoted 1 line
Quoted 1 line

"James Song" <[email protected]> 19.03.10 10:09 >>>

B.T.W, there are something wrong when compile c/s 21046!

Mind stating what it is that is wrong with this?

Jan

#7 lore raw

Re: [Patch] adjust the cpu-affinity to more than64 cpus

In reply to James Song

Hi James,

I tried changeset 21045, but I still meet the problem.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      31.9 any cpu
Domain-0                             0     1     0   r--      31.9 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      32.7 any cpu
Domain-0                             0     1     0   r--      32.7 any cpu

FYI, I tried changeset 21045 on IPF(ia64) server too.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      17.8 any cpu
Domain-0                             0     1     0   r--      17.8 any cpu
Domain-0                             0     2     0   r--      17.8 any cpu
Domain-0                             0     3     0   r--      17.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      18.0 0
Domain-0                             0     1     0   r--      18.0 0
Domain-0                             0     2     0   r--      18.0 0
Domain-0                             0     3     0   r--      18.0 0

Best regards,
Kan

Quoted 321 lines · James Song

I update upstream code to c/s 20145. This patch works fine and I haven't meet this problem.

linux-2j72:/home # xm vcpu-pin 0 1 0
linux-2j72:/home # xm vcpu-list 0
Name ID VCPU CPU State Time(s) CPU
Affinity

Domain-0                             0     0     0   ---      23.4 0
Domain-0                             0     1     0   r--      19.2 0 

B.T.W, there are something wrong when compile c/s 21046!

-James

Quoted 1 line
Quoted 1 line
Quoted 1 line

Masaki Kanno <[email protected]> 2010-3-19 11:14 >>>

Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     1   r--     116.5 any cpu
Domain-0                             0     1     1   r--     116.5 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     1   r--     117.0 any cpu
Domain-0                             0     1     1   r--     117.0 any cpu

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     0   r--      60.8 any cpu
Domain-0                             0     1     1   -b-      42.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     0   r--      61.6 0
Domain-0                             0     1     1   -b-      43.2 any cpu

Best regards,
Kan

Quoted 262 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +25 -11
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize); 
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");
        goto out;
@@ -129,9 +134,10 @@
 
    ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +4 -2
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
 * This function will return information about one or more domains. It is

diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +27 -11
@@ -217,8 +217,12 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap); 
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )
        return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+  
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )
    {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ ) 
        {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
 (cpumap_size * 8));
        }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
 cpumap_size) != 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
-    
+    }
    Py_INCREF(zero);
+    free(cpumap); 
    return zero;

}

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +19 -2
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )
        return NULL;
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 ) 
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
    if ( rc < 0 )
        return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8); 
+
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception(); 
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
 cpumap_size * size);
    if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",

"online", info.online,

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +4 -4
@@ -386,19 +419,19 @@
                              "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu);
-
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);
    Py_DECREF(cpulist);
+    free(cpumap);
    return info_dict;
 }
 

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

Signature · 9 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.


_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
author
#8 lore raw

issue with c/s 21046 (was Re: [Patch] adjust the cpu-affinity to more than 64 cpus)

In reply to James Song

sorry, I forgot to post the error message, belowing is the error message:
helper2.c: In function a?cpu_x86_inita?:
helper2.c:142: error: a?shared_iopage_ta? has no member named a?vcpu_iodataa?
helper2.c: In function a?sp_infoa?:
helper2.c:219: error: a?shared_iopage_ta? has no member named a?vcpu_iodataa?
helper2.c:224: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 6 has type a?uint32_ta? helper2.c:224: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 7 has type a?uint32_ta? helper2.c:226: error: a?ioreq_ta? has no member named a?io_counta?
helper2.c: In function a?__cpu_get_ioreqa?:
helper2.c:235: error: a?shared_iopage_ta? has no member named a?vcpu_iodataa?
helper2.c:242: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 7 has type a?uint32_ta? helper2.c:242: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 8 has type a?uint32_ta? helper2.c: In function a?cpu_handle_ioreqa?:
helper2.c:517: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 7 has type a?uint32_ta? helper2.c:517: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 8 has type a?uint32_ta

Quoted 3 lines
Quoted 2 lines
Quoted 2 lines

Jan Beulich 2010-3-19 17:39 >>>
"James Song" <[email protected]> 19.03.10 10:09 >>>

B.T.W, there are something wrong when compile c/s 21046!

Mind stating what it is that is wrong with this?

Jan

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#9 lore raw

issue with c/s 21046 (was Re: [Patch] adjust the cpu-affinity to more than 64 cpus)

In reply to James Song

That's in the qemu tree, so it has nothing to do with c/s 21046. Jan

Quoted 1 line
Quoted 1 line
Quoted 1 line

James Song 19.03.10 11:47 >>>

sorry, I forgot to post the error message, belowing is the error message:
helper2.c: In function a?cpu_x86_inita?:
helper2.c:142: error: a?shared_iopage_ta? has no member named a?vcpu_iodataa?
helper2.c: In function a?sp_infoa?:
helper2.c:219: error: a?shared_iopage_ta? has no member named a?vcpu_iodataa?
helper2.c:224: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 6 has type a?uint32_ta? helper2.c:224: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 7 has type a?uint32_ta? helper2.c:226: error: a?ioreq_ta? has no member named a?io_counta?
helper2.c: In function a?__cpu_get_ioreqa?:
helper2.c:235: error: a?shared_iopage_ta? has no member named a?vcpu_iodataa?
helper2.c:242: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 7 has type a?uint32_ta? helper2.c:242: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 8 has type a?uint32_ta? helper2.c: In function a?cpu_handle_ioreqa?:
helper2.c:517: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 7 has type a?uint32_ta? helper2.c:517: warning: format a?%lxa? expects type a?long unsigned inta?, but argument 8 has type a?uint32_ta

Quoted 3 lines
Quoted 2 lines
Quoted 2 lines

Jan Beulich 2010-3-19 17:39 >>>
"James Song" <[email protected]> 19.03.10 10:09 >>>

B.T.W, there are something wrong when compile c/s 21046!

Mind stating what it is that is wrong with this?

Jan

#10 lore raw
In reply to James Song

James and Masaki,

The issue Masaki said does exist, it appear under ia32pae. A patch (c/s 21087) is to fix the bug.

Thanks,
Jinsong

________________________________
From: [email protected] [mailto:[email protected]] On Behalf Of James Song
Sent: Friday, March 19, 2010 5:10 PM
To: Masaki Kanno; [email protected]
Subject: Re: [Xen-devel] [Patch] adjust the cpu-affinity to more than 64 cpus

I update upstream code to c/s 20145. This patch works fine and I haven't meet this problem.

linux-2j72:/home # xm vcpu-pin 0 1 0
linux-2j72:/home # xm vcpu-list 0
Name ID VCPU CPU State Time(s) CPU Affinity

Domain-0                             0     0     0   ---      23.4 0
Domain-0                             0     1     0   r--      19.2 0

B.T.W, there are something wrong when compile c/s 21046!

-James

Quoted 1 line
Quoted 1 line
Quoted 1 line

Masaki Kanno <[email protected]> 2010-3-19 11:14 >>>

Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     116.5 any cpu
Domain-0                             0     1     1   r--     116.5 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     117.0 any cpu
Domain-0                             0     1     1   r--     117.0 any cpu

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      60.8 any cpu
Domain-0                             0     1     1   -b-      42.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      61.6 0
Domain-0                             0     1     1   -b-      43.2 any cpu

Best regards,
Kan

Quoted 258 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +25 -11
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize);
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");
        goto out;
@@ -129,9 +134,10 @@
 
    ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +4 -2
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
 * This function will return information about one or more domains. It is

diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +27 -11
@@ -217,8 +217,12 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info;
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )
        return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )
    {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ )
        {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
 (cpumap_size * 8));
        }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
 cpumap_size) != 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
-
+    }
    Py_INCREF(zero);
+    free(cpumap);
    return zero;

}

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +19 -2
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )
        return NULL;
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 )
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
    if ( rc < 0 )
        return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception();
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
 cpumap_size * size);
    if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",

"online", info.online,

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +4 -4
@@ -386,19 +419,19 @@
                              "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu);
-
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);
    Py_DECREF(cpulist);
+    free(cpumap);
    return info_dict;
 }
 

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

--
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.

Quoted 4 lines

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

author
#11 lore raw

Yeah, perhaps, I haven't tested this patch on IA32PAE box. Thanks for fixed it.

-Song Wei (James)

Quoted 1 line
Quoted 1 line
Quoted 1 line

"Liu, Jinsong" <[email protected]> 2010-3-31 2:23 >>>

James and Masaki,

The issue Masaki said does exist, it appear under ia32pae. A patch (c/s 21087) is to fix the bug.

Thanks,
Jinsong

From: [email protected] [mailto:[email protected]] On Behalf Of James Song
Sent: Friday, March 19, 2010 5:10 PM
To: Masaki Kanno; [email protected]
Subject: Re: [Xen-devel] [Patch] adjust the cpu-affinity to more than 64 cpus

I update upstream code to c/s 20145. This patch works fine and I haven't meet this problem.

linux-2j72:/home # xm vcpu-pin 0 1 0
linux-2j72:/home # xm vcpu-list 0
Name ID VCPU CPU State Time(s) CPU Affinity

Domain-0                             0     0     0   ---      23.4 0
Domain-0                             0     1     0   r--      19.2 0 

B.T.W, there are something wrong when compile c/s 21046!

-James

Quoted 1 line
Quoted 1 line
Quoted 1 line

Masaki Kanno <[email protected]> 2010-3-19 11:14 >>>

Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     116.5 any cpu
Domain-0                             0     1     1   r--     116.5 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     1   r--     117.0 any cpu
Domain-0                             0     1     1   r--     117.0 any cpu

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      60.8 any cpu
Domain-0                             0     1     1   -b-      42.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
Domain-0                             0     0     0   r--      61.6 0
Domain-0                             0     1     1   -b-      43.2 any cpu

Best regards,
Kan

Quoted 247 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +25 -11
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize); 
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");
        goto out;
@@ -129,9 +134,10 @@
 
    ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +4 -2
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
 * This function will return information about one or more domains. It is

diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +27 -11
@@ -217,8 +217,12 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info; 
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap); 
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )
        return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+  
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )
    {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ ) 
        {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
 (cpumap_size * 8));
        }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
 cpumap_size) != 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
-    
+    }
    Py_INCREF(zero);
+    free(cpumap); 
    return zero;

}

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +19 -2
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )
        return NULL;
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 ) 
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
    if ( rc < 0 )
        return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8); 
+
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception(); 
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
 cpumap_size * size);
    if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",

"online", info.online,

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +4 -4
@@ -386,19 +419,19 @@
                              "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu);
-
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);
    Py_DECREF(cpulist);
+    free(cpumap);
    return info_dict;
 }
 
Quoted 15 lines

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

Signature · 9 lines
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.


_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#12 lore raw
In reply to Liu, Jinsong

Hi Jinsong,

Thank you for fixing the issue.
I confirmed that your patch works on both ia32pae and ia64.

ia32pae:

 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
 Domain-0                             0     0     1   r--      33.9 any cpu
 Domain-0                             0     1     0   -b-      22.5 any cpu
 # xm vcpu-pin Domain-0 0 0
 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
 Domain-0                             0     0     0   -b-      34.4 0
 Domain-0                             0     1     1   r--      23.2 any cpu

ia64:

 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
 Domain-0                             0     0     0   r--      23.7 any cpu
 Domain-0                             0     1     1   -b-      10.6 any cpu
 Domain-0                             0     2     2   -b-       6.8 any cpu
 Domain-0                             0     3     3   -b-       9.7 any cpu
 # xm vcpu-pin Domain-0 0 0
 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s) CPU Affinity
 Domain-0                             0     0     0   -b-      24.0 0
 Domain-0                             0     1     1   -b-      10.8 any cpu
 Domain-0                             0     2     2   r--       6.8 any cpu
 Domain-0                             0     3     3   -b-       9.8 any cpu

Best regards,
Kan

Quoted 343 lines · Jinsong

James and Masaki,

The issue Masaki said does exist, it appear under ia32pae. A patch (c/s 21087) is to fix the bug.

Thanks,
Jinsong

________________________________
From: [email protected] [mailto:xen-devel-bounces@lists .xensource.com] On Behalf Of James Song
Sent: Friday, March 19, 2010 5:10 PM
To: Masaki Kanno; [email protected]
Subject: Re: [Xen-devel] [Patch] adjust the cpu-affinity to more than 64 cpus

I update upstream code to c/s 20145. This patch works fine and I haven't meet this problem.

linux-2j72:/home # xm vcpu-pin 0 1 0
linux-2j72:/home # xm vcpu-list 0
Name ID VCPU CPU State Time(s) CPU
Affinity

Domain-0                             0     0     0   ---      23.4 0
Domain-0                             0     1     0   r--      19.2 0

B.T.W, there are something wrong when compile c/s 21046!

-James

Quoted 1 line
Quoted 1 line
Quoted 1 line

Masaki Kanno <[email protected]> 2010-3-19 11:14 >>>

Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     1   r--     116.5 any cpu
Domain-0                             0     1     1   r--     116.5 any cpu
# xm vcpu-pin Domain-0 0 0
Error: Cannot pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument')
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     1   r--     117.0 any cpu
Domain-0                             0     1     1   r--     117.0 any cpu

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     0   r--      60.8 any cpu
Domain-0                             0     1     1   -b-      42.8 any cpu
# xm vcpu-pin Domain-0 0 0
# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s) CPU 
Affinity
Domain-0                             0     0     0   r--      61.6 0
Domain-0                             0     1     1   -b-      43.2 any cpu

Best regards,
Kan

Quoted 258 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +25 -11
@@ -105,23 +105,28 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize);
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
 
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
 
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");
        goto out;
@@ -129,9 +134,10 @@
 
    ret = do_domctl(xc_handle, &domctl);
 
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;
 }
 
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize)
 {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall");
+        goto out;
+    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap, local);
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8);
+out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +4 -2
@@ -310,11 +310,13 @@
 int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);
 
 /**
 * This function will return information about one or more domains. It is

diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +27 -11
@@ -217,8 +217,12 @@
 {
    uint32_t dom;
    int vcpu = 0, i;
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info;
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
 
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )
        return NULL;
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )
    {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ )
        {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu %
 (cpumap_size * 8));
        }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap) != 0 )
+    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap, size *
 cpumap_size) != 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
-
+    }
    Py_INCREF(zero);
+    free(cpumap);
    return zero;

}

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +19 -2
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )
        return NULL;
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 )
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);
    if ( rc < 0 )
        return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception();
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
 cpumap_size * size);
    if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",

"online", info.online,

tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55 2010 +0800 renamed from tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +4 -4
@@ -386,19 +419,19 @@
                              "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu);
-
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);
    Py_DECREF(cpulist);
+    free(cpumap);
    return info_dict;
 }
 

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

--
View this message in context: http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html
Sent from the Xen - Dev mailing list archive at Nabble.com.

Quoted 4 lines

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

-------------------------------text/plain-------------------------------
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#13 lore raw

Yeah, now cpu affinity and pin are fine under all IA platform, with no 64 limit :) Thanks James for original patch, and thanks Masaki confirm it!

Jinsong

Quoted 363 lines · Masaki Kanno

Hi Jinsong,

Thank you for fixing the issue.
I confirmed that your patch works on both ia32pae and ia64.

ia32pae:

 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s)
 CPU Affinity Domain-0                             0     0     1  
 r--      33.9 any cpu Domain-0                             0     1  
 0   -b-      22.5 any cpu # xm vcpu-pin Domain-0 0 0
 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s)
 CPU Affinity Domain-0                             0     0     0  
 -b-      34.4 0 Domain-0                             0     1     1  
r--      23.2 any cpu 

ia64:

 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s)
 CPU Affinity Domain-0                             0     0     0  
 r--      23.7 any cpu Domain-0                             0     1  
 1   -b-      10.6 any cpu Domain-0                             0    
 2     2   -b-       6.8 any cpu Domain-0                            
 0     3     3   -b-       9.7 any cpu # xm vcpu-pin Domain-0 0 0
 # xm vcpu-list
 Name                                ID  VCPU   CPU State   Time(s)
 CPU Affinity Domain-0                             0     0     0  
 -b-      24.0 0 Domain-0                             0     1     1  
 -b-      10.8 any cpu Domain-0                             0     2  
 2   r--       6.8 any cpu Domain-0                             0    
3     3   -b-       9.8 any cpu 

Best regards,
Kan

Quoted 325 lines · Jinsong

James and Masaki,

The issue Masaki said does exist, it appear under ia32pae. A patch (c/s 21087) is to fix the bug.

Thanks,
Jinsong

________________________________
From: [email protected] [mailto:xen-devel-bounces@lists .xensource.com] On Behalf Of James Song
Sent: Friday, March 19, 2010 5:10 PM
To: Masaki Kanno; [email protected]
Subject: Re: [Xen-devel] [Patch] adjust the cpu-affinity to more than 64 cpus

I update upstream code to c/s 20145. This patch works fine and I haven't meet this problem.

linux-2j72:/home # xm vcpu-pin 0 1 0
linux-2j72:/home # xm vcpu-list 0
Name ID VCPU CPU State Time(s)
CPU Affinity Domain-0 0 0 0
--- 23.4 0 Domain-0 0 1 0
r-- 19.2 0

B.T.W, there are something wrong when compile c/s 21046!

-James

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

Masaki Kanno <[email protected]> 2010-3-19 11:14 >>> Hi James,

I tested xm vcpu-pin command with xen-unstable changeset 21044. The command failed as follows. Also Time(s) values by xm vcpu-list command were strange. The values were same in all VCPUs.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s)
CPU Affinity Domain-0                             0     0     1  
r--     116.5 any cpu Domain-0                             0     1  
1   r--     116.5 any cpu # xm vcpu-pin Domain-0 0 0 Error: Cannot
pin vcpu: 0 to cpu: [0] - (22, 'Invalid argument') 
Usage: xm vcpu-pin <Domain> <VCPU|all> <CPUs|all>

Set which CPUs a VCPU can use.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s)
CPU Affinity Domain-0                             0     0     1  
r--     117.0 any cpu Domain-0                             0     1  
1   r--     117.0 any cpu 

I reverted changeset 21044 and 21040, and then I tested xm vcpu-pin command again. The command succeeded as follows.

# xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s)
CPU Affinity Domain-0                             0     0     0  
r--      60.8 any cpu Domain-0                             0     1  
1   -b-      42.8 any cpu # xm vcpu-pin Domain-0 0 0 # xm vcpu-list
Name                                ID  VCPU   CPU State   Time(s)
CPU Affinity Domain-0                             0     0     0  
r--      61.6 0 Domain-0                             0     1     1  
-b-      43.2 any cpu 

Best regards,
Kan

Quoted 241 lines · James (song wei)

Keir, could you take a look at this issue.
New Patch for this issue:

diff -r 8b269215464b tools/libxc/xc_domain.c

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +5 -3
-105,23 +105,28 @@ int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap)
+                        uint64_t *cpumap, int cpusize) {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t *local = malloc(cpusize);
 
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall"); +     

goto out; + }

    domctl.cmd = XEN_DOMCTL_setvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu    = vcpu;
tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +1 -1
-    bitmap_64_to_byte(local, &cpumap, sizeof(cpumap) * 8);
+    bitmap_64_to_byte(local, cpumap, cpusize * 8);
 
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap,

local);

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +2 -2
-    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap) * 8;
+    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
-    if ( lock_pages(local, sizeof(local)) != 0 )
+    if ( lock_pages(local, cpusize) != 0 )
    {
        PERROR("Could not lock memory for Xen hypercall");        

goto out; @@ -129,9 +134,10 @@

    ret = do_domctl(xc_handle, &domctl);
tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +2 -1
-    unlock_pages(local, sizeof(local));
+    unlock_pages(local, cpusize);
 
 out:
+    free(local);
    return ret;

}

tools/libxc/xc_domain.cThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xc_domain.cWed Mar 10 14:01:32 2010 +0800 +9 -4
@@ -139,18 +145,25 @@
 int xc_vcpu_getaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t *cpumap)
+                        uint64_t *cpumap, int cpusize) {
    DECLARE_DOMCTL;
    int ret = -1;
-    uint8_t local[sizeof (cpumap)];
+    uint8_t * local = malloc(cpusize);
+
+    if(local == NULL)
+    {
+        PERROR("Could not alloc memory for Xen hypercall"); +     
 goto out; +    }
 
    domctl.cmd = XEN_DOMCTL_getvcpuaffinity;
    domctl.domain = (domid_t)domid;
    domctl.u.vcpuaffinity.vcpu = vcpu;
 
+
    set_xen_guest_handle(domctl.u.vcpuaffinity.cpumap.bitmap,
 local); -    domctl.u.vcpuaffinity.cpumap.nr_cpus = sizeof(cpumap)
 * 8; +    domctl.u.vcpuaffinity.cpumap.nr_cpus = cpusize * 8;
 
    if ( lock_pages(local, sizeof(local)) != 0 )
    {
@@ -161,8 +174,9 @@
    ret = do_domctl(xc_handle, &domctl);
 
    unlock_pages(local, sizeof (local));
-    bitmap_byte_to_64(cpumap, local, sizeof(local) * 8);
- out:
+    bitmap_byte_to_64(cpumap, local, cpusize * 8); +out:
+    free(local);
    return ret;
 }
 

diff -r 8b269215464b tools/libxc/xenctrl.h

tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +2 -2
-310,11 +310,13 @@ int xc_vcpu_setaffinity(int xc_handle,
                        uint32_t domid,
                        int vcpu,
-                        uint64_t cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);

int xc_vcpu_getaffinity(int xc_handle,

                        uint32_t domid,
                        int vcpu,
tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +2 -1
-                        uint64_t *cpumap);
+                        uint64_t *cpumap,
+                        int cpusize);

/**
* This function will return information about one or more domains.
It is diff -r 8b269215464b tools/python/xen/lowlevel/xc/xc.c --- a/tools/python/xen/lowlevel/xc/xc.cWed Mar 10 14:01:32 2010 +0800 +++ b/tools/python/xen/lowlevel/xc/xc.cThu Mar 18 11:37:55

2010 +0800 @@ -217,8 +217,12 @@ {
    uint32_t dom;
    int vcpu = 0, i;
tools/libxc/xenctrl.hThu Mar 18 11:37:55 2010 +0800 @@ renamed from tools/libxc/xenctrl.hWed Mar 10 14:01:32 2010 +0800 +46 -16
-    uint64_t  cpumap = ~0ULL;
+    uint64_t  *cpumap;
    PyObject *cpulist = NULL;
+    int nr_cpus, size;
+    xc_physinfo_t info;
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", "cpumap", NULL };
@@ -226,26 +230,38 @@
                                      &dom, &vcpu, &cpulist) )     
 return NULL; 
 
+    set_xen_guest_handle(info.cpu_to_node, map);
+    info.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &info) != 0 )
+        return pyxc_error_to_exception();
+
+    nr_cpus = info.nr_cpus;
+
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8);
+    cpumap = malloc(cpumap_size * size);
+    if(cpumap == NULL)
+        return pyxc_error_to_exception();
+
    if ( (cpulist != NULL) && PyList_Check(cpulist) )     {
-        cpumap = 0ULL;
+        for ( i = 0; i < size; i++)
+        {
+            cpumap[i] = 0ULL;
+        }
        for ( i = 0; i < PyList_Size(cpulist); i++ )         {
            long cpu = PyInt_AsLong(PyList_GetItem(cpulist, i));
-            if ( cpu >= 64 )
-            {
-                errno = EINVAL;
-                PyErr_SetFromErrno(xc_error_obj);
-                return NULL;
-            }
-            cpumap |= (uint64_t)1 << cpu;
+            cpumap[cpu / (cpumap_size * 8)] |= (uint64_t)1 << (cpu
        % (cpumap_size * 8)); }
    }
 
-    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu, cpumap)
 != 0 ) +    if ( xc_vcpu_setaffinity(self->xc_handle, dom, vcpu,
 cpumap, size * cpumap_size) != 0 ) +    {
+        free(cpumap);
        return pyxc_error_to_exception();
-
+    }
    Py_INCREF(zero);
+    free(cpumap);
    return zero;
 }
 
@@ -365,7 +381,11 @@
    uint32_t dom, vcpu = 0;
    xc_vcpuinfo_t info;
    int rc, i;
-    uint64_t cpumap;
+    uint64_t *cpumap;
+    int nr_cpus, size;
+    xc_physinfo_t pinfo = { 0 };
+    xc_cpu_to_node_t map[1];
+    uint64_t cpumap_size = sizeof(*cpumap);
 
    static char *kwd_list[] = { "domid", "vcpu", NULL };
 
@@ -373,12 +393,25 @@
                                      &dom, &vcpu) )         return
 NULL; 
 
+    set_xen_guest_handle(pinfo.cpu_to_node, map);
+    pinfo.max_cpu_id = 1;
+    if ( xc_physinfo(self->xc_handle, &pinfo) != 0 )
+        return pyxc_error_to_exception();
+    nr_cpus = pinfo.nr_cpus;
    rc = xc_vcpu_getinfo(self->xc_handle, dom, vcpu, &info);     if
        ( rc < 0 ) return pyxc_error_to_exception();
-    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, &cpumap);
+    size = (nr_cpus + cpumap_size * 8 - 1)/ (cpumap_size * 8); +
+    if((cpumap = malloc(cpumap_size * size)) == NULL)
+        return pyxc_error_to_exception();
+
+    rc = xc_vcpu_getaffinity(self->xc_handle, dom, vcpu, cpumap,
    cpumap_size * size); if ( rc < 0 )
+    {
+        free(cpumap);
        return pyxc_error_to_exception();
+    }
 
    info_dict = Py_BuildValue("{s:i,s:i,s:i,s:L,s:i}",
                              "online",   info.online, @@ -386,19
                              +419,19 @@ "running",  info.running,
                              "cpu_time", info.cpu_time,
                              "cpu",      info.cpu); -
    cpulist = PyList_New(0);
-    for ( i = 0; cpumap != 0; i++ )
+    for ( i = 0; i < nr_cpus; i++ )
    {
-        if ( cpumap & 1 ) {
+        if (*(cpumap + i / (cpumap_size * 8)) & 1 ) {
            PyObject *pyint = PyInt_FromLong(i);
            PyList_Append(cpulist, pyint);
            Py_DECREF(pyint);
        }
-        cpumap >>= 1;
+        cpumap[i / (cpumap_size * 8)] >>= 1;
    }
    PyDict_SetItemString(info_dict, "cpumap", cpulist);    
Py_DECREF(cpulist); +    free(cpumap);
    return info_dict;
}

http://old.nabble.com/file/p27941371/adjust_vcpuaffinity_more_cpu.patch
adjust_vcpuaffinity_more_cpu.patch

--
View this message in context:
http://old.nabble.com/-Patch--adjust-the-cpu-
affinity-to-more-than-64-cpus-tp27928229p27941371.html Sent from the Xen - Dev mailing list archive at Nabble.com.

Quoted 4 lines

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

-------------------------------text/plain-------------------------------
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel