add dom0 vcpu hotplug control
PATCH 1/1 add dom0 vcpu hotplug control
This patch adds new control messages for vcpu hotplug events. Via the xm vcpu_hotplug sub-program, VCPUS in domains can be enabled/disabled when CONFIG_HOTPLUG_CPU is enabled in the target domain's kernel.
Currently there is nothing that tracks whether a VCPU is up or down. My previous [1]patch added a new per-VCPU flag (VCPUF_down) which could be used to keep track of which VCPUS are up and down. Right now, there isn't a hypercall that returns the per-VCPU flag status (something equivalent to the per-domain flags in get_dominfo ). Have we thought about a get_vcpu_info hypercall? I'd like to get that implemented so I can report VCPU state in the xm list --vcpus output. That would also make it easier to skip sending control messages that don't change the VCPU's state (e.g. sending a down message to a vcpu that is already down).
Please apply.
1. http://lists.xensource.com/archives/html/xen-devel/2005-06/msg00192.html
Signature · 323 lines
Ryan Harper Software Engineer; Linux Technology Center IBM Corp., Austin, Tx (512) 838-9253 T/L: 678-9253 [email protected] diffstat output: linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 81 +++++++++++++++++ tools/python/xen/lowlevel/xu/xu.c | 16 +++ tools/python/xen/xend/XendClient.py | 6 + tools/python/xen/xend/XendDomain.py | 4 tools/python/xen/xend/XendDomainInfo.py | 12 ++ tools/python/xen/xend/server/SrvDomain.py | 8 + tools/python/xen/xend/server/messages.py | 18 +++ tools/python/xen/xm/main.py | 21 ++++ xen/include/public/io/domain_controller.h | 20 ++++ 9 files changed, 186 insertions(+) Signed-off-by: Ryan Harper <[email protected]> --- diff -urN a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-06 22:05:33.000000000 -0500 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-07 16:53:49.362987126 -0500 @@ -85,6 +85,13 @@ /* Set when the idlers are all forked */ int smp_threads_ready; +#ifdef CONFIG_HOTPLUG_CPU +struct vcpu_hotplug_handler_t { + void (*fn)(); + u32 vcpu; +}; +#endif + #if 0 /* * Trampoline 80x86 program as an array. @@ -1297,6 +1304,9 @@ } #ifdef CONFIG_HOTPLUG_CPU +#include <asm-xen/ctrl_if.h> +/* hotplug down/up funtion pointer and target vcpu */ +struct vcpu_hotplug_handler_t vcpu_hotplug_handler; /* must be called with the cpucontrol mutex held */ static int __devinit cpu_enable(unsigned int cpu) @@ -1357,6 +1367,77 @@ } printk(KERN_ERR "CPU %u didn't die...\n", cpu); } + +static int vcpu_hotplug_cpu_process(void *unused) +{ + struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler; + + if ( handler->fn ) { + (*(handler->fn))(handler->vcpu); + handler->fn = NULL; + } + return 0; +} + +static void __vcpu_hotplug_handler(void *unused) +{ + int err; + + err = kernel_thread(vcpu_hotplug_cpu_process, + NULL, CLONE_FS | CLONE_FILES); + if ( err < 0 ) + printk(KERN_ALERT "Error creating hotplug_cpu process!\n"); + +} + +static void vcpu_hotplug_event_handler(ctrl_msg_t *msg, unsigned long id) +{ + static DECLARE_WORK(vcpu_hotplug_work, __vcpu_hotplug_handler, NULL); + vcpu_hotplug_t *req = (vcpu_hotplug_t *)&msg->msg[0]; + struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler; + ssize_t ret; + + if ( msg->length != sizeof(vcpu_hotplug_t) ) + goto parse_error; + + /* grab target vcpu from msg */ + handler->vcpu = req->vcpu; + + /* determine which function to call based on msg subtype */ + switch ( msg->subtype ) { + case CMSG_VCPU_HOTPLUG_OFF: + handler->fn = (void *)&cpu_down; + ret = schedule_work(&vcpu_hotplug_work); + req->status = (u32) ret; + break; + case CMSG_VCPU_HOTPLUG_ON: + handler->fn = (void *)&cpu_up; + ret = schedule_work(&vcpu_hotplug_work); + req->status = (u32) ret; + break; + default: + goto parse_error; + } + + ctrl_if_send_response(msg); + return; + parse_error: + msg->length = 0; + ctrl_if_send_response(msg); +} + +static int __init setup_vcpu_hotplug_event(void) +{ + struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler; + + handler->fn = NULL; + ctrl_if_register_receiver(CMSG_VCPU_HOTPLUG, vcpu_hotplug_event_handler, 0); + + return 0; +} + +__initcall(setup_vcpu_hotplug_event); + #else /* ... !CONFIG_HOTPLUG_CPU */ int __cpu_disable(void) { diff -urN a/tools/python/xen/lowlevel/xu/xu.c b/tools/python/xen/lowlevel/xu/xu.c --- a/tools/python/xen/lowlevel/xu/xu.c 2005-06-06 22:05:28.000000000 -0500 +++ b/tools/python/xen/lowlevel/xu/xu.c 2005-06-07 16:51:43.084342691 -0500 @@ -744,6 +744,14 @@ C2P(mem_request_t, target, Int, Long); C2P(mem_request_t, status, Int, Long); return dict; + case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF): + C2P(vcpu_hotplug_t, vcpu, Int, Long); + C2P(vcpu_hotplug_t, status, Int, Long); + return dict; + case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON): + C2P(vcpu_hotplug_t, vcpu, Int, Long); + C2P(vcpu_hotplug_t, status, Int, Long); + return dict; } return PyString_FromStringAndSize((char *)xum->msg.msg, xum->msg.length); @@ -910,6 +918,14 @@ P2C(mem_request_t, target, u32); P2C(mem_request_t, status, u32); break; + case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF): + P2C(vcpu_hotplug_t, vcpu, u32); + P2C(vcpu_hotplug_t, status, u32); + break; + case TYPE(CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON): + P2C(vcpu_hotplug_t, vcpu, u32); + P2C(vcpu_hotplug_t, status, u32); + break; case TYPE(CMSG_USBIF_FE, CMSG_USBIF_FE_INTERFACE_STATUS_CHANGED): P2C(usbif_fe_interface_status_changed_t, status, u32); P2C(usbif_fe_interface_status_changed_t, evtchn, u16); diff -urN a/tools/python/xen/xend/XendClient.py b/tools/python/xen/xend/XendClient.py --- a/tools/python/xen/xend/XendClient.py 2005-06-06 22:05:27.000000000 -0500 +++ b/tools/python/xen/xend/XendClient.py 2005-06-07 16:51:43.086342400 -0500 @@ -271,6 +271,12 @@ 'target' : mem_target }) return val + def xend_domain_vcpu_hotplug(self, id, vcpu, state): + return self.xendPost(self.domainurl(id), + {'op' : 'vcpu_hotplug', + 'vcpu' : vcpu, + 'state' : state }) + def xend_domain_vif_limit(self, id, vif, credit, period): return self.xendPost(self.domainurl(id), { 'op' : 'vif_limit_set', diff -urN a/tools/python/xen/xend/XendDomain.py b/tools/python/xen/xend/XendDomain.py --- a/tools/python/xen/xend/XendDomain.py 2005-06-06 22:05:32.000000000 -0500 +++ b/tools/python/xen/xend/XendDomain.py 2005-06-07 16:51:43.090341819 -0500 @@ -703,6 +703,10 @@ dominfo = self.domain_lookup(id) return dominfo.mem_target_set(target) + def domain_vcpu_hotplug(self, id, vcpu, state): + dominfo = self.domain_lookup(id) + return dominfo.vcpu_hotplug(vcpu, state) + def instance(): diff -urN a/tools/python/xen/xend/XendDomainInfo.py b/tools/python/xen/xend/XendDomainInfo.py --- a/tools/python/xen/xend/XendDomainInfo.py 2005-06-06 22:05:33.000000000 -0500 +++ b/tools/python/xen/xend/XendDomainInfo.py 2005-06-07 16:51:43.092341528 -0500 @@ -1071,6 +1071,18 @@ msg = messages.packMsg('mem_request_t', { 'target' : target * (1 << 8)} ) self.channel.writeRequest(msg) + def vcpu_hotplug(self, vcpu, state): + """Disable or enable VCPU in domain. + """ + if self.channel: + if int(state) == 0: + msg = messages.packMsg('vcpu_hotplug_off_t', { 'vcpu' : vcpu} ) + else: + msg = messages.packMsg('vcpu_hotplug_on_t', { 'vcpu' : vcpu} ) + + self.channel.writeRequest(msg) + + def shutdown(self, reason, key=0): msgtype = shutdown_messages.get(reason) if not msgtype: diff -urN a/tools/python/xen/xend/server/SrvDomain.py b/tools/python/xen/xend/server/SrvDomain.py --- a/tools/python/xen/xend/server/SrvDomain.py 2005-06-06 22:05:31.000000000 -0500 +++ b/tools/python/xen/xend/server/SrvDomain.py 2005-06-07 16:51:43.095341092 -0500 @@ -180,6 +180,14 @@ val = fn(req.args, {'dom': self.dom.id}) return val + def op_vcpu_hotplug(self, op, req): + fn = FormFn(self.xd.domain_vcpu_hotplug, + [['dom', 'str'], + ['vcpu', 'int'], + ['state', 'int']]) + val = fn(req.args, {'dom': self.dom.id}) + return val + def render_POST(self, req): return self.perform(req) diff -urN a/tools/python/xen/xend/server/messages.py b/tools/python/xen/xend/server/messages.py --- a/tools/python/xen/xend/server/messages.py 2005-06-06 22:05:28.000000000 -0500 +++ b/tools/python/xen/xend/server/messages.py 2005-06-07 16:51:43.125336730 -0500 @@ -309,6 +309,24 @@ msg_formats.update(mem_request_formats) #============================================================================ +# Domain vcpu hotplug message. +#============================================================================ + +CMSG_VCPU_HOTPLUG = 10 +CMSG_VCPU_HOTPLUG_OFF = 0 +CMSG_VCPU_HOTPLUG_ON = 1 + +vcpu_hotplug_formats = { + 'vcpu_hotplug_off_t': + (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF), + + 'vcpu_hotplug_on_t': + (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON) + } + +msg_formats.update(vcpu_hotplug_formats) + +#============================================================================ class Msg: pass diff -urN a/tools/python/xen/xm/main.py b/tools/python/xen/xm/main.py --- a/tools/python/xen/xm/main.py 2005-06-06 22:05:31.000000000 -0500 +++ b/tools/python/xen/xm/main.py 2005-06-07 16:51:43.127336439 -0500 @@ -568,6 +568,27 @@ xm.prog(ProgBalloon) + +class ProgVcpuhotplug(Prog): + group = 'domain' + name = 'vcpu_hotplug' + info = """Enable or disable a VCPU in a domain.""" + + def help(self, args): + print args[0], "DOM VCPU [0|1]" + print """\nRequest virtual processor VCPU to be disabled or enabled in +domain DOM""" + + def main(self, args): + if len(args) != 4: self.err("%s: Invalid arguments(s)" % args[0]) + dom = args[1] + vcpu = args[2] + state = args[3] + server.xend_domain_vcpu_hotplug(dom, vcpu, state) + +xm.prog(ProgVcpuhotplug) + + class ProgDomid(Prog): group = 'domain' name = 'domid' diff -urN a/xen/include/public/io/domain_controller.h b/xen/include/public/io/domain_controller.h --- a/xen/include/public/io/domain_controller.h 2005-06-06 22:05:31.000000000 -0500 +++ b/xen/include/public/io/domain_controller.h 2005-06-07 16:51:43.139334694 -0500 @@ -61,6 +61,7 @@ #define CMSG_MEM_REQUEST 7 /* Memory reservation reqs */ #define CMSG_USBIF_BE 8 /* USB controller backend */ #define CMSG_USBIF_FE 9 /* USB controller frontend */ +#define CMSG_VCPU_HOTPLUG 10 /* Hotplug VCPU messages */ /****************************************************************************** * CONSOLE DEFINITIONS @@ -758,6 +759,25 @@ } PACKED shutdown_sysrq_t; /* 4 bytes */ /****************************************************************************** + * VCPU HOTPLUG CONTROLS + */ + +/* + * Subtypes for shutdown messages. + */ +#define CMSG_VCPU_HOTPLUG_OFF 0 /* turn vcpu off */ +#define CMSG_VCPU_HOTPLUG_ON 1 /* turn vcpu on */ + +/* + * CMSG_VCPU_HOTPLUG: + * Indicate which vcpu's state should change + */ +typedef struct { + u32 vcpu; /* 0: VCPU's whose state will change */ + u32 status; /* 4: Return code indicates success or failure. */ +} PACKED vcpu_hotplug_t; + +/****************************************************************************** * MEMORY CONTROLS */
PATCH 1/1 add dom0 vcpu hotplug control
Quoted 4 lines · Ryan Harper
This patch adds new control messages for vcpu hotplug events. Via the xm
vcpu_hotplugsub-program, VCPUS in domains can be enabled/disabled whenCONFIG_HOTPLUG_CPUis enabled in the target domain's kernel.
Updated this patch to use vcpu_to_cpu up/down info to control whether state changes are sent when hotplugging vcpus.
Built and tested with vcpu_down patch against 20050614 nightly unstable snapshot.
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
[email protected]
diffstat output:
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 81 +++++++++++++++++ tools/python/xen/lowlevel/xu/xu.c | 16 +++ tools/python/xen/xend/XendClient.py | 6 + tools/python/xen/xend/XendDomain.py | 12 ++ tools/python/xen/xend/XendDomainInfo.py | 12 ++ tools/python/xen/xend/server/SrvDomain.py | 8 + tools/python/xen/xend/server/messages.py | 18 +++ tools/python/xen/xm/main.py | 30 ++++++ xen/include/public/io/domain_controller.h | 20 ++++ 9 files changed, 203 insertions(+)
Signed-off-by Ryan Harper <[email protected]>
---
linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c +81 -0
@@ -85,6 +85,13 @@/* Set when the idlers are all forked */ int smp_threads_ready; +#ifdef CONFIG_HOTPLUG_CPU +struct vcpu_hotplug_handler_t { +void (*fn)(); +u32 vcpu; +}; +#endif +#if 0 /* * Trampoline 80x86 program as an array.
@@ -1297,6 +1304,9 @@ }#ifdef CONFIG_HOTPLUG_CPU +#include <asm-xen/ctrl_if.h> +/* hotplug down/up funtion pointer and target vcpu */ +struct vcpu_hotplug_handler_t vcpu_hotplug_handler;/* must be called with the cpucontrol mutex held */ static int __devinit cpu_enable (unsigned int cpu)
@@ -1357,6 +1367,77 @@ }printk (KERN_ERR "CPU %u didn't die...\n" , cpu); } + +static int vcpu_hotplug_cpu_process (void *unused) +{ +struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler; + +if ( handler->fn ) { + (*(handler->fn))(handler->vcpu); + handler->fn =NULL ; + } +return 0 ; +} + +static void __vcpu_hotplug_handler (void *unused) +{ +int err; + + err =kernel_thread (vcpu_hotplug_cpu_process, +NULL ,CLONE_FS |CLONE_FILES ); +if ( err <0 ) +printk (KERN_ALERT "Error creating hotplug_cpu process!\n" ); + +} + +static void vcpu_hotplug_event_handler (ctrl_msg_t *msg,unsigned long id) +{ + static DECLARE_WORK(vcpu_hotplug_work, __vcpu_hotplug_handler, NULL); +vcpu_hotplug_t *req = (vcpu_hotplug_t *)&msg->msg[0 ]; +struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler; +ssize_t ret; + +if ( msg->length !=sizeof (vcpu_hotplug_t) ) + goto parse_error; + +/* grab target vcpu from msg */ + handler->vcpu = req->vcpu; + +/* determine which function to call based on msg subtype */ +switch ( msg->subtype ) { +case CMSG_VCPU_HOTPLUG_OFF : + handler->fn = (void *)&cpu_down; + ret =schedule_work (&vcpu_hotplug_work); + req->status = (u32 ) ret; +break ; +case CMSG_VCPU_HOTPLUG_ON : + handler->fn = (void *)&cpu_up; + ret =schedule_work (&vcpu_hotplug_work); + req->status = (u32 ) ret; +break ; +default : + goto parse_error; + } + +ctrl_if_send_response (msg); +return ; + parse_error: + msg->length =0 ; +ctrl_if_send_response (msg); +} + +static int __init setup_vcpu_hotplug_event (void ) +{ +struct vcpu_hotplug_handler_t *handler = &vcpu_hotplug_handler; + + handler->fn =NULL ; +ctrl_if_register_receiver (CMSG_VCPU_HOTPLUG , vcpu_hotplug_event_handler,0 ); + +return 0 ; +} + +__initcall (setup_vcpu_hotplug_event); +#else /* ... !CONFIG_HOTPLUG_CPU */ int __cpu_disable (void ) {
tools/python/xen/lowlevel/xu/xu.c +16 -0
@@ -744,6 +744,14 @@C2P (mem_request_t, target, Int, Long);C2P (mem_request_t, status, Int, Long);return dict; +case TYPE (CMSG_VCPU_HOTPLUG ,CMSG_VCPU_HOTPLUG_OFF ): +C2P (vcpu_hotplug_t, vcpu, Int, Long); +C2P (vcpu_hotplug_t, status, Int, Long); +return dict; +case TYPE (CMSG_VCPU_HOTPLUG ,CMSG_VCPU_HOTPLUG_ON ): +C2P (vcpu_hotplug_t, vcpu, Int, Long); +C2P (vcpu_hotplug_t, status, Int, Long); +return dict; }return PyString_FromStringAndSize ((char *)xum->msg.msg, xum->msg.length);
@@ -909,6 +917,14 @@case TYPE (CMSG_MEM_REQUEST ,CMSG_MEM_REQUEST_SET ):P2C (mem_request_t, target, u32);break ; +case TYPE (CMSG_VCPU_HOTPLUG ,CMSG_VCPU_HOTPLUG_OFF ): +P2C (vcpu_hotplug_t, vcpu, u32); +P2C (vcpu_hotplug_t, status, u32); +break ; +case TYPE (CMSG_VCPU_HOTPLUG ,CMSG_VCPU_HOTPLUG_ON ): +P2C (vcpu_hotplug_t, vcpu, u32); +P2C (vcpu_hotplug_t, status, u32); +break ;case TYPE (CMSG_USBIF_FE ,CMSG_USBIF_FE_INTERFACE_STATUS_CHANGED ):P2C (usbif_fe_interface_status_changed_t, status, u32);P2C (usbif_fe_interface_status_changed_t, evtchn, u16);
tools/python/xen/xend/server/messages.py +18 -0
@@ -309,6 +309,24 @@ msg_formats.update(mem_request_formats) #============================================================================ +# Domain vcpu hotplug message. +#============================================================================ + +CMSG_VCPU_HOTPLUG = 10 +CMSG_VCPU_HOTPLUG_OFF = 0 +CMSG_VCPU_HOTPLUG_ON = 1 + +vcpu_hotplug_formats = { + 'vcpu_hotplug_off_t': + (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_OFF), + + 'vcpu_hotplug_on_t': + (CMSG_VCPU_HOTPLUG, CMSG_VCPU_HOTPLUG_ON) + } + +msg_formats.update(vcpu_hotplug_formats) + +#============================================================================ class Msg: pass
tools/python/xen/xend/server/SrvDomain.py +8 -0
@@ -180,6 +180,14 @@ val = fn(req.args, {'dom': self.dom.id}) return val + def op_vcpu_hotplug(self, op, req): + fn = FormFn(self.xd.domain_vcpu_hotplug, + [['dom', 'int'], + ['vcpu', 'int'], + ['state', 'int']]) + val = fn(req.args, {'dom': self.dom.id}) + return val + def render_POST(self, req): return self.perform(req)
tools/python/xen/xend/XendClient.py +6 -0
@@ -271,6 +271,12 @@ 'target' : mem_target }) return val + def xend_domain_vcpu_hotplug(self, id, vcpu, state): + return self.xendPost(self.domainurl(id), + {'op' : 'vcpu_hotplug', + 'vcpu' : vcpu, + 'state' : state }) + def xend_domain_vif_limit(self, id, vif, credit, period): return self.xendPost(self.domainurl(id), { 'op' : 'vif_limit_set',
tools/python/xen/xend/XendDomainInfo.py +12 -0
@@ -949,6 +949,18 @@ msg = messages.packMsg('mem_request_t', { 'target' : target * (1 << 8)} ) self.channel.writeRequest(msg) + def vcpu_hotplug(self, vcpu, state): + """Disable or enable VCPU in domain. + """ + if self.channel: + if int(state) == 0: + msg = messages.packMsg('vcpu_hotplug_off_t', { 'vcpu' : vcpu} ) + else: + msg = messages.packMsg('vcpu_hotplug_on_t', { 'vcpu' : vcpu} ) + + self.channel.writeRequest(msg) + + def shutdown(self, reason, key=0): msgtype = shutdown_messages.get(reason) if not msgtype:
tools/python/xen/xend/XendDomain.py +12 -0
@@ -710,6 +710,18 @@ dominfo = self.domain_lookup(id) return dominfo.mem_target_set(mem) + def domain_vcpu_hotplug(self, id, vcpu, state): + """Enable or disable VCPU vcpu in DOM id + + @param id: domain + @param vcpu: target VCPU in domain + @param state: which state VCPU will become + @return: 0 on success, -1 on error + """ + + dominfo = self.domain_lookup(id) + return dominfo.vcpu_hotplug(vcpu, state) + def domain_dumpcore(self, id): """Save a core dump for a crashed domain.
tools/python/xen/xm/main.py +30 -0
@@ -571,6 +571,36 @@ xm.prog(ProgBalloon) + +class ProgVcpuhotplug(Prog): + group = 'domain' + name = 'vcpu_hotplug' + info = """Enable or disable a VCPU in a domain.""" + + def help(self, args): + print args[0], "DOM VCPU [0|1]" + print """\nRequest virtual processor VCPU to be disabled or enabled in +domain DOM""" + + def main(self, args): + if len(args) != 4: self.err("%s: Invalid arguments(s)" % args[0]) + dom = args[1] + vcpu = args[2] + state = args[3] + info = server.xend_domain(dom) + vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').replace('-1','#') + # only send state change if states differ + try: + # (down going up) or (up going down) + if (vcpu_to_cpu[int(vcpu)] == "#" and state == "1") or \ + (vcpu_to_cpu[int(vcpu)] != "#" and state == "0"): + server.xend_domain_vcpu_hotplug(int(dom), int(vcpu), int(state)) + except IndexError: + print "Invalid VCPU(%s)"%(vcpu) + +xm.prog(ProgVcpuhotplug) + + class ProgDomid(Prog): group = 'domain' name = 'domid'
xen/include/public/io/domain_controller.h +20 -0
@@ -61,6 +61,7 @@#define CMSG_MEM_REQUEST 7/* Memory reservation reqs */ #define CMSG_USBIF_BE 8/* USB controller backend */ #define CMSG_USBIF_FE 9/* USB controller frontend */ +#define CMSG_VCPU_HOTPLUG 10/* Hotplug VCPU messages */ /****************************************************************************** * CONSOLE DEFINITIONS
@@ -758,6 +759,25 @@ }PACKED shutdown_sysrq_t;/* 4 bytes */ /****************************************************************************** +* VCPU HOTPLUG CONTROLS +*/ + +/* +* Subtypes for shutdown messages. +*/ +#define CMSG_VCPU_HOTPLUG_OFF 0/* turn vcpu off */ +#define CMSG_VCPU_HOTPLUG_ON 1/* turn vcpu on */ + +/* +* CMSG_VCPU_HOTPLUG: +* Indicate which vcpu's state should change +*/ +typedef struct { +u32 vcpu;/* 0: VCPU's whose state will change */ +u32 status;/* 4: Return code indicates success or failure. */ +} PACKED vcpu_hotplug_t; + +/****************************************************************************** * MEMORY CONTROLS */
Quoted 8 lines · Ryan Harper
Quoted 4 lines · Ryan Harper
This patch adds new control messages for vcpu hotplug events. Via the xm
vcpu_hotplugsub-program, VCPUS in domains can be enabled/disabled whenCONFIG_HOTPLUG_CPUis enabled in the target domain's kernel.Updated this patch to use
vcpu_to_cpuup/down info to control whether state changes are sent when hotplugging vcpus.
I've applied both patches. I think there's a flaw in the way how vcpu_to_cpu gets passed up to python, the -1 -> # hack is kind of gross and it won't work for cpus > 9?
Also the whole hotplug stuff is still missing interrupt re-routing when a vcpu is taken down. To do this, we need an evtchn operation to change the vcpu affinity of a port by changing notify_vcpu_id.
christian
Quoted 10 lines · Christian Limpach
Quoted 8 lines · Ryan Harper
Quoted 4 lines · Ryan Harper
This patch adds new control messages for vcpu hotplug events. Via the xm
vcpu_hotplugsub-program, VCPUS in domains can be enabled/disabled whenCONFIG_HOTPLUG_CPUis enabled in the target domain's kernel.Updated this patch to use
vcpu_to_cpuup/down info to control whether state changes are sent when hotplugging vcpus.
Quoted 7 lines
I've applied both patches. I think there's a flaw in the way how
vcpu_to_cpugets passed up to python, the -1 -> # hack is kind of gross and it won't work for cpus > 9?Also the whole hotplug stuff is still missing interrupt re-routing when a vcpu is taken down. To do this, we need an evtchn operation to change the vcpu affinity of a port by changing
notify_vcpu_id.
OK, I'll take a look and address the issues.
Thanks.
Signature · 5 lines
Ryan Harper Software Engineer; Linux Technology Center IBM Corp., Austin, Tx (512) 838-9253 T/L: 678-9253 [email protected]
Quoted 7 lines · Christian Limpach
Quoted 6 lines · Ryan Harper
Quoted 4 lines · Ryan Harper
This patch adds new control messages for vcpu hotplug events. Via the xm
vcpu_hotplugsub-program, VCPUS in domains can be enabled/disabled whenCONFIG_HOTPLUG_CPUis enabled in the target domain's kernel.
Quoted 6 lines
Quoted 2 lines
Updated this patch to use
vcpu_to_cpuup/down info to control whether state changes are sent when hotplugging vcpus.I've applied both patches. I think there's a flaw in the way how
vcpu_to_cpugets passed up to python, the -1 -> # hack is kind of gross and it won't work for cpus > 9?
I updated the vcpu_to_cpu string creation to include a field separator, which gets rid of the -1 -> # hack and works for cpus > 9.
I ran into some issues with stale vcpu_to_cpu lists when running the hotplug subprogram. I would take a vcpu offline, and then issue the command to bring it back and the vcpu_to_cpu list would not have changed to indicate the the vcpu actually went down. If I injected a xm list -v (which always showed the correct mapping) then subsequent hotplug commands would see the state change and fire off the hotplug request. I don't know that not sending the event when not changing state saves that much work so I took the state check out and now just send the hotplug event directly.
Quoted 3 lines
Also the whole hotplug stuff is still missing interrupt re-routing when a vcpu is taken down. To do this, we need an evtchn operation to change the vcpu affinity of a port by changing
notify_vcpu_id.
I don't fully understand all of the mappings that are happening, so this part of the patch might be way off. In any case, I've added a new evtchn op to set the notify_vcpu_id field of a channel. I updated the HOTPLUG_CPU code to use the new routines when bringing cpus up and down. When taking down a cpu, I route the IPI irq channels to CPU 0, and when the cpu comes up, it re-routes the channels back to the awakened CPU.
Signature · 240 lines
Ryan Harper Software Engineer; Linux Technology Center IBM Corp., Austin, Tx (512) 838-9253 T/L: 678-9253 [email protected] diffstat output: linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c | 19 +++++++--- linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c | 32 +++++++++++++++++ tools/python/xen/xend/XendDomainInfo.py | 6 ++- tools/python/xen/xm/main.py | 18 +-------- xen/common/event_channel.c | 29 +++++++++++++++ xen/include/public/event_channel.h | 10 ++++- 6 files changed, 91 insertions(+), 23 deletions(-) Signed-off-by: Ryan Harper <[email protected]> --- diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c --- b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-21 22:10:55.000000000 -0500 +++ c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-06-22 14:31:48.765503988 -0500 @@ -103,6 +103,11 @@ DEFINE_PER_CPU(int, cpu_state) = { 0 }; #endif +static DEFINE_PER_CPU(int, resched_irq); +static DEFINE_PER_CPU(int, callfunc_irq); +static char resched_name[NR_CPUS][15]; +static char callfunc_name[NR_CPUS][15]; + #if 0 /* * Currently trivial. Write the real->protected mode @@ -1328,6 +1333,10 @@ while (!cpu_online(cpu)) cpu_relax(); + /* re-route bound IRQs 0 to cpu */ + rebind_evtchn_from_irq(0, cpu, per_cpu(resched_irq, cpu)); + rebind_evtchn_from_irq(0, cpu, per_cpu(callfunc_irq, cpu)); + fixup_irqs(cpu_online_map); /* counter the disable in fixup_irqs() */ local_irq_enable(); @@ -1357,6 +1366,11 @@ cpu_clear(cpu, map); fixup_irqs(map); + + /* re-route IRQs from dead vcpu to another */ + rebind_evtchn_from_irq(cpu, 0, per_cpu(resched_irq, cpu)); + rebind_evtchn_from_irq(cpu, 0, per_cpu(callfunc_irq, cpu)); + /* It's now safe to remove this processor from the online map */ cpu_clear(cpu, cpu_online_map); @@ -1514,11 +1528,6 @@ extern irqreturn_t smp_reschedule_interrupt(int, void *, struct pt_regs *); extern irqreturn_t smp_call_function_interrupt(int, void *, struct pt_regs *); -static DEFINE_PER_CPU(int, resched_irq); -static DEFINE_PER_CPU(int, callfunc_irq); -static char resched_name[NR_CPUS][15]; -static char callfunc_name[NR_CPUS][15]; - void __init smp_intr_init(void) { int cpu = smp_processor_id(); diff -urN b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c --- b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c 2005-06-21 22:10:53.000000000 -0500 +++ c/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c 2005-06-22 14:31:48.767503708 -0500 @@ -271,6 +271,38 @@ return irq; } +void rebind_evtchn_from_ipi(int cpu, int newcpu, int ipi) +{ + evtchn_op_t op; + int evtchn = per_cpu(ipi_to_evtchn, cpu)[ipi]; + + spin_lock(&irq_mapping_update_lock); + + op.cmd = EVTCHNOP_rebind; + op.u.rebind.port = evtchn; + op.u.rebind.vcpu = newcpu; + if ( HYPERVISOR_event_channel_op(&op) != 0 ) + printk(KERN_INFO "Failed to rebind IPI%d to CPU%d\n",ipi,newcpu); + + spin_unlock(&irq_mapping_update_lock); +} + +void rebind_evtchn_from_irq(int cpu, int newcpu, int irq) +{ + evtchn_op_t op; + int evtchn = irq_to_evtchn[irq]; + + spin_lock(&irq_mapping_update_lock); + + op.cmd = EVTCHNOP_rebind; + op.u.rebind.port = evtchn; + op.u.rebind.vcpu = newcpu; + if ( HYPERVISOR_event_channel_op(&op) != 0 ) + printk(KERN_INFO "Failed to rebind IRQ%d to CPU%d\n",irq,newcpu); + + spin_unlock(&irq_mapping_update_lock); +} + void unbind_ipi_on_cpu_from_irq(int cpu, int ipi) { evtchn_op_t op; diff -urN b/tools/python/xen/xend/XendDomainInfo.py c/tools/python/xen/xend/XendDomainInfo.py --- b/tools/python/xen/xend/XendDomainInfo.py 2005-06-21 22:10:54.000000000 -0500 +++ c/tools/python/xen/xend/XendDomainInfo.py 2005-06-22 14:31:48.000000000 -0500 @@ -423,8 +423,10 @@ sxpr.append(['cpu_time', self.info['cpu_time']/1e9]) sxpr.append(['vcpus', self.info['vcpus']]) sxpr.append(['cpumap', self.info['cpumap']]) - sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x), - self.info['vcpu_to_cpu'][0:self.info['vcpus']]))]) + # build a string, using '|' to seperate items, show only up + # to number of vcpus in domain, and trim the trailing '|' + sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x)+'|', + self.info['vcpu_to_cpu'][0:self.info['vcpus']]))[:-1]]) if self.start_time: up_time = time.time() - self.start_time diff -urN b/tools/python/xen/xm/main.py c/tools/python/xen/xm/main.py --- b/tools/python/xen/xm/main.py 2005-06-21 22:10:53.000000000 -0500 +++ c/tools/python/xen/xm/main.py 2005-06-22 14:54:50.810258798 -0500 @@ -410,8 +410,7 @@ print 'Name Id VCPU CPU CPUMAP' for dom in doms: info = server.xend_domain(dom) - # XXX this is quite broken for cpu's > 9 - vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '?').replace('-1','#') + vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').split('|') cpumap = sxp.child_value(info, 'cpumap', []) mask = ((int(sxp.child_value(info, 'vcpus', '0')))**2) - 1 count = 0 @@ -420,10 +419,7 @@ d['name'] = sxp.child_value(info, 'name', '??') d['dom'] = int(sxp.child_value(info, 'id', '-1')) d['vcpu'] = int(count) - if cpu == "#": - d['cpu'] = int("-1") - else: - d['cpu'] = int(cpu) + d['cpu'] = int(cpu) d['cpumap'] = int(cpumap[count])&mask count = count + 1 print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d 0x%(cpumap)x" % d) @@ -593,15 +589,7 @@ state = int(args[3]) dom = server.xend_domain(name) id = sxp.child_value(dom, 'id') - vcpu_to_cpu = sxp.child_value(dom, 'vcpu_to_cpu', '-1') - # only send state change if states differ - try: - # (down going up) or (up going down) - if (vcpu_to_cpu[vcpu] == "-1" and state == 1) or \ - (vcpu_to_cpu[vcpu] != "-1" and state == 0): - server.xend_domain_vcpu_hotplug(id, vcpu, state) - except IndexError: - print "Invalid VCPU(%d)"%(vcpu) + server.xend_domain_vcpu_hotplug(id, vcpu, state) xm.prog(ProgVcpuhotplug) diff -urN b/xen/common/event_channel.c c/xen/common/event_channel.c --- b/xen/common/event_channel.c 2005-06-21 22:10:55.000000000 -0500 +++ c/xen/common/event_channel.c 2005-06-22 14:55:20.936237917 -0500 @@ -579,6 +579,29 @@ return rc; } +static long evtchn_rebind(evtchn_rebind_t *bind) +{ + struct domain *d = current->domain; + int port = bind->port; + int vcpu = bind->vcpu; + struct evtchn *chn; + long rc = 0; + + spin_lock(&d->evtchn_lock); + + if ( !port_is_valid(d, port) ) + { + rc = -EINVAL; + goto out; + } + + chn = evtchn_from_port(d, port); + chn->notify_vcpu_id = vcpu; + + out: + spin_unlock(&d->evtchn_lock); + return rc; +} long do_event_channel_op(evtchn_op_t *uop) { @@ -637,6 +660,12 @@ rc = -EFAULT; break; + case EVTCHNOP_rebind: + rc = evtchn_rebind(&op.u.rebind); + if ( (rc == 0) && (copy_to_user(uop, &op, sizeof(op)) != 0) ) + rc = -EFAULT; + break; + default: rc = -ENOSYS; break; diff -urN b/xen/include/public/event_channel.h c/xen/include/public/event_channel.h --- b/xen/include/public/event_channel.h 2005-06-21 22:10:55.000000000 -0500 +++ c/xen/include/public/event_channel.h 2005-06-22 14:31:48.801498950 -0500 @@ -162,6 +162,13 @@ } PACKED u; } PACKED evtchn_status_t; /* 20 bytes */ +#define EVTCHNOP_rebind 8 +typedef struct { + /* IN parameters. */ + u32 port; /* 0 */ + u32 vcpu; /* 4 */ +} PACKED evtchn_rebind_t; /* 8 bytes */ + typedef struct { u32 cmd; /* EVTCHNOP_* */ /* 0 */ u32 __reserved; /* 4 */ @@ -174,7 +181,8 @@ evtchn_close_t close; evtchn_send_t send; evtchn_status_t status; - u8 __dummy[24]; + evtchn_rebind_t rebind; + u8 __dummy[16]; } PACKED u; } PACKED evtchn_op_t; /* 32 bytes */