tools: set migration constraints from cmdline
PATCH 1/1 tools: set migration constraints from cmdline
# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359394320 -3600
# Node ID 577b051fca174be9f7b3d6590c62735cdf214021 # Parent 6727070b4129cf852199b66b6a81042ee6966a98
tools: set migration constraints from cmdline
Add new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>
-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM)
--max_factor <factor>
-N Abort migration instead of doing final suspend.
--no_suspend
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
tools/libxc/xc_domain_save.c +15 -1
@@ -43,6 +43,8 @@*/ #define DEF_MAX_ITERS 29/* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3/* never send more than 3x p2m_size */ +/* Suspend guest for final transit once number of dirty pages is that low */ +#define DEF_MIN_REMAINING 50struct save_ctx {unsigned long hvirt_start;/* virtual starting address of the hypervisor */
@@ -804,9 +806,11 @@ int xc_domain_save(xc_interface *xch, inint rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG ); +int nosuspend = (flags &XCFLAGS_DOMSAVE_NOSUSPEND );int superpages = !!hvm;int race =0 , sent_last_iter, skip_this_iter =0 ;unsigned int sent_this_iter =0 ; +int min_reached;int tmem_saved =0 ;/* The new domain's shared-info frame number. */
@@ -1525,10 +1529,20 @@ int xc_domain_save(xc_interface *xch, inif ( live ) { + min_reached = sent_this_iter + skip_this_iter <DEF_MIN_REMAINING ;if ( (iter >= max_iters) || - (sent_this_iter+skip_this_iter <50 ) || + min_reached || (total_sent > dinfo->p2m_size*max_factor) ) { +if ( min_reached && nosuspend ) + { +ERROR ("Live migration aborted, as requested. (guest too busy?)" +" total_sent %lu iter %d, max_iters %u max_factor %u" , + total_sent, iter, max_iters, max_factor); + rc =1 ; + goto out; + } +DPRINTF ("Start last iteration\n" ); last_iter =1 ;
tools/libxc/xenguest.h +2 -0
@@ -28,6 +28,8 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8#define XCFLAGS_CHECKPOINT_COMPRESS 16 +#define XCFLAGS_DOMSAVE_NOSUSPEND 32 +#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32
tools/libxl/libxl.c +4 -0
@@ -757,6 +757,7 @@ static void domain_suspend_cb(libxl__egc }int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int max_iters,int max_factor,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);
@@ -779,6 +780,9 @@ int libxl_domain_suspend(libxl_ctx *ctx, dss->type = type; dss->live = flags &LIBXL_SUSPEND_LIVE ; dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->max_iters = max_iters; + dss->max_factor = max_factor; + dss->xlflags = flags;libxl__domain_suspend (egc, dss);return AO_INPROGRESS ;
tools/libxl/libxl.h +2 -0
@@ -502,10 +502,12 @@ void libxl_domain_config_dispose(libxl_d int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ + int max_iters, int max_factor, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY;#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest
tools/libxl/libxl_dom.c +1 -0
@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live) ?XCFLAGS_LIVE :0 | (debug) ?XCFLAGS_DEBUG :0 + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ) ?XCFLAGS_DOMSAVE_NOSUSPEND :0 | (dss->hvm) ?XCFLAGS_HVM :0 ; dss->suspend_eventchn =-1 ;
tools/libxl/libxl_internal.h +3 -0
@@ -2253,6 +2253,9 @@ struct libxl__domain_suspend_state {xc_evtchn *xce;/* event channel handle */ int suspend_eventchn;int hvm; +int max_iters; +int max_factor; +int xlflags;int xcflags;int guest_responded;const char *dm_savefile;
tools/libxl/xl_cmdimpl.c +21 -5
@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );close (fd);if (rc <0 )
@@ -3332,6 +3332,8 @@ static void migrate_do_preamble(int send }static void migrate_domain (uint32_t domid,const char *rune, +int max_iters,int max_factor, +int no_suspend,const char *override_config_file) {pid_t child =-1 ;
@@ -3341,6 +3343,7 @@ static void migrate_domain(uint32_t domichar rc_buf;uint8_t *config_data;int config_len; +int flags =LIBXL_SUSPEND_LIVE ;save_domain_core_begin (domid, override_config_file, &config_data, &config_len);
@@ -3358,7 +3361,10 @@ static void migrate_domain(uint32_t domixtl_stdiostream_adjust_flags (logger,XTL_STDIOSTREAM_HIDE_PROGRESS ,0 ); - rc =libxl_domain_suspend (ctx, domid, send_fd,LIBXL_SUSPEND_LIVE ,NULL ); +if (no_suspend) + flags |=LIBXL_SUSPEND_NO_FINAL_SUSPEND ; + + rc =libxl_domain_suspend (ctx, domid, send_fd, flags, max_iters, max_factor,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
@@ -3751,8 +3757,9 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , no_suspend =0 ; + + SWITCH_FOREACH_OPT(opt, "FC:s:edM:m:N", NULL, "migrate", 2) {case 'C' : config_filename = optarg;break ;
@@ -3769,6 +3776,15 @@ int main_migrate(int argc, char **argv)case 'd' : debug =1 ;break ; +case 'M' : + max_iters =atoi (optarg); +break ; +case 'm' : + max_factor =atoi (optarg); +break ; +case 'N' : + no_suspend =1 ; +break ; } domid =find_domain (argv[optind]);
@@ -3784,7 +3800,7 @@ int main_migrate(int argc, char **argv)return 1 ; } -migrate_domain (domid, rune, config_filename); +migrate_domain (domid, rune, max_iters, max_factor, no_suspend, config_filename);return 0 ; }
tools/libxl/xl_cmdtable.c +3 -0
@@ -152,6 +152,9 @@ struct cmd_spec cmd_table[] = {"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" " to sh. If empty, run <host> instead of ssh <host> xl\n" " migrate-receive [-d -e]\n" +"-M --max_iters <number> Number of iterations before final suspend (default: 30)\n" +"-m --max_factor <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n" +"-N --no_suspend Abort migration instead of doing final suspend.\n" "-e Do not wait in the background (on <host>) for the death\n" " of the domain." },
tools/python/xen/xend/XendCheckpoint.py +12 -2
@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst # enabled. Passing "0" simply uses the defaults compiled into # libxenguest; see the comments and/or code in xc_linux_save() for # more information. + max_iters = dominfo.info.get('max_iters', "0") + max_factor = dominfo.info.get('max_factor', "0") + no_suspend = dominfo.info.get('no_suspend', None) + if max_iters == "None": + max_iters = "0" + if max_factor == "None": + max_factor = "0" + if no_suspend == "None": + no_suspend = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", - str(int(live) | (int(hvm) << 2)) ] + str(dominfo.getDomid()), + max_iters, max_factor, + str(int(live) | (int(hvm) << 2) | (int(no_suspend) << 5) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild):
tools/python/xen/xend/XendDomain.py +12 -0
@@ -1832,6 +1832,18 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) + def domain_migrate_constraints_set((self, domid, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param domid: Domain ID or Name + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + dominfo = self.domain_lookup_nr(domid) + if not dominfo: + raise XendInvalidDomain(str(domid)) + dominfo.setMigrateConstraints(max_iters, max_factor, no_suspend) + def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain.
tools/python/xen/xend/XendDomainInfo.py +12 -0
@@ -1459,6 +1459,18 @@ class XendDomainInfo: pci_conf = self.info['devices'][dev_uuid][1] return map(pci_dict_to_bdf_str, pci_conf['devs']) + def setMigrateConstraints(self, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + log.debug("Setting setMigrateConstraints of domain %s (%s) to '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, no_suspend) + self.info['max_iters'] = str(max_iters) + self.info['max_factor'] = str(max_factor) + self.info['no_suspend'] = str(no_suspend) + def setMemoryTarget(self, target): """Set the memory target of this domain. @param target: In MiB.
tools/python/xen/xm/migrate.py +16 -0
@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c fn=set_true, default=0, use="Change home server for managed domains.") +gopts.opt('max_iters', short='M', val='max_iters', + fn=set_int, default=None, + use="Number of iterations before final suspend (default: 30).") + +gopts.opt('max_factor', short='m', val='max_factor', + fn=set_int, default=None, + use="Max amount of memory to transfer before final suspend (default: 3*RAM).") + +gopts.opt('no_suspend', short='S', + fn=set_true, default=None, + use="Abort migration instead of doing final suspend.") + def help(): return str(gopts)
@@ -80,6 +92,10 @@ def main(argv): server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config) else: + server.xend.domain.migrate_constraints_set(dom, + opts.vals.max_iters, + opts.vals.max_factor, + opts.vals.no_suspend) server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node,
Quoted 24 lines · Olaf Hering
# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359394320 -3600
# Node ID577b051fca174be9f7b3d6590c62735cdf214021# Parent6727070b4129cf852199b66b6a81042ee6966a98
tools: set migration constraints from cmdlineAdd new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
(quoted diff) +0
-6-M <number> Number of iterations before final suspend (default: 30) --max_iters <number> -m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM) --max_factor <factor> -N Abort migration instead of doing final suspend. --no_suspend
Quoted 3 lines
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
I'm afraid not, the compatibility requirements are covered in the comment near the top of libxl.h.
So you either need a new function or to leverage the LIBXL_API_VERSION define (which the user must supply) such that people providing 0x040200 see the current interface and people providing 0x040300 (or nothing) see the new one.
Quoted 320 lines
Signed-off-by Olaf Hering <[email protected]>diff -r
6727070b4129-r577b051fca17tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c +15
-1@@ -43,6 +43,8 @@*/ #define DEF_MAX_ITERS 29/* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3/* never send more than 3x p2m_size */ +/* Suspend guest for final transit once number of dirty pages is that low */ +#define DEF_MIN_REMAINING 50struct save_ctx {unsigned long hvirt_start;/* virtual starting address of the hypervisor */ @@ -804,9 +806,11 @@ int xc_domain_save(xc_interface *xch, inint rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG ); +int nosuspend = (flags &XCFLAGS_DOMSAVE_NOSUSPEND );int superpages = !!hvm;int race =0 , sent_last_iter, skip_this_iter =0 ;unsigned int sent_this_iter =0 ; +int min_reached;int tmem_saved =0 ;/* The new domain's shared-info frame number. */ @@ -1525,10 +1529,20 @@ int xc_domain_save(xc_interface *xch, inif ( live ) { + min_reached = sent_this_iter + skip_this_iter <DEF_MIN_REMAINING ;if ( (iter >= max_iters) || - (sent_this_iter+skip_this_iter <50 ) || + min_reached || (total_sent > dinfo->p2m_size*max_factor) ) { +if ( min_reached && nosuspend ) + { +ERROR ("Live migration aborted, as requested. (guest too busy?)" +" total_sent %lu iter %d, max_iters %u max_factor %u" , + total_sent, iter, max_iters, max_factor); + rc =1 ; + goto out; + } +DPRINTF ("Start last iteration\n" ); last_iter =1 ;diff -r
6727070b4129-r577b051fca17tools/libxc/xenguest.htools/libxc/xenguest.h +2
-0@@ -28,6 +28,8 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8#define XCFLAGS_CHECKPOINT_COMPRESS 16 +#define XCFLAGS_DOMSAVE_NOSUSPEND 32 +#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32diff -r
6727070b4129-r577b051fca17tools/libxl/libxl.ctools/libxl/libxl.c +4
-0@@ -757,6 +757,7 @@ static void domain_suspend_cb(libxl__egc }int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int max_iters,int max_factor,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);@@ -779,6 +780,9 @@ int libxl_domain_suspend(libxl_ctx *ctx, dss->type = type; dss->live = flags &LIBXL_SUSPEND_LIVE ; dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->max_iters = max_iters; + dss->max_factor = max_factor; + dss->xlflags = flags;libxl__domain_suspend (egc, dss);return AO_INPROGRESS ;diff -r
6727070b4129-r577b051fca17tools/libxl/libxl.htools/libxl/libxl.h +2
-0@@ -502,10 +502,12 @@ void libxl_domain_config_dispose(libxl_d int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ + int max_iters, int max_factor, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY;#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest diff -r
6727070b4129-r577b051fca17tools/libxl/libxl_dom.ctools/libxl/libxl_dom.c +1
-0@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live) ?XCFLAGS_LIVE :0 | (debug) ?XCFLAGS_DEBUG :0 + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ) ?XCFLAGS_DOMSAVE_NOSUSPEND :0 | (dss->hvm) ?XCFLAGS_HVM :0 ; dss->suspend_eventchn =-1 ;diff -r
6727070b4129-r577b051fca17tools/libxl/libxl_internal.htools/libxl/libxl_internal.h +3
-0@@ -2253,6 +2253,9 @@ struct libxl__domain_suspend_state {xc_evtchn *xce;/* event channel handle */ int suspend_eventchn;int hvm; +int max_iters; +int max_factor; +int xlflags;int xcflags;int guest_responded;const char *dm_savefile;diff -r
6727070b4129-r577b051fca17tools/libxl/xl_cmdimpl.ctools/libxl/xl_cmdimpl.c +21
-5@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );close (fd);if (rc <0 )@@ -3332,6 +3332,8 @@ static void migrate_do_preamble(int send }static void migrate_domain (uint32_t domid,const char *rune, +int max_iters,int max_factor, +int no_suspend,const char *override_config_file) {pid_t child =-1 ;@@ -3341,6 +3343,7 @@ static void migrate_domain(uint32_t domichar rc_buf;uint8_t *config_data;int config_len; +int flags =LIBXL_SUSPEND_LIVE ;save_domain_core_begin (domid, override_config_file, &config_data, &config_len);@@ -3358,7 +3361,10 @@ static void migrate_domain(uint32_t domixtl_stdiostream_adjust_flags (logger,XTL_STDIOSTREAM_HIDE_PROGRESS ,0 ); - rc =libxl_domain_suspend (ctx, domid, send_fd,LIBXL_SUSPEND_LIVE ,NULL ); +if (no_suspend) + flags |=LIBXL_SUSPEND_NO_FINAL_SUSPEND ; + + rc =libxl_domain_suspend (ctx, domid, send_fd, flags, max_iters, max_factor,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);@@ -3751,8 +3757,9 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , no_suspend =0 ; + + SWITCH_FOREACH_OPT(opt, "FC:s:edM:m:N", NULL, "migrate", 2) {case 'C' : config_filename = optarg;break ;@@ -3769,6 +3776,15 @@ int main_migrate(int argc, char **argv)case 'd' : debug =1 ;break ; +case 'M' : + max_iters =atoi (optarg); +break ; +case 'm' : + max_factor =atoi (optarg); +break ; +case 'N' : + no_suspend =1 ; +break ; } domid =find_domain (argv[optind]);@@ -3784,7 +3800,7 @@ int main_migrate(int argc, char **argv)return 1 ; } -migrate_domain (domid, rune, config_filename); +migrate_domain (domid, rune, max_iters, max_factor, no_suspend, config_filename);return 0 ; }diff -r
6727070b4129-r577b051fca17tools/libxl/xl_cmdtable.ctools/libxl/xl_cmdtable.c +3
-0@@ -152,6 +152,9 @@ struct cmd_spec cmd_table[] = {"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" " to sh. If empty, run <host> instead of ssh <host> xl\n" " migrate-receive [-d -e]\n" +"-M --max_iters <number> Number of iterations before final suspend (default: 30)\n" +"-m --max_factor <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n" +"-N --no_suspend Abort migration instead of doing final suspend.\n" "-e Do not wait in the background (on <host>) for the death\n" " of the domain." },diff -r
6727070b4129-r577b051fca17tools/python/xen/xend/XendCheckpoint.pytools/python/xen/xend/XendCheckpoint.py +12
-2@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst # enabled. Passing "0" simply uses the defaults compiled into # libxenguest; see the comments and/or code in xc_linux_save() for # more information. + max_iters = dominfo.info.get('max_iters', "0") + max_factor = dominfo.info.get('max_factor', "0") + no_suspend = dominfo.info.get('no_suspend', None) + if max_iters == "None": + max_iters = "0" + if max_factor == "None": + max_factor = "0" + if no_suspend == "None": + no_suspend = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", - str(int(live) | (int(hvm) << 2)) ] + str(dominfo.getDomid()), + max_iters, max_factor, + str(int(live) | (int(hvm) << 2) | (int(no_suspend) << 5) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild):diff -r
6727070b4129-r577b051fca17tools/python/xen/xend/XendDomain.pytools/python/xen/xend/XendDomain.py +12
-0@@ -1832,6 +1832,18 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) + def domain_migrate_constraints_set((self, domid, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param domid: Domain ID or Name + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + dominfo = self.domain_lookup_nr(domid) + if not dominfo: + raise XendInvalidDomain(str(domid)) + dominfo.setMigrateConstraints(max_iters, max_factor, no_suspend) + def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain.diff -r
6727070b4129-r577b051fca17tools/python/xen/xend/XendDomainInfo.pytools/python/xen/xend/XendDomainInfo.py +12
-0@@ -1459,6 +1459,18 @@ class XendDomainInfo: pci_conf = self.info['devices'][dev_uuid][1] return map(pci_dict_to_bdf_str, pci_conf['devs']) + def setMigrateConstraints(self, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + log.debug("Setting setMigrateConstraints of domain %s (%s) to '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, no_suspend) + self.info['max_iters'] = str(max_iters) + self.info['max_factor'] = str(max_factor) + self.info['no_suspend'] = str(no_suspend) + def setMemoryTarget(self, target): """Set the memory target of this domain. @param target: In MiB.diff -r
6727070b4129-r577b051fca17tools/python/xen/xm/migrate.pytools/python/xen/xm/migrate.py +16
-0@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c fn=set_true, default=0, use="Change home server for managed domains.") +gopts.opt('max_iters', short='M', val='max_iters', + fn=set_int, default=None, + use="Number of iterations before final suspend (default: 30).") + +gopts.opt('max_factor', short='m', val='max_factor', + fn=set_int, default=None, + use="Max amount of memory to transfer before final suspend (default: 3*RAM).") + +gopts.opt('no_suspend', short='S', + fn=set_true, default=None, + use="Abort migration instead of doing final suspend.") + def help(): return str(gopts)@@ -80,6 +92,10 @@ def main(argv): server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config) else: + server.xend.domain.migrate_constraints_set(dom, + opts.vals.max_iters, + opts.vals.max_factor, + opts.vals.no_suspend) server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node,_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel
Quoted 5 lines · Ian Campbell
Quoted 3 lines · Olaf Hering
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
Quoted 7 lines
I'm afraid not, the compatibility requirements are covered in the comment near the top of
libxl.h.So you either need a new function or to leverage the
LIBXL_API_VERSIONdefine (which the user must supply) such that people providing0x040200see the current interface and people providing0x040300(or nothing) see the new one.
I sent v2 of this patch which includes an (the first) attempt to use LIBXL_API_VERSION. I think in this patch the SONAME needs to be changed as well now.
Olaf
Quoted 5 lines · Ian Campbell
Quoted 3 lines · Olaf Hering
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
Quoted 7 lines
I'm afraid not, the compatibility requirements are covered in the comment near the top of
libxl.h.So you either need a new function or to leverage the
LIBXL_API_VERSIONdefine (which the user must supply) such that people providing0x040200see the current interface and people providing0x040300(or nothing) see the new one.
And to avoid the API change at all, should max_iters and max_factor be passed via xenstore to xc_domain_save()? So that either the caller of xc_domain_save reads the values from xenstore, or the function itself reads it from there. What do you think?
Olaf
Quoted 15 lines · Olaf Hering
Quoted 5 lines · Ian Campbell
Quoted 3 lines · Olaf Hering
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
Quoted 7 lines
I'm afraid not, the compatibility requirements are covered in the comment near the top of
libxl.h.So you either need a new function or to leverage the
LIBXL_API_VERSIONdefine (which the user must supply) such that people providing0x040200see the current interface and people providing0x040300(or nothing) see the new one.
Quoted 4 lines
And to avoid the API change at all, should
max_itersandmax_factorbe passed via xenstore toxc_domain_save()? So that either the caller ofxc_domain_savereads the values from xenstore, or the function itself reads it from there. What do you think?
Unfortunately libxc cannot access xenstore.
Ian.
Quoted 16 lines · Ian Campbell
Quoted 15 lines · Olaf Hering
Quoted 12 lines · Ian Campbell
Quoted 3 lines · Olaf Hering
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
I'm afraid not, the compatibility requirements are covered in the comment near the top of
libxl.h.So you either need a new function or to leverage the
LIBXL_API_VERSIONdefine (which the user must supply) such that people providing0x040200see the current interface and people providing0x040300(or nothing) see the new one.
Quoted 6 lines
Quoted 4 lines
And to avoid the API change at all, should
max_itersandmax_factorbe passed via xenstore toxc_domain_save()? So that either the caller ofxc_domain_savereads the values from xenstore, or the function itself reads it from there. What do you think?Unfortunately libxc cannot access xenstore.
In case of xm, it would be the xc_save binary, which can receive both values from argv[].
In case of xl, it would be a xenstore write in main_migrate and a read in libxl_domain_suspend. Which would cover also other callers of libxl_domain_suspend.
Olaf
Quoted 23 lines · Olaf Hering
Quoted 21 lines · Ian Campbell
Quoted 19 lines · Olaf Hering
Quoted 5 lines · Ian Campbell
Quoted 3 lines · Olaf Hering
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
Quoted 7 lines
I'm afraid not, the compatibility requirements are covered in the comment near the top of
libxl.h.So you either need a new function or to leverage the
LIBXL_API_VERSIONdefine (which the user must supply) such that people providing0x040200see the current interface and people providing0x040300(or nothing) see the new one.And to avoid the API change at all, should
max_itersandmax_factorbe passed via xenstore toxc_domain_save()? So that either the caller ofxc_domain_savereads the values from xenstore, or the function itself reads it from there. What do you think?
Quoted 7 lines
Quoted 1 line
Unfortunately libxc cannot access xenstore.
In case of xm, it would be the
xc_savebinary, which can receive both values from argv[].
In case of xl, it would be a xenstore write inmain_migrateand a read inlibxl_domain_suspend. Which would cover also other callers oflibxl_domain_suspend.
I see, I think this is worse than the LIBXL_API_VERSION approach you've already implemented.
Also, now I think about it, users of libxl are not expected to have to use xenstore either.
Ian.
PATCH v2 1/1 tools: set migration constraints from cmdline
In reply to Olaf Hering# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359563423 -3600
# Node ID e44917a64f31036dccb81864e942c49e3a36cab5 # Parent 6727070b4129cf852199b66b6a81042ee6966a98
tools: set migration constraints from cmdline
Add new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>
-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM)
--max_factor <factor>
-N Abort migration instead of doing final suspend.
--no_suspend
A variant of this change has been tested with xend, the patch below is only compile tested. The changes to libxl change the API, is that approach acceptable?
v2:
- use LIBXL_API_VERSION and define libxl_domain_suspend_0x040200
- fix logic error in min_reached check in xc_domain_save
- add longopts
- update --help text
- correct description of migrate --help text
tools/libxc/xc_domain_save.c +14 -1
@@ -43,6 +43,8 @@*/ #define DEF_MAX_ITERS 29/* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3/* never send more than 3x p2m_size */ +/* Suspend guest for final transit once number of dirty pages is that low */ +#define DEF_MIN_REMAINING 50struct save_ctx {unsigned long hvirt_start;/* virtual starting address of the hypervisor */
@@ -804,6 +806,7 @@ int xc_domain_save(xc_interface *xch, inint rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG ); +int no_suspend = (flags &XCFLAGS_DOMSAVE_NOSUSPEND );int superpages = !!hvm;int race =0 , sent_last_iter, skip_this_iter =0 ;unsigned int sent_this_iter =0 ;
@@ -1525,10 +1528,20 @@ int xc_domain_save(xc_interface *xch, inif ( live ) { +int min_reached = sent_this_iter + skip_this_iter <DEF_MIN_REMAINING ;if ( (iter >= max_iters) || - (sent_this_iter+skip_this_iter <50 ) || + min_reached || (total_sent > dinfo->p2m_size*max_factor) ) { +if ( !min_reached && no_suspend ) + { +ERROR ("Live migration aborted, as requested. (guest too busy?)" +" total_sent %lu iter %d, max_iters %u max_factor %u" , + total_sent, iter, max_iters, max_factor); + rc =1 ; + goto out; + } +DPRINTF ("Start last iteration\n" ); last_iter =1 ;
tools/libxc/xenguest.h +2 -0
@@ -28,6 +28,8 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8#define XCFLAGS_CHECKPOINT_COMPRESS 16 +#define XCFLAGS_DOMSAVE_NOSUSPEND 32 +#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32
tools/libxl/libxl.c +19 -1
@@ -756,7 +756,8 @@ static void domain_suspend_cb(libxl__egc } -int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +static int __libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,int max_iters,int max_factor,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);
@@ -779,6 +780,9 @@ int libxl_domain_suspend(libxl_ctx *ctx, dss->type = type; dss->live = flags &LIBXL_SUSPEND_LIVE ; dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->max_iters = max_iters; + dss->max_factor = max_factor; + dss->xlflags = flags;libxl__domain_suspend (egc, dss);return AO_INPROGRESS ;
@@ -787,6 +791,20 @@ int libxl_domain_suspend(libxl_ctx *ctx,return AO_ABORT (rc); } +int libxl_domain_suspend_0x040200 (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, flags,0 ,0 , ao_how); +} + +int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int max_iters,int max_factor, +const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, flags, max_iters, + max_factor, ao_how); +} +int libxl_domain_pause (libxl_ctx *ctx,uint32_t domid) {int ret;
tools/libxl/libxl.h +14 -1
@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200 +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#endif +#else +int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, + int flags, /* LIBXL_SUSPEND_* */ + int max_iters, int max_factor, + const libxl_asyncop_how *ao_how) + LIBXL_EXTERNAL_CALLERS_ONLY; +#endif +#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest
tools/libxl/libxl_dom.c +1 -0
@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live) ?XCFLAGS_LIVE :0 | (debug) ?XCFLAGS_DEBUG :0 + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ) ?XCFLAGS_DOMSAVE_NOSUSPEND :0 | (dss->hvm) ?XCFLAGS_HVM :0 ; dss->suspend_eventchn =-1 ;
tools/libxl/libxl_internal.h +3 -0
@@ -2253,6 +2253,9 @@ struct libxl__domain_suspend_state {xc_evtchn *xce;/* event channel handle */ int suspend_eventchn;int hvm; +int max_iters; +int max_factor; +int xlflags;int xcflags;int guest_responded;const char *dm_savefile;
tools/libxl/xl_cmdimpl.c +28 -5
@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );close (fd);if (rc <0 )
@@ -3332,6 +3332,8 @@ static void migrate_do_preamble(int send }static void migrate_domain (uint32_t domid,const char *rune, +int max_iters,int max_factor, +int no_suspend,const char *override_config_file) {pid_t child =-1 ;
@@ -3341,6 +3343,7 @@ static void migrate_domain(uint32_t domichar rc_buf;uint8_t *config_data;int config_len; +int flags =LIBXL_SUSPEND_LIVE ;save_domain_core_begin (domid, override_config_file, &config_data, &config_len);
@@ -3358,7 +3361,10 @@ static void migrate_domain(uint32_t domixtl_stdiostream_adjust_flags (logger,XTL_STDIOSTREAM_HIDE_PROGRESS ,0 ); - rc =libxl_domain_suspend (ctx, domid, send_fd,LIBXL_SUSPEND_LIVE ,NULL ); +if (no_suspend) + flags |=LIBXL_SUSPEND_NO_FINAL_SUSPEND ; + + rc =libxl_domain_suspend (ctx, domid, send_fd, flags, max_iters, max_factor,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
@@ -3751,8 +3757,16 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , no_suspend =0 ; +static struct option opts[] = { + {"max_iters" ,1 ,0 ,'M' }, + {"max_factor" ,1 ,0 ,'m' }, + {"no_suspend" ,0 ,0 ,'N' }, +COMMON_LONG_OPTS , + {0 ,0 ,0 ,0 } + }; + + SWITCH_FOREACH_OPT(opt, "FC:s:edM:m:N", opts, "migrate", 2) {case 'C' : config_filename = optarg;break ;
@@ -3769,6 +3783,15 @@ int main_migrate(int argc, char **argv)case 'd' : debug =1 ;break ; +case 'M' : + max_iters =atoi (optarg); +break ; +case 'm' : + max_factor =atoi (optarg); +break ; +case 'N' : + no_suspend =1 ; +break ; } domid =find_domain (argv[optind]);
@@ -3784,7 +3807,7 @@ int main_migrate(int argc, char **argv)return 1 ; } -migrate_domain (domid, rune, config_filename); +migrate_domain (domid, rune, max_iters, max_factor, no_suspend, config_filename);return 0 ; }
tools/libxl/xl_cmdtable.c +13 -8
@@ -145,15 +145,20 @@ struct cmd_spec cmd_table[] = { }, {"migrate" , &main_migrate,0 ,1 , -"Save a domain state to restore later" , +"Migrate a domain to <host>" ,"[options] <Domain> <host>" , -"-h Print this help.\n" -"-C <config> Send <config> instead of config file from creation.\n" -"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" -" to sh. If empty, run <host> instead of ssh <host> xl\n" -" migrate-receive [-d -e]\n" -"-e Do not wait in the background (on <host>) for the death\n" -" of the domain." +"-h Print this help.\n" +"-C <config> Send <config> instead of config file from creation.\n" +"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" +" to sh. If empty, run <host> instead of ssh <host> xl\n" +" migrate-receive [-d -e]\n" +"-e Do not wait in the background (on <host>) for the death\n" +" of the domain.\n" +"-M <number> Number of iterations before final suspend (default: 30)\n" +"--max_iters <number>\n" +"-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n" +"--max_factor <factor>\n" +"-N, --no_suspend Abort migration instead of doing final suspend." }, { "dump-core", &main_dump_core, 0, 1,
tools/python/xen/xend/XendCheckpoint.py +12 -2
@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst # enabled. Passing "0" simply uses the defaults compiled into # libxenguest; see the comments and/or code in xc_linux_save() for # more information. + max_iters = dominfo.info.get('max_iters', "0") + max_factor = dominfo.info.get('max_factor', "0") + no_suspend = dominfo.info.get('no_suspend', None) + if max_iters == "None": + max_iters = "0" + if max_factor == "None": + max_factor = "0" + if no_suspend == "None": + no_suspend = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", - str(int(live) | (int(hvm) << 2)) ] + str(dominfo.getDomid()), + max_iters, max_factor, + str(int(live) | (int(hvm) << 2) | (int(no_suspend) << 5) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild):
tools/python/xen/xend/XendDomain.py +12 -0
@@ -1832,6 +1832,18 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) + def domain_migrate_constraints_set((self, domid, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param domid: Domain ID or Name + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + dominfo = self.domain_lookup_nr(domid) + if not dominfo: + raise XendInvalidDomain(str(domid)) + dominfo.setMigrateConstraints(max_iters, max_factor, no_suspend) + def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain.
tools/python/xen/xend/XendDomainInfo.py +12 -0
@@ -1459,6 +1459,18 @@ class XendDomainInfo: pci_conf = self.info['devices'][dev_uuid][1] return map(pci_dict_to_bdf_str, pci_conf['devs']) + def setMigrateConstraints(self, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + log.debug("Setting setMigrateConstraints of domain %s (%s) to '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, no_suspend) + self.info['max_iters'] = str(max_iters) + self.info['max_factor'] = str(max_factor) + self.info['no_suspend'] = str(no_suspend) + def setMemoryTarget(self, target): """Set the memory target of this domain. @param target: In MiB.
tools/python/xen/xm/migrate.py +16 -0
@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c fn=set_true, default=0, use="Change home server for managed domains.") +gopts.opt('max_iters', short='M', val='max_iters', + fn=set_int, default=None, + use="Number of iterations before final suspend (default: 30).") + +gopts.opt('max_factor', short='m', val='max_factor', + fn=set_int, default=None, + use="Max amount of memory to transfer before final suspend (default: 3*RAM).") + +gopts.opt('no_suspend', short='S', + fn=set_true, default=None, + use="Abort migration instead of doing final suspend.") + def help(): return str(gopts)
@@ -80,6 +92,10 @@ def main(argv): server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config) else: + server.xend.domain.migrate_constraints_set(dom, + opts.vals.max_iters, + opts.vals.max_factor, + opts.vals.no_suspend) server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node,
PATCH v3 1/1 tools: set migration constraints from cmdline
In reply to Olaf Hering# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359649053 -3600
# Node ID bd1412703913e2e77ec54da8600887368a893597 # Parent 256d59b2bc8a413876559dc8daf4c52ba46677de
tools: set migration constraints from cmdline
Add new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>
-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM)
--max_factor <factor>
-N Abort migration instead of doing final suspend.
--no_suspend
The changes to libxl change the API, is that approach acceptable?
v3:
- move logic errors in libxl__domain_suspend and fixed help text in cmd_table to separate patches
- fix syntax error in XendCheckpoint.py
- really pass max_iters and max_factor in libxl__xc_domain_save
- make libxl_domain_suspend_0x040200 declaration globally visible
- bump libxenlight.so SONAME from 2.0 to 3.0 due to changed libxl_domain_suspend
v2:
- use LIBXL_API_VERSION and define libxl_domain_suspend_0x040200
- fix logic error in min_reached check in xc_domain_save
- add longopts
- update --help text
- correct description of migrate --help text
tools/libxc/xc_domain_save.c +14 -1
@@ -43,6 +43,8 @@*/ #define DEF_MAX_ITERS 29/* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3/* never send more than 3x p2m_size */ +/* Suspend guest for final transit once number of dirty pages is that low */ +#define DEF_MIN_REMAINING 50struct save_ctx {unsigned long hvirt_start;/* virtual starting address of the hypervisor */
@@ -804,6 +806,7 @@ int xc_domain_save(xc_interface *xch, inint rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG ); +int no_suspend = (flags &XCFLAGS_DOMSAVE_NOSUSPEND );int superpages = !!hvm;int race =0 , sent_last_iter, skip_this_iter =0 ;unsigned int sent_this_iter =0 ;
@@ -1525,10 +1528,20 @@ int xc_domain_save(xc_interface *xch, inif ( live ) { +int min_reached = sent_this_iter + skip_this_iter <DEF_MIN_REMAINING ;if ( (iter >= max_iters) || - (sent_this_iter+skip_this_iter <50 ) || + min_reached || (total_sent > dinfo->p2m_size*max_factor) ) { +if ( !min_reached && no_suspend ) + { +ERROR ("Live migration aborted, as requested. (guest too busy?)" +" total_sent %lu iter %d, max_iters %u max_factor %u" , + total_sent, iter, max_iters, max_factor); + rc =1 ; + goto out; + } +DPRINTF ("Start last iteration\n" ); last_iter =1 ;
tools/libxc/xenguest.h +2 -0
@@ -28,6 +28,8 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8#define XCFLAGS_CHECKPOINT_COMPRESS 16 +#define XCFLAGS_DOMSAVE_NOSUSPEND 32 +#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32
tools/libxl/Makefile +1 -1
@@ -5,7 +5,7 @@ XEN_ROOT = $(CURDIR)/../.. include $(XEN_ROOT)/tools/Rules.mk -MAJOR = 2.0 +MAJOR = 2.1 MINOR = 0 XLUMAJOR = 1.0
tools/libxl/libxl.c +19 -1
@@ -756,7 +756,8 @@ static void domain_suspend_cb(libxl__egc } -int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +static int __libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,int max_iters,int max_factor,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);
@@ -779,6 +780,9 @@ int libxl_domain_suspend(libxl_ctx *ctx, dss->type = type; dss->live = flags &LIBXL_SUSPEND_LIVE ; dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->max_iters = max_iters; + dss->max_factor = max_factor; + dss->xlflags = flags;libxl__domain_suspend (egc, dss);return AO_INPROGRESS ;
@@ -787,6 +791,20 @@ int libxl_domain_suspend(libxl_ctx *ctx,return AO_ABORT (rc); } +int libxl_domain_suspend_0x040200 (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, flags,0 ,0 , ao_how); +} + +int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int max_iters,int max_factor, +const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, flags, max_iters, + max_factor, ao_how); +} +int libxl_domain_pause (libxl_ctx *ctx,uint32_t domid) {int ret;
tools/libxl/libxl.h +14 -1
@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200 +#endif +#else +int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, + int flags, /* LIBXL_SUSPEND_* */ + int max_iters, int max_factor, + const libxl_asyncop_how *ao_how) + LIBXL_EXTERNAL_CALLERS_ONLY; +#endif +#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest
tools/libxl/libxl_dom.c +1 -0
@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live ?XCFLAGS_LIVE :0 ) | (debug ?XCFLAGS_DEBUG :0 ) + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ?XCFLAGS_DOMSAVE_NOSUSPEND :0 ) | (dss->hvm ?XCFLAGS_HVM :0 ); dss->suspend_eventchn =-1 ;
tools/libxl/libxl_internal.h +3 -0
@@ -2253,6 +2253,9 @@ struct libxl__domain_suspend_state {xc_evtchn *xce;/* event channel handle */ int suspend_eventchn;int hvm; +int max_iters; +int max_factor; +int xlflags;int xcflags;int guest_responded;const char *dm_savefile;
tools/libxl/libxl_save_callout.c +2 -2
@@ -108,8 +108,8 @@ void libxl__xc_domain_save(libxl__egc *e }const unsigned long argnums[] = { - dss->domid,0 ,0 , dss->xcflags, dss->hvm, vm_generationid_addr, - toolstack_data_fd, toolstack_data_len, + dss->domid, dss->max_iters, dss->max_factor, dss->xcflags, dss->hvm, + vm_generationid_addr, toolstack_data_fd, toolstack_data_len, cbflags, };
tools/libxl/xl_cmdimpl.c +26 -5
@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );close (fd);if (rc <0 )
@@ -3332,6 +3332,7 @@ static void migrate_do_preamble(int send }static void migrate_domain (uint32_t domid,const char *rune,int debug, +int max_iters,int max_factor,int no_suspend,const char *override_config_file) {pid_t child =-1 ;
@@ -3360,7 +3361,10 @@ static void migrate_domain(uint32_t domiif (debug) flags |=LIBXL_SUSPEND_DEBUG ; - rc =libxl_domain_suspend (ctx, domid, send_fd, flags,NULL ); +if (no_suspend) + flags |=LIBXL_SUSPEND_NO_FINAL_SUSPEND ; + + rc =libxl_domain_suspend (ctx, domid, send_fd, flags, max_iters, max_factor,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
@@ -3753,8 +3757,16 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , no_suspend =0 ; +static struct option opts[] = { + {"max_iters" ,1 ,0 ,'M' }, + {"max_factor" ,1 ,0 ,'m' }, + {"no_suspend" ,0 ,0 ,'N' }, +COMMON_LONG_OPTS , + {0 ,0 ,0 ,0 } + }; + + SWITCH_FOREACH_OPT(opt, "FC:s:edM:m:N", opts, "migrate", 2) {case 'C' : config_filename = optarg;break ;
@@ -3771,6 +3783,15 @@ int main_migrate(int argc, char **argv)case 'd' : debug =1 ;break ; +case 'M' : + max_iters =atoi (optarg); +break ; +case 'm' : + max_factor =atoi (optarg); +break ; +case 'N' : + no_suspend =1 ; +break ; } domid =find_domain (argv[optind]);
@@ -3786,7 +3807,7 @@ int main_migrate(int argc, char **argv)return 1 ; } -migrate_domain (domid, rune, debug, config_filename); +migrate_domain (domid, rune, debug, max_iters, max_factor, no_suspend, config_filename);return 0 ; }
tools/libxl/xl_cmdtable.c +13 -8
@@ -147,14 +147,19 @@ struct cmd_spec cmd_table[] = { &main_migrate,0 ,1 ,"Migrate a domain to another host" ,"[options] <Domain> <host>" , -"-h Print this help.\n" -"-C <config> Send <config> instead of config file from creation.\n" -"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" -" to sh. If empty, run <host> instead of ssh <host> xl\n" -" migrate-receive [-d -e]\n" -"-e Do not wait in the background (on <host>) for the death\n" -" of the domain.\n" -"-d Print huge (!) amount of debug during the migration process." +"-h Print this help.\n" +"-C <config> Send <config> instead of config file from creation.\n" +"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" +" to sh. If empty, run <host> instead of ssh <host> xl\n" +" migrate-receive [-d -e]\n" +"-e Do not wait in the background (on <host>) for the death\n" +" of the domain.\n" +"-d Print huge (!) amount of debug during the migration process.\n" +"-M <number> Number of iterations before final suspend (default: 30)\n" +"--max_iters <number>\n" +"-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n" +"--max_factor <factor>\n" +"-N, --no_suspend Abort migration instead of doing final suspend." }, { "dump-core", &main_dump_core, 0, 1,
tools/python/xen/xend/XendCheckpoint.py +12 -2
@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst # enabled. Passing "0" simply uses the defaults compiled into # libxenguest; see the comments and/or code in xc_linux_save() for # more information. + max_iters = dominfo.info.get('max_iters', "0") + max_factor = dominfo.info.get('max_factor', "0") + no_suspend = dominfo.info.get('no_suspend', None) + if max_iters == "None": + max_iters = "0" + if max_factor == "None": + max_factor = "0" + if no_suspend == "None": + no_suspend = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", - str(int(live) | (int(hvm) << 2)) ] + str(dominfo.getDomid()), + max_iters, max_factor, + str(int(live) | (int(hvm) << 2) | (int(no_suspend) << 5)) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild):
tools/python/xen/xend/XendDomain.py +12 -0
@@ -1832,6 +1832,18 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) + def domain_migrate_constraints_set((self, domid, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param domid: Domain ID or Name + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + dominfo = self.domain_lookup_nr(domid) + if not dominfo: + raise XendInvalidDomain(str(domid)) + dominfo.setMigrateConstraints(max_iters, max_factor, no_suspend) + def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain.
tools/python/xen/xend/XendDomainInfo.py +12 -0
@@ -1459,6 +1459,18 @@ class XendDomainInfo: pci_conf = self.info['devices'][dev_uuid][1] return map(pci_dict_to_bdf_str, pci_conf['devs']) + def setMigrateConstraints(self, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + log.debug("Setting setMigrateConstraints of domain %s (%s) to '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, no_suspend) + self.info['max_iters'] = str(max_iters) + self.info['max_factor'] = str(max_factor) + self.info['no_suspend'] = str(no_suspend) + def setMemoryTarget(self, target): """Set the memory target of this domain. @param target: In MiB.
tools/python/xen/xm/migrate.py +16 -0
@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c fn=set_true, default=0, use="Change home server for managed domains.") +gopts.opt('max_iters', short='M', val='max_iters', + fn=set_int, default=None, + use="Number of iterations before final suspend (default: 30).") + +gopts.opt('max_factor', short='m', val='max_factor', + fn=set_int, default=None, + use="Max amount of memory to transfer before final suspend (default: 3*RAM).") + +gopts.opt('no_suspend', short='S', + fn=set_true, default=None, + use="Abort migration instead of doing final suspend.") + def help(): return str(gopts)
@@ -80,6 +92,10 @@ def main(argv): server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config) else: + server.xend.domain.migrate_constraints_set(dom, + opts.vals.max_iters, + opts.vals.max_factor, + opts.vals.no_suspend) server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node,
PATCH v4 1/1 tools: set migration constraints from cmdline
In reply to Olaf Hering# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359746832 -3600
# Node ID 785c8f34e1f802106e53ca4d2c54dce55c8ee166 # Parent fd711ebdce9a58556d62c2daaf5d49ab06de4a3c
tools: set migration constraints from cmdline
Add new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>
-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM)
--max_factor <factor>
-N Abort migration instead of doing final suspend.
--no_suspend
The changes to libxl change the API, is that approach acceptable?
v4:
- update default for no_suspend from None to 0 in XendCheckpoint.py:save
- update logoutput in setMigrateConstraints
- change xm migrate defaults from None to 0
- add new options to xl.1
- fix syntax error in XendDomain.py:domain_migrate_constraints_set
- fix xm migrate -N option name to match xl migrate
v3:
- move logic errors in libxl__domain_suspend and fixed help text in cmd_table to separate patches
- fix syntax error in XendCheckpoint.py
- really pass max_iters and max_factor in libxl__xc_domain_save
- make libxl_domain_suspend_0x040200 declaration globally visible
- bump libxenlight.so SONAME from 2.0 to 2.1 due to changed libxl_domain_suspend
v2:
- use LIBXL_API_VERSION and define libxl_domain_suspend_0x040200
- fix logic error in min_reached check in xc_domain_save
- add longopts
- update --help text
- correct description of migrate --help text
docs/man/xl.pod.1 +12 -0
@@ -391,6 +391,18 @@ Send <config> instead of config file fro Print huge (!) amount of debug during the migration process. +=item B<-M> I<number>, B<--max_iters> I<number> + +Number of iterations before final suspend (default: 30) + +=item B<-m> I<factor>, B<--max_factor> I<factor> + +Max amount of memory to transfer before final suspend (default: 3*RAM) + +=item B<-N>, B<--no_suspend> + +Abort migration instead of doing final suspend. + =back =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/libxc/xc_domain_save.c +14 -1
@@ -43,6 +43,8 @@*/ #define DEF_MAX_ITERS 29/* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3/* never send more than 3x p2m_size */ +/* Suspend guest for final transit once number of dirty pages is that low */ +#define DEF_MIN_REMAINING 50struct save_ctx {unsigned long hvirt_start;/* virtual starting address of the hypervisor */
@@ -804,6 +806,7 @@ int xc_domain_save(xc_interface *xch, inint rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG ); +int no_suspend = (flags &XCFLAGS_DOMSAVE_NOSUSPEND );int superpages = !!hvm;int race =0 , sent_last_iter, skip_this_iter =0 ;unsigned int sent_this_iter =0 ;
@@ -1525,10 +1528,20 @@ int xc_domain_save(xc_interface *xch, inif ( live ) { +int min_reached = sent_this_iter + skip_this_iter <DEF_MIN_REMAINING ;if ( (iter >= max_iters) || - (sent_this_iter+skip_this_iter <50 ) || + min_reached || (total_sent > dinfo->p2m_size*max_factor) ) { +if ( !min_reached && no_suspend ) + { +ERROR ("Live migration aborted, as requested. (guest too busy?)" +" total_sent %lu iter %d, max_iters %u max_factor %u" , + total_sent, iter, max_iters, max_factor); + rc =1 ; + goto out; + } +DPRINTF ("Start last iteration\n" ); last_iter =1 ;
tools/libxc/xenguest.h +2 -0
@@ -28,6 +28,8 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8#define XCFLAGS_CHECKPOINT_COMPRESS 16 +#define XCFLAGS_DOMSAVE_NOSUSPEND 32 +#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32
tools/libxl/Makefile +1 -1
@@ -5,7 +5,7 @@ XEN_ROOT = $(CURDIR)/../.. include $(XEN_ROOT)/tools/Rules.mk -MAJOR = 2.0 +MAJOR = 2.1 MINOR = 0 XLUMAJOR = 1.0
tools/libxl/libxl.c +19 -1
@@ -756,7 +756,8 @@ static void domain_suspend_cb(libxl__egc } -int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +static int __libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,int max_iters,int max_factor,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);
@@ -779,6 +780,9 @@ int libxl_domain_suspend(libxl_ctx *ctx, dss->type = type; dss->live = flags &LIBXL_SUSPEND_LIVE ; dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->max_iters = max_iters; + dss->max_factor = max_factor; + dss->xlflags = flags;libxl__domain_suspend (egc, dss);return AO_INPROGRESS ;
@@ -787,6 +791,20 @@ int libxl_domain_suspend(libxl_ctx *ctx,return AO_ABORT (rc); } +int libxl_domain_suspend_0x040200 (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, flags,0 ,0 , ao_how); +} + +int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int max_iters,int max_factor, +const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, flags, max_iters, + max_factor, ao_how); +} +int libxl_domain_pause (libxl_ctx *ctx,uint32_t domid) {int ret;
tools/libxl/libxl.h +14 -1
@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200 +#endif +#else +int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, + int flags, /* LIBXL_SUSPEND_* */ + int max_iters, int max_factor, + const libxl_asyncop_how *ao_how) + LIBXL_EXTERNAL_CALLERS_ONLY; +#endif +#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest
tools/libxl/libxl_dom.c +1 -0
@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live ?XCFLAGS_LIVE :0 ) | (debug ?XCFLAGS_DEBUG :0 ) + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ?XCFLAGS_DOMSAVE_NOSUSPEND :0 ) | (dss->hvm ?XCFLAGS_HVM :0 ); dss->suspend_eventchn =-1 ;
tools/libxl/libxl_internal.h +3 -0
@@ -2253,6 +2253,9 @@ struct libxl__domain_suspend_state {xc_evtchn *xce;/* event channel handle */ int suspend_eventchn;int hvm; +int max_iters; +int max_factor; +int xlflags;int xcflags;int guest_responded;const char *dm_savefile;
tools/libxl/libxl_save_callout.c +2 -2
@@ -108,8 +108,8 @@ void libxl__xc_domain_save(libxl__egc *e }const unsigned long argnums[] = { - dss->domid,0 ,0 , dss->xcflags, dss->hvm, vm_generationid_addr, - toolstack_data_fd, toolstack_data_len, + dss->domid, dss->max_iters, dss->max_factor, dss->xcflags, dss->hvm, + vm_generationid_addr, toolstack_data_fd, toolstack_data_len, cbflags, };
tools/libxl/xl_cmdimpl.c +26 -5
@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );close (fd);if (rc <0 )
@@ -3332,6 +3332,7 @@ static void migrate_do_preamble(int send }static void migrate_domain (uint32_t domid,const char *rune,int debug, +int max_iters,int max_factor,int no_suspend,const char *override_config_file) {pid_t child =-1 ;
@@ -3360,7 +3361,10 @@ static void migrate_domain(uint32_t domiif (debug) flags |=LIBXL_SUSPEND_DEBUG ; - rc =libxl_domain_suspend (ctx, domid, send_fd, flags,NULL ); +if (no_suspend) + flags |=LIBXL_SUSPEND_NO_FINAL_SUSPEND ; + + rc =libxl_domain_suspend (ctx, domid, send_fd, flags, max_iters, max_factor,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
@@ -3753,8 +3757,16 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , no_suspend =0 ; +static struct option opts[] = { + {"max_iters" ,1 ,0 ,'M' }, + {"max_factor" ,1 ,0 ,'m' }, + {"no_suspend" ,0 ,0 ,'N' }, +COMMON_LONG_OPTS , + {0 ,0 ,0 ,0 } + }; + + SWITCH_FOREACH_OPT(opt, "FC:s:edM:m:N", opts, "migrate", 2) {case 'C' : config_filename = optarg;break ;
@@ -3771,6 +3783,15 @@ int main_migrate(int argc, char **argv)case 'd' : debug =1 ;break ; +case 'M' : + max_iters =atoi (optarg); +break ; +case 'm' : + max_factor =atoi (optarg); +break ; +case 'N' : + no_suspend =1 ; +break ; } domid =find_domain (argv[optind]);
@@ -3786,7 +3807,7 @@ int main_migrate(int argc, char **argv)return 1 ; } -migrate_domain (domid, rune, debug, config_filename); +migrate_domain (domid, rune, debug, max_iters, max_factor, no_suspend, config_filename);return 0 ; }
tools/libxl/xl_cmdtable.c +13 -8
@@ -147,14 +147,19 @@ struct cmd_spec cmd_table[] = { &main_migrate,0 ,1 ,"Migrate a domain to another host" ,"[options] <Domain> <host>" , -"-h Print this help.\n" -"-C <config> Send <config> instead of config file from creation.\n" -"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" -" to sh. If empty, run <host> instead of ssh <host> xl\n" -" migrate-receive [-d -e]\n" -"-e Do not wait in the background (on <host>) for the death\n" -" of the domain.\n" -"-d Print huge (!) amount of debug during the migration process." +"-h Print this help.\n" +"-C <config> Send <config> instead of config file from creation.\n" +"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" +" to sh. If empty, run <host> instead of ssh <host> xl\n" +" migrate-receive [-d -e]\n" +"-e Do not wait in the background (on <host>) for the death\n" +" of the domain.\n" +"-d Print huge (!) amount of debug during the migration process.\n" +"-M <number> Number of iterations before final suspend (default: 30)\n" +"--max_iters <number>\n" +"-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n" +"--max_factor <factor>\n" +"-N, --no_suspend Abort migration instead of doing final suspend." }, { "dump-core", &main_dump_core, 0, 1,
tools/python/xen/xend/XendCheckpoint.py +12 -2
@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst # enabled. Passing "0" simply uses the defaults compiled into # libxenguest; see the comments and/or code in xc_linux_save() for # more information. + max_iters = dominfo.info.get('max_iters', "0") + max_factor = dominfo.info.get('max_factor', "0") + no_suspend = dominfo.info.get('no_suspend', "0") + if max_iters == "None": + max_iters = "0" + if max_factor == "None": + max_factor = "0" + if no_suspend == "None": + no_suspend = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", - str(int(live) | (int(hvm) << 2)) ] + str(dominfo.getDomid()), + max_iters, max_factor, + str(int(live) | (int(hvm) << 2) | (int(no_suspend) << 5)) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild):
tools/python/xen/xend/XendDomain.py +12 -0
@@ -1832,6 +1832,18 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) + def domain_migrate_constraints_set(self, domid, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param domid: Domain ID or Name + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + dominfo = self.domain_lookup_nr(domid) + if not dominfo: + raise XendInvalidDomain(str(domid)) + dominfo.setMigrateConstraints(max_iters, max_factor, no_suspend) + def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain.
tools/python/xen/xend/XendDomainInfo.py +12 -0
@@ -1459,6 +1459,18 @@ class XendDomainInfo: pci_conf = self.info['devices'][dev_uuid][1] return map(pci_dict_to_bdf_str, pci_conf['devs']) + def setMigrateConstraints(self, max_iters, max_factor, no_suspend): + """Set the Migrate Constraints of this domain. + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param no_suspend: Abort migration instead of doing final suspend + """ + log.debug("Setting migration constraints of domain %s (%s) to '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, no_suspend) + self.info['max_iters'] = str(max_iters) + self.info['max_factor'] = str(max_factor) + self.info['no_suspend'] = str(no_suspend) + def setMemoryTarget(self, target): """Set the memory target of this domain. @param target: In MiB.
tools/python/xen/xm/migrate.py +16 -0
@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c fn=set_true, default=0, use="Change home server for managed domains.") +gopts.opt('max_iters', short='M', val='max_iters', + fn=set_int, default=0, + use="Number of iterations before final suspend (default: 30).") + +gopts.opt('max_factor', short='m', val='max_factor', + fn=set_int, default=0, + use="Max amount of memory to transfer before final suspend (default: 3*RAM).") + +gopts.opt('no_suspend', short='N', + fn=set_true, default=0, + use="Abort migration instead of doing final suspend.") + def help(): return str(gopts)
@@ -80,6 +92,10 @@ def main(argv): server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config) else: + server.xend.domain.migrate_constraints_set(dom, + opts.vals.max_iters, + opts.vals.max_factor, + opts.vals.no_suspend) server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node,
Quoted 14 lines · Olaf Hering
# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359746832 -3600
# Node ID785c8f34e1f802106e53ca4d2c54dce55c8ee166# Parentfd711ebdce9a58556d62c2daaf5d49ab06de4a3c
tools: set migration constraints from cmdlineAdd new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
Review
Quoted 8 lines
(quoted diff) +0
-6-M <number> Number of iterations before final suspend (default: 30) --max_iters <number> -m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM) --max_factor <factor> -N Abort migration instead of doing final suspend. --no_suspend
Is this last one a debug option? (Having read the patch I think not, but the description here doesn't really explain it)
Quoted 1 line
The changes to libxl change the API, is that approach acceptable?
I think this is now along the right lines (see below though).
tools/libxc/xc_domain_save.c
Quoted 51 lines
v4:
- update default forno_suspendfrom None to 0 in XendCheckpoint.py:save
- update logoutput in setMigrateConstraints
- change xm migrate defaults from None to 0
- add new options to xl.1
- fix syntax error in XendDomain.py:domain_migrate_constraints_set
- fix xm migrate -N option name to match xl migratev3:
- move logic errors inlibxl__domain_suspendand fixed help text incmd_tableto separate patches
- fix syntax error inXendCheckpoint.py
- really passmax_itersandmax_factorinlibxl__xc_domain_save
- makelibxl_domain_suspend_0x040200declaration globally visible
- bump libxenlight.so SONAME from 2.0 to 2.1 due to changedlibxl_domain_suspendv2:
- useLIBXL_API_VERSIONand definelibxl_domain_suspend_0x040200
- fix logic error inmin_reachedcheck inxc_domain_save
- add longopts
- update --help text
- correct description of migrate --help textSigned-off-by Olaf Hering <[email protected]>diff -r
fd711ebdce9a-r785c8f34e1f8docs/man/xl.pod.1docs/man/xl.pod.1 +12
-0@@ -391,6 +391,18 @@ Send <config> instead of config file fro Print huge (!) amount of debug during the migration process. +=item B<-M> I<number>, B<--max_iters> I<number> + +Number of iterations before final suspend (default: 30) + +=item B<-m> I<factor>, B<--max_factor> I<factor> + +Max amount of memory to transfer before final suspend (default: 3*RAM) + +=item B<-N>, B<--no_suspend> + +Abort migration instead of doing final suspend. + =back =item B<remus> [I<OPTIONS>] I<domain-id> I<host>diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c +0
-0
The changes to this file only seem to implement the -N and not the other two?
Quoted 5 lines
diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/libxl.htools/libxl/libxl.h +0
-0@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);
tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 509
Quoted 10 lines
tools/libxl/libxl.h +4
-1void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200
int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
int flags, /* LIBXL_SUSPEND_* */ int max_iters, int max_factor, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdef LIBXL_API_VERSION
#if LIBXL_API_VERSION == 0x040200
#define libxl_domain_suspend(ctx, domid, fd, flags, ao_how) \
libxl_domain_suspend(ctx, domid, fd, flags, 0, 0, ao_how)
#endif /* LIBXL_API_VERSION == 0x040200 */
#endif /* defined(LIBXL_API_VERSION) */
Should work?
Not sure if
#if defined(LIBXL_API_VERSION) && LIBXL_API_VERSION == 0x040200 works in CPP.
Maybe we should
#ifndef LIBXL_API_VERSION
#define LIBXL_API_VERSION LIBXL_API_VERSION_LATEST #endif
?
tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 521
Quoted 4 lines
tools/libxl/libxl.h +2
-0+#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4
(These should really be in the IDL, but this is a pre-existing condition)
The name of this new option doesn't quite describe what it does, since it doesn't disable the final suspend always, only under certain condition.
LIBXL_SUSPEND_ABORT_ON_<xxx>
Where the <xxx> is tricky ;-)
<xxx> == DIRTYING_TOO_FAST
<xxx> == GUEST_TOO_BUSY
<xxx> == YOUR_SUGGESTIONS_HERE
Quoted 19 lines
tools/libxl/libxl.h +0
-0/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/libxl_dom.ctools/libxl/libxl_dom.c +1
-0@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live ?XCFLAGS_LIVE :0 ) | (debug ?XCFLAGS_DEBUG :0 ) + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ?XCFLAGS_DOMSAVE_NOSUSPEND :0 ) | (dss->hvm ?XCFLAGS_HVM :0 ); dss->suspend_eventchn =-1 ;diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/xl_cmdimpl.c
tools/libxl/xl_cmdimpl.c static int save_domain(uint32_t domid, c · line 3175
Quoted 5 lines
tools/libxl/xl_cmdimpl.c +1
-1@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );
Doesn't this need to pass down the selected values?
tools/libxl/xl_cmdimpl.c int main_migrate(int argc, char **argv) · line 3764
Quoted 11 lines
tools/libxl/xl_cmdimpl.c +5
-2@@ -3753,8 +3757,16 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , no_suspend =0 ; +static struct option opts[] = { + {"max_iters" ,1 ,0 ,'M' }, + {"max_factor" ,1 ,0 ,'m' }, + {"no_suspend" ,0 ,0 ,'N' },
I think _ in arguments is pretty ugly... But I see we have others already.
tools/python/xen/xend/XendCheckpoint.py
Quoted 3 lines
diff -r
fd711ebdce9a-r785c8f34e1f8tools/python/xen/xend/XendCheckpoint.pytools/python/xen/xend/XendCheckpoint.py +0
-0
I didn't look at any of the python stuff.
Ian.
Quoted 18 lines · Ian Campbell
Quoted 17 lines · Olaf Hering
# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359746832 -3600
# Node ID785c8f34e1f802106e53ca4d2c54dce55c8ee166# Parentfd711ebdce9a58556d62c2daaf5d49ab06de4a3c
tools: set migration constraints from cmdlineAdd new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>
Quoted 8 lines
Review
Quoted 5 lines
(quoted diff) +0
-4-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM) --max_factor <factor> -N Abort migration instead of doing final suspend. --no_suspendIs this last one a debug option? (Having read the patch I think not, but the description here doesn't really explain it)
No, its not debug. And the naming of that -N option is not really good, I agree.
What it is supposed to do is to avoid the final suspend+migrate+resume step when either there were too many iterations or too many pages transfered when the guest continues to dirty many pages. Its not predictable how long the guest will be suspended to transfer the remaining pages to the new host. I think even transfering 1GB RAM at 1000/GBs takes maybe 10 seconds and with large guests there may be several GB dirty pages, so that a guest may be suspended for a minute. This caused issues in a customer environment.
Instead the migration should be aborted and the guest should continue to run on the old host.
Quoted 7 lines
tools/libxc/xc_domain_save.c
Quoted 4 lines
=item B<remus> [I<OPTIONS>] I<domain-id> I<host>
diff -rfd711ebdce9a-r785c8f34e1f8tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c +0
-0The changes to this file only seem to implement the -N and not the other two?
xc_domain_save already has max_iters and max_factor as arguments.
Quoted 28 lines
Quoted 5 lines
diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/libxl.htools/libxl/libxl.h +0
-0@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 509
Quoted 10 lines
tools/libxl/libxl.h +4
-1void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200int libxl_domain_suspend(libxl_ctx *ctx,
uint32_tdomid, int fd,
int flags, /* LIBXL_SUSPEND_* */ intmax_iters, intmax_factor, constlibxl_asyncop_how*ao_how)LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION==0x040200
#definelibxl_domain_suspend(ctx, domid, fd, flags, ao_how)\
libxl_domain_suspend(ctx, domid, fd, flags, 0, 0, ao_how)
#endif /*LIBXL_API_VERSION==0x040200*/
#endif /*defined(LIBXL_API_VERSION)*/
Quoted 1 line
Should work?
That may work, have to try it. In the end if we make such a change it would serve as example for other upcoming API changes.
Quoted 11 lines
tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 521
Quoted 3 lines
tools/libxl/libxl.h +1
-0#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_NO_FINAL_SUSPEND 4(These should really be in the IDL, but this is a pre-existing condition)
The name of this new option doesn't quite describe what it does, since it doesn't disable the final suspend always, only under certain condition.
Quoted 7 lines
LIBXL_SUSPEND_ABORT_ON_<xxx>
Where the <xxx> is tricky ;-)
<xxx> ==
DIRTYING_TOO_FAST
<xxx> ==GUEST_TOO_BUSY
<xxx> ==YOUR_SUGGESTIONS_HERE
Thats alot more descriptive, thanks. DIRTYING_TOO_FAST describes it well I think.
Quoted 19 lines
Quoted 19 lines
tools/libxl/libxl.h +0
-0/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/libxl_dom.ctools/libxl/libxl_dom.c +1
-0@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live ?XCFLAGS_LIVE :0 ) | (debug ?XCFLAGS_DEBUG :0 ) + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ?XCFLAGS_DOMSAVE_NOSUSPEND :0 ) | (dss->hvm ?XCFLAGS_HVM :0 ); dss->suspend_eventchn =-1 ;diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/xl_cmdimpl.ctools/libxl/xl_cmdimpl.c +0
-0@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, c
Quoted 6 lines
tools/libxl/xl_cmdimpl.c static int save_domain(uint32_t domid, c · line 3175
Quoted 4 lines
tools/libxl/xl_cmdimpl.c +1
-1save_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );Doesn't this need to pass down the selected values?
save_domain is different from migrate_domain, I think xl save will suspend the guest anyway? But I notice the -c option for "save", so perhaps it would be useful to catch busy guests as well here?
Olaf
Quoted 41 lines · Olaf Hering
Quoted 18 lines · Ian Campbell
Quoted 17 lines · Olaf Hering
# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1359746832 -3600
# Node ID785c8f34e1f802106e53ca4d2c54dce55c8ee166# Parentfd711ebdce9a58556d62c2daaf5d49ab06de4a3c
tools: set migration constraints from cmdlineAdd new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>Quoted 8 lines
Review
Quoted 5 lines
(quoted diff) +0
-4-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM) --max_factor <factor> -N Abort migration instead of doing final suspend. --no_suspendIs this last one a debug option? (Having read the patch I think not, but the description here doesn't really explain it)
No, its not debug. And the naming of that -N option is not really good, I agree.
What it is supposed to do is to avoid the final suspend+migrate+resume step when either there were too many iterations or too many pages transfered when the guest continues to dirty many pages. Its not predictable how long the guest will be suspended to transfer the remaining pages to the new host. I think even transfering 1GB RAM at 1000/GBs takes maybe 10 seconds and with large guests there may be several GB dirty pages, so that a guest may be suspended for a minute. This caused issues in a customer environment.
Quoted 2 lines
Instead the migration should be aborted and the guest should continue to run on the old host.
That's what I understood from reading the code, can we make the docs say it too ;-)
Quoted 11 lines
Quoted 9 lines
tools/libxc/xc_domain_save.c
Quoted 6 lines
=item B<remus> [I<OPTIONS>] I<domain-id> I<host>
diff -rfd711ebdce9a-r785c8f34e1f8tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c +0
-0The changes to this file only seem to implement the -N and not the other two?
xc_domain_savealready hasmax_itersandmax_factoras arguments.
Ah, I didn't appreciate that.
BTW did you want to add a way to override DEF_MIN_REMAINING?
Quoted 29 lines
Quoted 29 lines
Quoted 6 lines
diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/libxl.htools/libxl/libxl.h +0
-0@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 509
Quoted 10 lines
tools/libxl/libxl.h +4
-1void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200int libxl_domain_suspend(libxl_ctx *ctx,
uint32_tdomid, int fd,
int flags, /* LIBXL_SUSPEND_* */ intmax_iters, intmax_factor, constlibxl_asyncop_how*ao_how)LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION==0x040200
#definelibxl_domain_suspend(ctx, domid, fd, flags, ao_how)\
libxl_domain_suspend(ctx, domid, fd, flags, 0, 0, ao_how)
#endif /*LIBXL_API_VERSION==0x040200*/
#endif /*defined(LIBXL_API_VERSION)*/
Quoted 4 lines
Quoted 1 line
Should work?
That may work, have to try it. In the end if we make such a change it would serve as example for other upcoming API changes.
Agreed.
Quoted 21 lines
Quoted 21 lines
Quoted 21 lines
tools/libxl/libxl.h +0
-0/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/libxl_dom.ctools/libxl/libxl_dom.c +1
-0@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live ?XCFLAGS_LIVE :0 ) | (debug ?XCFLAGS_DEBUG :0 ) + | (dss->xlflags &LIBXL_SUSPEND_NO_FINAL_SUSPEND ?XCFLAGS_DOMSAVE_NOSUSPEND :0 ) | (dss->hvm ?XCFLAGS_HVM :0 ); dss->suspend_eventchn =-1 ;diff -r
fd711ebdce9a-r785c8f34e1f8tools/libxl/xl_cmdimpl.ctools/libxl/xl_cmdimpl.c +0
-0@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len);
Quoted 7 lines
Quoted 4 lines
tools/libxl/xl_cmdimpl.c static int save_domain(uint32_t domid, c · line 3175
Quoted 2 lines
tools/libxl/xl_cmdimpl.c +1
-1-int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,0 ,0 ,0 ,NULL );Doesn't this need to pass down the selected values?
save_domainis different frommigrate_domain, I think xl save will suspend the guest anyway?
Oh yes, I didn't spot this as the save path (despite it being right there in the function name).
Quoted 2 lines
But I notice the -c option for "save", so
perhaps it would be useful to catch busy guests as well here?
I'm not really sure what the usecase is for save -c. I'd be a bit inclined to leave it until someone wants it I think.
Quoted 5 lines · Ian Campbell
Quoted 2 lines
Instead the migration should be aborted and the guest should continue to run on the old host.
That's what I understood from reading the code, can we make the docs say it too ;-)
-A --abort-if-busy maybe?
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
tools/libxc/xc_domain_save.c
Quoted 4 lines
=item B<remus> [I<OPTIONS>] I<domain-id> I<host>
diff -rfd711ebdce9a-r785c8f34e1f8tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c +0
-0The changes to this file only seem to implement the -N and not the other two?
Quoted 5 lines
Quoted 1 line
xc_domain_savealready hasmax_itersandmax_factoras arguments.Ah, I didn't appreciate that.
BTW did you want to add a way to override
DEF_MIN_REMAINING?
I did consider it, but havent looked closer how to do it.
Quoted 2 lines
I'm not really sure what the usecase is for save -c. I'd be a bit inclined to leave it until someone wants it I think.
save -c looks like a snapshot feature, but does not seem to take storage into accound. So yes, I will leave it alone.
Olaf
Quoted 9 lines · Olaf Hering
Quoted 5 lines · Ian Campbell
Quoted 2 lines
Instead the migration should be aborted and the guest should continue to run on the old host.
That's what I understood from reading the code, can we make the docs say it too ;-)
-A --abort-if-busy maybe?
Could work. Not sure if everything needs to have a short option, but I suppose there's no harm.
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
tools/libxc/xc_domain_save.c
Quoted 4 lines
=item B<remus> [I<OPTIONS>] I<domain-id> I<host>
diff -rfd711ebdce9a-r785c8f34e1f8tools/libxc/xc_domain_save.ctools/libxc/xc_domain_save.c +0
-0The changes to this file only seem to implement the -N and not the other two?
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
xc_domain_savealready hasmax_itersandmax_factoras arguments.Ah, I didn't appreciate that.
BTW did you want to add a way to override
DEF_MIN_REMAINING?I did consider it, but havent looked closer how to do it.
Might be best to do while you are changing the API anyway?
It just occurred to me, instead of adding lots of individual arguments perhaps packing them into a (e.g.) libxl_save_properties and adding a pointer would be a nicer and more extensible (in the future) interface?
Quoted 5 lines
Quoted 2 lines
I'm not really sure what the usecase is for save -c. I'd be a bit inclined to leave it until someone wants it I think.
save -c looks like a snapshot feature, but does not seem to take storage into accound. So yes, I will leave it alone.
Ack.
Quoted 3 lines · Ian Campbell
It just occurred to me, instead of adding lots of individual arguments perhaps packing them into a (e.g.)
libxl_save_propertiesand adding a pointer would be a nicer and more extensible (in the future) interface?
Something like this (copy&paste from hg diff)?
I did not find a way to put also libxl_asyncop_how into libxl_save_properties, the checker complains about missing LIBXL_EXTERNAL_CALLERS_ONLY.
Olaf
tools/libxl/libxl.h +12 -6
@@ -500,18 +500,24 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); +typedef struct { +uint32_t domid; +int fd; +int flags;/* LIBXL_SUSPEND_* */ +int max_iters; +int max_factor; +}libxl_save_properties ; + int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ - const libxl_asyncop_how *ao_how) - LIBXL_EXTERNAL_CALLERS_ONLY; + int flags, /* LIBXL_SUSPEND_* */ + const libxl_asyncop_how *ao_how) + LIBXL_EXTERNAL_CALLERS_ONLY;#ifdef LIBXL_API_VERSION #if LIBXL_API_VERSION ==0x040200 #define libxl_domain_suspend libxl_domain_suspend_0x040200#endif #else -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ - int max_iters, int max_factor, +int libxl_domain_suspend(libxl_ctx *ctx, const libxl_save_properties *props, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY;#endif
Quoted 7 lines · Olaf Hering
Quoted 3 lines · Ian Campbell
It just occurred to me, instead of adding lots of individual arguments perhaps packing them into a (e.g.)
libxl_save_propertiesand adding a pointer would be a nicer and more extensible (in the future) interface?Something like this (copy&paste from hg diff)?
It's a tricky balance of what goes in the struct and what goes in the prototype but I didn't imagine putting domid or fd into the struct, domid because that's how other libxl APIs behave and fd because it isn't really a property of the save/migrate as such.
Moving flags into the struct could be optional is it complicates the compat code too much.
Quoted 3 lines
I did not find a way to put also
libxl_asyncop_howintolibxl_save_properties, the checker complains about missingLIBXL_EXTERNAL_CALLERS_ONLY.
I think this is fine to remain in the prototype alongside domid etc.
Quoted 38 lines
Olaf
diff -r
6087ff7a1aeatools/libxl/libxl.htools/libxl/libxl.h +12
-6@@ -500,18 +500,24 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); +typedef struct { +uint32_t domid; +int fd; +int flags;/* LIBXL_SUSPEND_* */ +int max_iters; +int max_factor; +}libxl_save_properties ; + int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ - const libxl_asyncop_how *ao_how) - LIBXL_EXTERNAL_CALLERS_ONLY; + int flags, /* LIBXL_SUSPEND_* */ + const libxl_asyncop_how *ao_how) + LIBXL_EXTERNAL_CALLERS_ONLY;#ifdef LIBXL_API_VERSION #if LIBXL_API_VERSION ==0x040200 #define libxl_domain_suspend libxl_domain_suspend_0x040200#endif #else -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ - int max_iters, int max_factor, +int libxl_domain_suspend(libxl_ctx *ctx, const libxl_save_properties *props, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY;#endif
Quoted 9 lines · Ian Campbell
Quoted 7 lines · Olaf Hering
Quoted 3 lines · Ian Campbell
It just occurred to me, instead of adding lots of individual arguments perhaps packing them into a (e.g.)
libxl_save_propertiesand adding a pointer would be a nicer and more extensible (in the future) interface?Something like this (copy&paste from hg diff)?
Quoted 7 lines
It's a tricky balance of what goes in the struct and what goes in the prototype but I didn't imagine putting domid or fd into the struct, domid because that's how other libxl APIs behave and fd because it isn't really a property of the save/migrate as such.
Moving flags into the struct could be optional is it complicates the compat code too much.
I sent v5 with a new libxl_domain_suspend prototype. If that direction is ok I will look how to improve the LIBXL_API_VERSION handling and how to provide a knob for DEF_MIN_REMAINING.
While I do that I would like to drop the hvm parameter from xc_domain_save because there is a flag for this.
Olaf
Quoted 33 lines · Ian Campbell
tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 509
Quoted 13 lines · Olaf Hering
+++
b/tools/libxl/libxl.htools/libxl/libxl.h +4
-1@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200int libxl_domain_suspend(libxl_ctx *ctx,
uint32_tdomid, int fd,
int flags, /* LIBXL_SUSPEND_* */ intmax_iters, intmax_factor, constlibxl_asyncop_how*ao_how)LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION==0x040200
#definelibxl_domain_suspend(ctx, domid, fd, flags, ao_how)\
libxl_domain_suspend(ctx, domid, fd, flags, 0, 0, ao_how)
#endif /*LIBXL_API_VERSION==0x040200*/
#endif /*defined(LIBXL_API_VERSION)*/Should work?
Not sure if
#ifdefined(LIBXL_API_VERSION)&&LIBXL_API_VERSION==0x040200works in CPP.
Quoted 5 lines
Maybe we should
#ifndefLIBXL_API_VERSION
#defineLIBXL_API_VERSIONLIBXL_API_VERSION_LATEST#endif
?
Here is another attempt to make use of LIBXL_API_VERSION. Two versions just for testing. I would use the static inline variant because we code in C, not cpp.
...
/* API compatibility. */
#ifdef LIBXL_API_VERSION
#if LIBXL_API_VERSION != 0x040200 && LIBXL_API_VERSION != 0x040300 #error Unknown LIBXL_API_VERSION
#endif
#endif
typedef struct {
int xlflags; /* LIBXL_SUSPEND_* */
int max_iters;
int max_factor;
} libxl_save_properties;int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
const libxl_save_properties *props, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdef LIBXL_API_VERSION
#if LIBXL_API_VERSION == 0x040200
#define libxl_domain_suspend(__ctx, __domid, __fd, __flags, __ao_how) \
({ \
libxl_save_properties __props = { .xlflags = (__flags) }; \
int __ret = libxl_domain_suspend((__ctx), (__domid), (__fd), &__props, (__ao_how)); \
__ret; \
})
#elif LIBXL_API_VERSION == 0x040300
static inline int libxl_domain_suspend_0x040300(libxl_ctx *ctx, uint32_t domid, int fd,
int flags, const libxl_asyncop_how *ao_how)
{
libxl_save_properties props = { .xlflags = flags };
return libxl_domain_suspend(ctx, domid, fd, &props, ao_how);
}#define libxl_domain_suspend libxl_domain_suspend_0x040300 #endif
#endif
/* cat > t.c
* gcc -Wall -o t t.c --save-temps -g
* -I tools/libxc -I tools/libxl
* -Ltools/libxc -L tools/libxl -lxenlight
* -Wl,-rpath,$PWD/tools/libxc,-rpath,$PWD/tools/libxl */
#define LIBXL_API_VERSION 0x040300
#include "tools/libxl/libxl.h"
int main(void)
{
int ret;
#if LIBXL_API_VERSION == 0x040200
ret = libxl_domain_suspend(NULL, 0, 0, 0, NULL);
#elif LIBXL_API_VERSION == 0x040300
ret = libxl_domain_suspend(NULL, 0, 0, 0, NULL);
#else
ret = libxl_domain_suspend(NULL, 0, 0, NULL, NULL);
#endif
return ret;
}Olaf
Quoted 54 lines · Olaf Hering
Quoted 33 lines · Ian Campbell
tools/libxl/libxl.h int libxl_domain_create_restore(libxl_ct · line 509
Quoted 13 lines · Olaf Hering
+++
b/tools/libxl/libxl.htools/libxl/libxl.h +4
-1@@ -500,12 +500,25 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); -int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, int flags, /* LIBXL_SUSPEND_* */ const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200int libxl_domain_suspend(libxl_ctx *ctx,
uint32_tdomid, int fd,
int flags, /* LIBXL_SUSPEND_* */ intmax_iters, intmax_factor, constlibxl_asyncop_how*ao_how)LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION==0x040200
#definelibxl_domain_suspend(ctx, domid, fd, flags, ao_how)\
libxl_domain_suspend(ctx, domid, fd, flags, 0, 0, ao_how)
#endif /*LIBXL_API_VERSION==0x040200*/
#endif /*defined(LIBXL_API_VERSION)*/Should work?
Not sure if
#ifdefined(LIBXL_API_VERSION)&&LIBXL_API_VERSION==0x040200works in CPP.Quoted 5 lines
Maybe we should
#ifndefLIBXL_API_VERSION
#defineLIBXL_API_VERSIONLIBXL_API_VERSION_LATEST#endif
?Here is another attempt to make use of
LIBXL_API_VERSION. Two versions just for testing. I would use the static inline variant because we code in C, not cpp....
/* API compatibility. */
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION!=0x040200&&LIBXL_API_VERSION!=0x040300#error UnknownLIBXL_API_VERSION
#endif
#endif
Quoted 2 lines
typedef struct {
int xlflags; /* LIBXL_SUSPEND_* */
Why is this called "xlflags"? xl isn't the only user of this interface.
Quoted 3 lines
int max_iters;
int max_factor;
} libxl_save_properties;s/save/suspend/ to match the function it is passed to? Perhaps libxl_domain_suspend_properties?
Quoted 19 lines
int libxl_domain_suspend(libxl_ctx *ctx,
uint32_tdomid, int fd,
constlibxl_save_properties*props, constlibxl_asyncop_how*ao_how)LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION==0x040200
#definelibxl_domain_suspend(__ctx, __domid, __fd, __flags, __ao_how)\({ \ libxl_save_properties __props = { .xlflags = (__flags) }; \ int __ret = libxl_domain_suspend((__ctx), (__domid), (__fd), &__props, (__ao_how)); \ __ret; \ }) #elif LIBXL_API_VERSION == 0x040300 static inline int libxl_domain_suspend_0x040300(libxl_ctx *ctx, uint32_t domid, int fd, int flags, const libxl_asyncop_how *ao_how) { libxl_save_properties props = { .xlflags = flags }; return libxl_domain_suspend(ctx, domid, fd, &props, ao_how); }
Quoted 1 line
#define
libxl_domain_suspendlibxl_domain_suspend_0x040300
I don't understand the need for this alternative, you are defining version 0x040300 in this patch and in the absence of LIBXL_API_VERSION this should therefore be the expected interface I think. We haven't released Xen 0x040300 yet.
Quoted 31 lines
#endif
#endif/* cat >
t.c
* gcc -Wall -o tt.c--save-temps -g
* -Itools/libxc-Itools/libxl
* -Ltools/libxc -Ltools/libxl-lxenlight
* -Wl,-rpath,$PWD/tools/libxc,-rpath,$PWD/tools/libxl */
#defineLIBXL_API_VERSION0x040300#include "
tools/libxl/libxl.h"int main(void)
{ int ret; #if LIBXL_API_VERSION == 0x040200 ret = libxl_domain_suspend(NULL, 0, 0, 0, NULL); #elif LIBXL_API_VERSION == 0x040300 ret = libxl_domain_suspend(NULL, 0, 0, 0, NULL); #else ret = libxl_domain_suspend(NULL, 0, 0, NULL, NULL);#endif return ret; }Olaf
Quoted 4 lines · Ian Campbell
Quoted 2 lines
typedef struct {
int xlflags; /* LIBXL_SUSPEND_* */Why is this called "xlflags"? xl isn't the only user of this interface.
To make it clear that libxl is the consumer, not libxc. But I think the comment above makes it clear that libxl flags are expected. I will rename it to 'flags'.
Quoted 6 lines
Quoted 3 lines
int max_iters; int max_factor; } libxl_save_properties;s/save/suspend/ to match the function it is passed to? Perhaps
libxl_domain_suspend_properties?
Will rename it to libxl_domain_suspend_properties.
Quoted 21 lines
Quoted 19 lines
int libxl_domain_suspend(libxl_ctx *ctx,
uint32_tdomid, int fd,
constlibxl_save_properties*props, constlibxl_asyncop_how*ao_how)LIBXL_EXTERNAL_CALLERS_ONLY;
#ifdefLIBXL_API_VERSION
#ifLIBXL_API_VERSION==0x040200
#definelibxl_domain_suspend(__ctx, __domid, __fd, __flags, __ao_how)\({ \ libxl_save_properties __props = { .xlflags = (__flags) }; \ int __ret = libxl_domain_suspend((__ctx), (__domid), (__fd), &__props, (__ao_how)); \ __ret; \ }) #elif LIBXL_API_VERSION == 0x040300 static inline int libxl_domain_suspend_0x040300(libxl_ctx *ctx, uint32_t domid, int fd, int flags, const libxl_asyncop_how *ao_how) { libxl_save_properties props = { .xlflags = flags }; return libxl_domain_suspend(ctx, domid, fd, &props, ao_how); }Quoted 1 line
#define
libxl_domain_suspendlibxl_domain_suspend_0x040300
Quoted 4 lines
I don't understand the need for this alternative, you are defining version
0x040300in this patch and in the absence ofLIBXL_API_VERSIONthis should therefore be the expected interface I think. We haven't released Xen0x040300yet.
Sorry, I was in a hurry and did not explain the stuff above properly before sending the mail:
Both versions of a backwards compatible libxl_domain_suspend interface work, they are here just for reference.
I like the static inline variant because its C and it does not depend on the '({ ... })' syntax, which is a gcc feature AFAIK.
Olaf
PATCH v5 1/1 tools: set migration constraints from cmdline
In reply to Olaf Hering# HG changeset patch
# User Olaf Hering <[email protected]>
# Date 1360058505 -3600
# Node ID f474238aa14e1216e297ddb8fa20e8cdbacfa78a # Parent fd711ebdce9a58556d62c2daaf5d49ab06de4a3c
tools: set migration constraints from cmdline
Add new options to xm/xl migrate to control the process of migration. The intention is to optionally abort the migration if it takes too long to migrate a busy guest due to the high number of dirty pages. Currently the guest is suspended to transfer the remaining dirty pages. This transfer can take too long, which can confuse the guest if its suspended for too long.
-M <number> Number of iterations before final suspend (default: 30)
--max_iters <number>
-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM)
--max_factor <factor>
-A Abort migration instead of doing final suspend.
--abort_if_busy
The changes to libxl change the API, is that approach acceptable?
v5:
- adjust libxl_domain_suspend prototype, move flags, max_iters, max_factor into a new, optional struct libxl_save_properties
- rename XCFLAGS_DOMSAVE_NOSUSPEND to XCFLAGS_DOMSAVE_ABORT_IF_BUSY
- rename LIBXL_SUSPEND_NO_FINAL_SUSPEND to LIBXL_SUSPEND_ABORT_IF_BUSY
- rename variables no_suspend to abort_if_busy
- rename option -N/--no_suspend to -A/--abort_if_busy
- update xl.1, extend description of -A option
v4:
- update default for no_suspend from None to 0 in XendCheckpoint.py:save
- update logoutput in setMigrateConstraints
- change xm migrate defaults from None to 0
- add new options to xl.1
- fix syntax error in XendDomain.py:domain_migrate_constraints_set
- fix xm migrate -N option name to match xl migrate
v3:
- move logic errors in libxl__domain_suspend and fixed help text in cmd_table to separate patches
- fix syntax error in XendCheckpoint.py
- really pass max_iters and max_factor in libxl__xc_domain_save
- make libxl_domain_suspend_0x040200 declaration globally visible
- bump libxenlight.so SONAME from 2.0 to 2.1 due to changed libxl_domain_suspend
v2:
- use LIBXL_API_VERSION and define libxl_domain_suspend_0x040200
- fix logic error in min_reached check in xc_domain_save
- add longopts
- update --help text
- correct description of migrate --help text
docs/man/xl.pod.1 +15 -0
@@ -391,6 +391,21 @@ Send <config> instead of config file fro Print huge (!) amount of debug during the migration process. +=item B<-M> I<number>, B<--max_iters> I<number> + +Number of iterations before final suspend (default: 30) + +=item B<-m> I<factor>, B<--max_factor> I<factor> + +Max amount of memory to transfer before final suspend (default: 3*RAM) + +=item B<-A>, B<--abort_if_busy> + +Abort migration instead of doing final suspend/transfer/resume if the +guest has still dirty pages after the number of iterations and/or the +amount of RAM transfered. This avoids long periods of time were the +guest is suspended. + =back =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/libxc/xc_domain_save.c +14 -1
@@ -43,6 +43,8 @@*/ #define DEF_MAX_ITERS 29/* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3/* never send more than 3x p2m_size */ +/* Suspend guest for final transit once number of dirty pages is that low */ +#define DEF_MIN_REMAINING 50struct save_ctx {unsigned long hvirt_start;/* virtual starting address of the hypervisor */
@@ -804,6 +806,7 @@ int xc_domain_save(xc_interface *xch, inint rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG ); +int abort_if_busy = (flags &XCFLAGS_DOMSAVE_ABORT_IF_BUSY );int superpages = !!hvm;int race =0 , sent_last_iter, skip_this_iter =0 ;unsigned int sent_this_iter =0 ;
@@ -1525,10 +1528,20 @@ int xc_domain_save(xc_interface *xch, inif ( live ) { +int min_reached = sent_this_iter + skip_this_iter <DEF_MIN_REMAINING ;if ( (iter >= max_iters) || - (sent_this_iter+skip_this_iter <50 ) || + min_reached || (total_sent > dinfo->p2m_size*max_factor) ) { +if ( !min_reached && abort_if_busy ) + { +ERROR ("Live migration aborted, as requested. (guest too busy?)" +" total_sent %lu iter %d, max_iters %u max_factor %u" , + total_sent, iter, max_iters, max_factor); + rc =1 ; + goto out; + } +DPRINTF ("Start last iteration\n" ); last_iter =1 ;
tools/libxc/xenguest.h +2 -0
@@ -28,6 +28,8 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8#define XCFLAGS_CHECKPOINT_COMPRESS 16 +#define XCFLAGS_DOMSAVE_ABORT_IF_BUSY 32 +#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32
tools/libxl/Makefile +1 -1
@@ -5,7 +5,7 @@ XEN_ROOT = $(CURDIR)/../.. include $(XEN_ROOT)/tools/Rules.mk -MAJOR = 2.0 +MAJOR = 2.1 MINOR = 0 XLUMAJOR = 1.0
tools/libxl/libxl.c +26 -4
@@ -756,8 +756,9 @@ static void domain_suspend_cb(libxl__egc } -int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, -const libxl_asyncop_how *ao_how) +static int __libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +const libxl_save_properties *props, +const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);int rc;
@@ -777,8 +778,13 @@ int libxl_domain_suspend(libxl_ctx *ctx, dss->domid = domid; dss->fd = fd; dss->type = type; - dss->live = flags &LIBXL_SUSPEND_LIVE ; - dss->debug = flags &LIBXL_SUSPEND_DEBUG ; +if (props) { + dss->live = props->xlflags &LIBXL_SUSPEND_LIVE ; + dss->debug = props->xlflags &LIBXL_SUSPEND_DEBUG ; + dss->max_iters = props->max_iters; + dss->max_factor = props->max_factor; + dss->xlflags = props->xlflags; + }libxl__domain_suspend (egc, dss);return AO_INPROGRESS ;
@@ -787,6 +793,22 @@ int libxl_domain_suspend(libxl_ctx *ctx,return AO_ABORT (rc); } +int libxl_domain_suspend_0x040200 (libxl_ctx *ctx,uint32_t domid,int fd, +int flags,const libxl_asyncop_how *ao_how) +{ +libxl_save_properties props = { + .xlflags = flags, + }; +return __libxl_domain_suspend (ctx, domid, fd, &props, ao_how); +} + +int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +const libxl_save_properties *props, +const libxl_asyncop_how *ao_how) +{ +return __libxl_domain_suspend (ctx, domid, fd, props, ao_how); +} +int libxl_domain_pause (libxl_ctx *ctx,uint32_t domid) {int ret;
tools/libxl/libxl.h +19 -1
@@ -500,12 +500,30 @@ int libxl_domain_create_restore(libxl_ctvoid libxl_domain_config_init (libxl_domain_config *d_config);void libxl_domain_config_dispose (libxl_domain_config *d_config); +typedef struct { +int xlflags;/* LIBXL_SUSPEND_* */ +int max_iters; +int max_factor; +}libxl_save_properties ; + +int libxl_domain_suspend_0x040200(libxl_ctx *ctx, uint32_t domid, int fd, + int flags, /* LIBXL_SUSPEND_* */ + const libxl_asyncop_how *ao_how) + LIBXL_EXTERNAL_CALLERS_ONLY; +#ifdef LIBXL_API_VERSION +#if LIBXL_API_VERSION ==0x040200 +#define libxl_domain_suspend libxl_domain_suspend_0x040200 +#endif +#else int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ + const libxl_save_properties *props, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; +#endif +#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_ABORT_IF_BUSY 4/* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )] * If this parameter is true, use co-operative resume. The guest
tools/libxl/libxl_dom.c +1 -0
@@ -1240,6 +1240,7 @@ void libxl__domain_suspend(libxl__egc *e dss->xcflags = (live ?XCFLAGS_LIVE :0 ) | (debug ?XCFLAGS_DEBUG :0 ) + | (dss->xlflags &LIBXL_SUSPEND_ABORT_IF_BUSY ?XCFLAGS_DOMSAVE_ABORT_IF_BUSY :0 ) | (dss->hvm ?XCFLAGS_HVM :0 ); dss->suspend_eventchn =-1 ;
tools/libxl/libxl_internal.h +3 -0
@@ -2253,6 +2253,9 @@ struct libxl__domain_suspend_state {xc_evtchn *xce;/* event channel handle */ int suspend_eventchn;int hvm; +int max_iters; +int max_factor; +int xlflags;int xcflags;int guest_responded;const char *dm_savefile;
tools/libxl/libxl_save_callout.c +2 -2
@@ -108,8 +108,8 @@ void libxl__xc_domain_save(libxl__egc *e }const unsigned long argnums[] = { - dss->domid,0 ,0 , dss->xcflags, dss->hvm, vm_generationid_addr, - toolstack_data_fd, toolstack_data_len, + dss->domid, dss->max_iters, dss->max_factor, dss->xcflags, dss->hvm, + vm_generationid_addr, toolstack_data_fd, toolstack_data_len, cbflags, };
tools/libxl/xl_cmdimpl.c +33 -7
@@ -3172,7 +3172,7 @@ static int save_domain(uint32_t domid, csave_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd,NULL ,NULL );close (fd);if (rc <0 )
@@ -3332,6 +3332,7 @@ static void migrate_do_preamble(int send }static void migrate_domain (uint32_t domid,const char *rune,int debug, +int max_iters,int max_factor,int abort_if_busy,const char *override_config_file) {pid_t child =-1 ;
@@ -3340,7 +3341,12 @@ static void migrate_domain(uint32_t domichar *away_domname;char rc_buf;uint8_t *config_data; -int config_len, flags =LIBXL_SUSPEND_LIVE ; +int config_len; +libxl_save_properties props = { + .xlflags =LIBXL_SUSPEND_LIVE , + .max_iters = max_iters, + .max_factor = max_factor, + };save_domain_core_begin (domid, override_config_file, &config_data, &config_len);
@@ -3359,8 +3365,11 @@ static void migrate_domain(uint32_t domixtl_stdiostream_adjust_flags (logger,XTL_STDIOSTREAM_HIDE_PROGRESS ,0 );if (debug) - flags |=LIBXL_SUSPEND_DEBUG ; - rc =libxl_domain_suspend (ctx, domid, send_fd, flags,NULL ); + props.xlflags |=LIBXL_SUSPEND_DEBUG ; +if (abort_if_busy) + props.xlflags |=LIBXL_SUSPEND_ABORT_IF_BUSY ; + + rc =libxl_domain_suspend (ctx, domid, send_fd, &props,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
@@ -3753,8 +3762,16 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 ; - - SWITCH_FOREACH_OPT(opt, "FC:s:ed", NULL, "migrate", 2) { +int max_iters =0 , max_factor =0 , abort_if_busy =0 ; +static struct option opts[] = { + {"max_iters" ,1 ,0 ,'M' }, + {"max_factor" ,1 ,0 ,'m' }, + {"abort_if_busy" ,0 ,0 ,'A' }, +COMMON_LONG_OPTS , + {0 ,0 ,0 ,0 } + }; + + SWITCH_FOREACH_OPT(opt, "FC:s:edM:m:A", opts, "migrate", 2) {case 'C' : config_filename = optarg;break ;
@@ -3771,6 +3788,15 @@ int main_migrate(int argc, char **argv)case 'd' : debug =1 ;break ; +case 'M' : + max_iters =atoi (optarg); +break ; +case 'm' : + max_factor =atoi (optarg); +break ; +case 'A' : + abort_if_busy =1 ; +break ; } domid =find_domain (argv[optind]);
@@ -3786,7 +3812,7 @@ int main_migrate(int argc, char **argv)return 1 ; } -migrate_domain (domid, rune, debug, config_filename); +migrate_domain (domid, rune, debug, max_iters, max_factor, abort_if_busy, config_filename);return 0 ; }
tools/libxl/xl_cmdtable.c +13 -8
@@ -147,14 +147,19 @@ struct cmd_spec cmd_table[] = { &main_migrate,0 ,1 ,"Migrate a domain to another host" ,"[options] <Domain> <host>" , -"-h Print this help.\n" -"-C <config> Send <config> instead of config file from creation.\n" -"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" -" to sh. If empty, run <host> instead of ssh <host> xl\n" -" migrate-receive [-d -e]\n" -"-e Do not wait in the background (on <host>) for the death\n" -" of the domain.\n" -"-d Print huge (!) amount of debug during the migration process." +"-h Print this help.\n" +"-C <config> Send <config> instead of config file from creation.\n" +"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" +" to sh. If empty, run <host> instead of ssh <host> xl\n" +" migrate-receive [-d -e]\n" +"-e Do not wait in the background (on <host>) for the death\n" +" of the domain.\n" +"-d Print huge (!) amount of debug during the migration process.\n" +"-M <number> Number of iterations before final suspend (default: 30)\n" +"--max_iters <number>\n" +"-m <factor> Max amount of memory to transfer before final suspend (default: 3*RAM).\n" +"--max_factor <factor>\n" +"-A, --abort_if_busy Abort migration instead of doing final suspend." }, { "dump-core", &main_dump_core, 0, 1,
tools/python/xen/xend/XendCheckpoint.py +12 -2
@@ -118,9 +118,19 @@ def save(fd, dominfo, network, live, dst # enabled. Passing "0" simply uses the defaults compiled into # libxenguest; see the comments and/or code in xc_linux_save() for # more information. + max_iters = dominfo.info.get('max_iters', "0") + max_factor = dominfo.info.get('max_factor', "0") + abort_if_busy = dominfo.info.get('abort_if_busy', "0") + if max_iters == "None": + max_iters = "0" + if max_factor == "None": + max_factor = "0" + if abort_if_busy == "None": + abort_if_busy = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), - str(dominfo.getDomid()), "0", "0", - str(int(live) | (int(hvm) << 2)) ] + str(dominfo.getDomid()), + max_iters, max_factor, + str(int(live) | (int(hvm) << 2) | (int(abort_if_busy) << 5)) ] log.debug("[xc_save]: %s", string.join(cmd)) def saveInputHandler(line, tochild):
tools/python/xen/xend/XendDomain.py +12 -0
@@ -1832,6 +1832,18 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) + def domain_migrate_constraints_set(self, domid, max_iters, max_factor, abort_if_busy): + """Set the Migrate Constraints of this domain. + @param domid: Domain ID or Name + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param abort_if_busy: Abort migration instead of doing final suspend + """ + dominfo = self.domain_lookup_nr(domid) + if not dominfo: + raise XendInvalidDomain(str(domid)) + dominfo.setMigrateConstraints(max_iters, max_factor, abort_if_busy) + def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain.
tools/python/xen/xend/XendDomainInfo.py +12 -0
@@ -1459,6 +1459,18 @@ class XendDomainInfo: pci_conf = self.info['devices'][dev_uuid][1] return map(pci_dict_to_bdf_str, pci_conf['devs']) + def setMigrateConstraints(self, max_iters, max_factor, abort_if_busy): + """Set the Migrate Constraints of this domain. + @param max_iters: Number of iterations before final suspend + @param max_factor: Max amount of memory to transfer before final suspend + @param abort_if_busy: Abort migration instead of doing final suspend + """ + log.debug("Setting migration constraints of domain %s (%s) to '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, abort_if_busy) + self.info['max_iters'] = str(max_iters) + self.info['max_factor'] = str(max_factor) + self.info['abort_if_busy'] = str(abort_if_busy) + def setMemoryTarget(self, target): """Set the memory target of this domain. @param target: In MiB.
tools/python/xen/xm/migrate.py +16 -0
@@ -55,6 +55,18 @@ gopts.opt('change_home_server', short='c fn=set_true, default=0, use="Change home server for managed domains.") +gopts.opt('max_iters', short='M', val='max_iters', + fn=set_int, default=0, + use="Number of iterations before final suspend (default: 30).") + +gopts.opt('max_factor', short='m', val='max_factor', + fn=set_int, default=0, + use="Max amount of memory to transfer before final suspend (default: 3*RAM).") + +gopts.opt('abort_if_busy', short='A', + fn=set_true, default=0, + use="Abort migration instead of doing final suspend.") + def help(): return str(gopts)
@@ -80,6 +92,10 @@ def main(argv): server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config) else: + server.xend.domain.migrate_constraints_set(dom, + opts.vals.max_iters, + opts.vals.max_factor, + opts.vals.abort_if_busy) server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node,