leftover from 2020
v20210111 cover leftover from 2020
Various unreviewed changes.
Olaf Hering (39):
stubdom: fix tpm_version
xl: use proper name for bash_completion file
docs: remove stale create example from xl.1
docs: substitute XEN_CONFIG_DIR in xl.conf.5
tools: add with-xen-scriptdir configure option
Use XEN_SCRIPT_DIR to refer to /etc/xen/scripts
xl: optionally print timestamps during xl migrate
xl: fix description of migrate --debug
tools: add readv_exact to libxenctrl
tools: add xc_is_known_page_type to libxenctrl
tools: use xc_is_known_page_type
tools: unify type checking for data pfns in migration stream
tools: show migration transfer rate in send_dirty_pages tools/guest: prepare to allocate arrays once tools/guest: save: move batch_pfns
tools/guest: save: move mfns array
tools/guest: save: move types array
tools/guest: save: move errors array
tools/guest: save: move iov array
tools/guest: save: move rec_pfns array
tools/guest: save: move guest_data array
tools/guest: save: move local_pages array
tools/guest: restore: move pfns array
tools/guest: restore: move types array
tools/guest: restore: move mfns array
tools/guest: restore: move map_errs array
tools/guest: restore: move mfns array in populate_pfns tools/guest: restore: move pfns array in populate_pfns tools/guest: restore: split record processing tools/guest: restore: split handle_page_data tools/guest: restore: write data directly into guest
tools: remove tabs from code produced by libxl_save_msgs_gen.pl
tools: recognize LIBXL_API_VERSION for 4.15
tools: adjust libxl_domain_suspend to receive a struct props
tools: change struct precopy_stats to precopy_stats_t
tools: add callback to libxl for precopy_policy and precopy_stats_t
tools: add --max_iters to libxl_domain_suspend
tools: add --min_remaining to libxl_domain_suspend
tools: add --abort_if_busy to libxl_domain_suspend
.gitignore | 3 +
docs/configure.ac | 3 +
...n.5.pod => xl-disk-configuration.5.pod.in} | 2 +-
....pod => xl-network-configuration.5.pod.in} | 4 +-
docs/man/xl.1.pod.in | 39 +-
docs/man/{xl.conf.5.pod => xl.conf.5.pod.in} | 8 +-
docs/misc/block-scripts.txt | 2 +-
m4/paths.m4 | 8 +-
stubdom/vtpmmgr/vtpmmgr.h | 2 +-
tools/include/libxl.h | 32 +-
tools/include/xenguest.h | 7 +-
tools/libs/ctrl/xc_private.c | 55 +-
tools/libs/ctrl/xc_private.h | 34 ++
tools/libs/guest/xg_sr_common.c | 33 +-
tools/libs/guest/xg_sr_common.h | 88 ++-
tools/libs/guest/xg_sr_restore.c | 562 ++++++++++++------
tools/libs/guest/xg_sr_save.c | 164 ++---
tools/libs/guest/xg_sr_save_x86_hvm.c | 5 +-
tools/libs/guest/xg_sr_save_x86_pv.c | 31 +-
tools/libs/light/libxl_dom_save.c | 24 +
tools/libs/light/libxl_domain.c | 10 +-
tools/libs/light/libxl_internal.h | 6 +
tools/libs/light/libxl_save_msgs_gen.pl | 23 +-
tools/libs/light/libxl_stream_write.c | 9 +-
tools/libs/light/libxl_types.idl | 1 +
tools/ocaml/libs/xl/xenlight_stubs.c | 3 +-
tools/xl/Makefile | 4 +-
tools/xl/bash-completion | 2 +-
tools/xl/xl_cmdtable.c | 29 +-
tools/xl/xl_migrate.c | 79 ++-
tools/xl/xl_saverestore.c | 3 +-
31 files changed, 901 insertions(+), 374 deletions(-)
rename docs/man/{xl-disk-configuration.5.pod => xl-disk-configuration.5.pod.in} (99%)
rename docs/man/{xl-network-configuration.5.pod => xl-network-configuration.5.pod.in} (98%)
rename docs/man/{xl.conf.5.pod => xl.conf.5.pod.in} (96%)PATCH v20210111 1/39 stubdom: fix tpm_version
It is just a declaration, not a variable.
ld: /home/abuild/rpmbuild/BUILD/xen-4.14.20200616T103126.3625b04991/non-dbg/stubdom/vtpmmgr/vtpmmgr.a(vtpm_cmd_handler.o):(.bss+0x0): multiple definition of `tpm_version'; /home/abuild/rpmbuild/BUILD/xen-4.14.20200616T103126.3625b04991/non-dbg/stubdom/vtpmmgr/vtpmmgr.a(vtpmmgr.o):(.bss+0x0): first defined here
--- stubdom/vtpmmgr/vtpmmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
stubdom/vtpmmgr/vtpmmgr.h +1 -1
@@ -53,7 +53,7 @@enum {TPM1_HARDWARE =1 ,TPM2_HARDWARE , -} tpm_version; +};struct tpm_hardware_version {int hw_version;
Re: [PATCH v20210111 01/39] stubdom: fix tpm_version
Olaf Hering, le lun. 11 janv. 2021 18:41:46 +0100, a ecrit:
Quoted 5 lines
It is just a declaration, not a variable.
ld: /home/abuild/rpmbuild/BUILD/xen-4.14.20200616T103126.3625b04991/non-dbg/stubdom/vtpmmgr/vtpmmgr.a(vtpm_cmd_handler.o):(.bss+0x0): multiple definition of `tpm_version'; /home/abuild/rpmbuild/BUILD/xen-4.14.20200616T103126.3625b04991/non-dbg/stubdom/vtpmmgr/vtpmmgr.a(vtpmmgr.o):(.bss+0x0): first defined here
Signed-off-by Olaf Hering <[email protected]>
Quoted 18 lines
--- stubdom/vtpmmgr/vtpmmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)stubdom/vtpmmgr/vtpmmgr.h +1
-1@@ -53,7 +53,7 @@enum {TPM1_HARDWARE =1 ,TPM2_HARDWARE , -} tpm_version; +};struct tpm_hardware_version {int hw_version;
PATCH v20210111 2/39 xl: use proper name for bash_completion file
In reply to Olaf HeringFiles in the bash-completion dirs should be named like the commands, without suffix. Without this change 'xl' will not be recognized as a command with completion support if BASH_COMPLETION_DIR is set to /usr/share/bash-completion/completions.
Fixes commit 9136a919b19929ecb242ef327053d55d824397df
--- tools/xl/Makefile | 4 ++-- tools/xl/bash-completion | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
tools/xl/Makefile +2 -2
@@ -45,11 +45,11 @@ install: all $(INSTALL_DIR) $(DESTDIR)$(sbindir) $(INSTALL_DIR) $(DESTDIR)$(BASH_COMPLETION_DIR) $(INSTALL_PROG) xl $(DESTDIR)$(sbindir) - $(INSTALL_DATA) bash-completion $(DESTDIR)$(BASH_COMPLETION_DIR)/xl.sh + $(INSTALL_DATA) bash-completion $(DESTDIR)$(BASH_COMPLETION_DIR)/xl .PHONY: uninstall uninstall: - rm -f $(DESTDIR)$(BASH_COMPLETION_DIR)/xl.sh + rm -f $(DESTDIR)$(BASH_COMPLETION_DIR)/xl rm -f $(DESTDIR)$(sbindir)/xl .PHONY: clean
tools/xl/bash-completion +1 -1
@@ -1,4 +1,4 @@ -# Copy this file to /etc/bash_completion.d/xl.sh +# Copy this file to /etc/bash_completion.d/xl _xl() {
Re: [PATCH v20210111 02/39] xl: use proper name for bash_completion file
Olaf Hering writes ("[PATCH v20210111 02/39] xl: use proper name for bash_completion file"):
Quoted 6 lines
Files in the bash-completion dirs should be named like the commands, without suffix. Without this change 'xl' will not be recognized as a command with completion support if
BASH_COMPLETION_DIRis set to/usr/share/bash-completion/completions.Fixes commit
9136a919b19929ecb242ef327053d55d824397df
This was committed already.
Ian.
PATCH v20210111 3/39 docs: remove stale create example from xl.1
In reply to Olaf HeringMaybe xm create had a feature to create a domU based on a configuration file. xl create requires the '-f' option to refer to a file. There is no code to look into XEN_CONFIG_DIR, so remove the example.
--- docs/man/xl.1.pod.in | 7 ------- 1 file changed, 7 deletions(-)
docs/man/xl.1.pod.in +0 -7
@@ -171,13 +171,6 @@ B<EXAMPLES> =over 4 -=item I<with config file> - - xl create DebianLenny - -This creates a domain with the file /etc/xen/DebianLenny, and returns as -soon as it is run. - =item I<with extra parameters> xl create hvm.cfg 'cpus="0-3"; pci=["01:05.1","01:05.2"]'
Re: [PATCH v20210111 03/39] docs: remove stale create example from xl.1
Olaf Hering writes ("[PATCH v20210111 03/39] docs: remove stale create example from xl.1"):
Quoted 3 lines
Maybe xm create had a feature to create a domU based on a configuration file. xl create requires the '-f' option to refer to a file. There is no code to look into
XEN_CONFIG_DIR, so remove the example.
This series seems to contain a number of patches which seem to have slipped through the net before the freeze, despite some having reviews even :-/.
I'm going to go through them and look at the status of each and hope to commit and/or review and/or release-ack as many of them as possible.
Ian.
Re: [PATCH v20210111 03/39] docs: remove stale create example from xl.1
In reply to Olaf HeringOlaf Hering writes ("[PATCH v20210111 03/39] docs: remove stale create example from xl.1"):
Quoted 3 lines
Maybe xm create had a feature to create a domU based on a configuration file. xl create requires the '-f' option to refer to a file. There is no code to look into
XEN_CONFIG_DIR, so remove the example.
This is docs so
I'm queueing it now.
Ian.
PATCH v20210111 4/39 docs: substitute XEN_CONFIG_DIR in xl.conf.5
In reply to Olaf Heringxl(1) opens xl.conf in XEN_CONFIG_DIR.
Substitute this variable also in the man page.
--- docs/man/xl.1.pod.in | 2 +- docs/man/xl.conf.5.pod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
docs/man/xl.1.pod.in +1 -1
@@ -50,7 +50,7 @@ setup the bridge. If you specify the amount of memory dom0 has, passing B<dom0_mem> to Xen, it is highly recommended to disable B<autoballoon>. Edit -B</etc/xen/xl.conf> and set it to 0. +B<@XEN_CONFIG_DIR@/xl.conf> and set it to 0. =item run xl as B<root>
docs/man/xl.conf.5.pod +1 -1
@@ -1,6 +1,6 @@ =head1 NAME -/etc/xen/xl.conf - XL Global/Host Configuration +@XEN_CONFIG_DIR@/xl.conf - XL Global/Host Configuration =head1 DESCRIPTION
Re: [PATCH v20210111 04/39] docs: substitute XEN_CONFIG_DIR in xl.conf.5
Olaf Hering writes ("[PATCH v20210111 04/39] docs: substitute XEN_CONFIG_DIR in xl.conf.5"):
Quoted 5 lines
xl(1) opens xl.conf in
XEN_CONFIG_DIR.
Substitute this variable also in the man page.Signed-off-by Olaf Hering <[email protected]>Reviewed-by Anthony PERARD <[email protected]>
This is docs. Worst risk is slight chance of breaking the build.
Ian.
PATCH v20210111 5/39 tools: add with-xen-scriptdir configure option
In reply to Olaf HeringIn the near future all fresh installations will have an empty /etc. The content of this directory will not be controlled by the package manager anymore. One of the reasons for this move is to make snapshots more robust.
As a first step into this direction, add a knob to configure to allow storing the hotplug scripts to libexec because they are not exactly configuration. The current default is unchanged, which is /etc/xen/scripts.
--- m4/paths.m4 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
m4/paths.m4 +7 -1
@@ -70,6 +70,12 @@ AC_ARG_WITH([libexec-leaf-dir], [libexec_subdir=$withval], [libexec_subdir=$PACKAGE_TARNAME]) +AC_ARG_WITH([xen-scriptdir], + AS_HELP_STRING([--with-xen-scriptdir=DIR], + [Path to directory for dom0 hotplug scripts. [SYSCONFDIR/xen/scripts]]), + [xen_scriptdir_path=$withval], + [xen_scriptdir_path=$sysconfdir/xen/scripts]) + AC_ARG_WITH([xen-dumpdir], AS_HELP_STRING([--with-xen-dumpdir=DIR], [Path to directory for domU crash dumps. [LOCALSTATEDIR/lib/xen/dump]]),
@@ -137,7 +143,7 @@ AC_SUBST(INITD_DIR) XEN_CONFIG_DIR=$CONFIG_DIR/xen AC_SUBST(XEN_CONFIG_DIR) -XEN_SCRIPT_DIR=$XEN_CONFIG_DIR/scripts +XEN_SCRIPT_DIR=$xen_scriptdir_path AC_SUBST(XEN_SCRIPT_DIR) case "$host_os" in
Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option
Olaf Hering writes ("[PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option"):
Quoted 4 lines
In the near future all fresh installations will have an empty /etc. The content of this directory will not be controlled by the package manager anymore. One of the reasons for this move is to make snapshots more robust.
As I wrote in September 2019 I don't agree with it, if it's intended as a claim about all systems that Xen will run on.
However, this seems to be an accidental retention in the commit message, since the content of the commit is now what I asked for, which is to make this directory configurable.
So I approve of the contents of this patch in principle.
Quoted 4 lines
As a first step into this direction, add a knob to configure to allow storing the hotplug scripts to libexec because they are not exactly configuration. The current default is unchanged, which is
/etc/xen/scripts.
I suggest this commit message as a compromise:
Some distros plan for fresh installations will have an empty /etc, whose content will not be controlled by the package manager anymore.
To make this possible, add a knob to configure to allow storing the hotplug scripts to libexec instead of /etc/xen/scripts.
Olaf, would that be OK with you ?
As for detailed review:
Quoted 4 lines
m4/paths.m4 +0
-0
...
Quoted 5 lines
m4/paths.m4 +5
-0+AC_ARG_WITH([xen-scriptdir], + AS_HELP_STRING([--with-xen-scriptdir=DIR], + [Path to directory for dom0 hotplug scripts. [SYSCONFDIR/xen/scripts]]), + [xen_scriptdir_path=$withval], + [xen_scriptdir_path=$sysconfdir/xen/scripts])
...
m4/paths.m4 AC_SUBST(INITD_DIR) · line 147
Quoted 3 lines
m4/paths.m4 +1
-1-XEN_SCRIPT_DIR=$XEN_CONFIG_DIR/scripts +XEN_SCRIPT_DIR=$xen_scriptdir_path AC_SUBST(XEN_SCRIPT_DIR)
It is not clear to me why the deefault is changed from "$XEN_CONFIG_DIR/scripts" to "$sysconfdir/xen/scripts" and there isn't any explanation for this in the commit message. I think this may make no difference but an explanation is called for.
As for the release ack question: there is a risk that this patch would break the build, by causing the scripts to go somewhere wrong. This risk seems fairly low and osstest would catch it.
However, I find it difficult to analyse the consequence of the changed way the default is calculated.
So, a conditional release ack:
subject to changing this patch to make the actual implemented default to still be $XEN_CONFIG_DIR/scripts.
Ian.
Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option
Am Mon, 8 Feb 2021 16:03:29 +0000
schrieb Ian Jackson <[email protected]>:
Quoted 10 lines
I suggest this commit message as a compromise:
Some distros plan for fresh installations will have an empty /etc, whose content will not be controlled by the package manager anymore.
To make this possible, add a knob to configure to allow storing the hotplug scripts to libexec instead of
/etc/xen/scripts.Olaf, would that be OK with you ?
Yes, this is fine. Thanks.
Quoted 13 lines
As for detailed review:
Quoted 4 lines
m4/paths.m4 +0
-0...
Quoted 5 lines
m4/paths.m4 +5
-0+AC_ARG_WITH([xen-scriptdir], + AS_HELP_STRING([--with-xen-scriptdir=DIR], + [Path to directory for dom0 hotplug scripts. [SYSCONFDIR/xen/scripts]]), + [xen_scriptdir_path=$withval], + [xen_scriptdir_path=$sysconfdir/xen/scripts])...
Quoted 8 lines
m4/paths.m4 AC_SUBST(INITD_DIR) · line 147
Quoted 3 lines
m4/paths.m4 +1
-1-XEN_SCRIPT_DIR=$XEN_CONFIG_DIR/scripts +XEN_SCRIPT_DIR=$xen_scriptdir_path AC_SUBST(XEN_SCRIPT_DIR)It is not clear to me why the deefault is changed from "$XEN_CONFIG_DIR/scripts" to "$sysconfdir/xen/scripts" and there isn't any explanation for this in the commit message. I think this may make no difference but an explanation is called for.
The reason is the ordering of assignments in the file:AC_ARG_WITH comes early in the file, XEN_CONFIG_DIR= is assigned much later.
It seems the assignments for CONFIG_DIR and XEN_CONFIG_DIR can be moved up, because $sysconfdir is expected to be set already. As a result the new AC_ARG_WITH= can continue to use "$XEN_CONFIG_DIR/scripts" for the default case. I assume the current ordering is to have a separate AC_ARG_WITH and AC_SUBST section.
Olaf
Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option
Olaf Hering writes ("Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option"):
Quoted 2 lines
The reason is the ordering of assignments in the file:
AC_ARG_WITHcomes early in the file, XEN_CONFIG_DIR= is assigned much later.
Ah.
Quoted 1 line
It seems the assignments for
CONFIG_DIRandXEN_CONFIG_DIRcan be moved up, because $sysconfdir is expected to be set already. As a result the new AC_ARG_WITH= can continue to use "$XEN_CONFIG_DIR/scripts" for the default case.
That would be best I think.
Quoted 1 line
I assume the current ordering is to have a separate
AC_ARG_WITHandAC_SUBSTsection.
I could be wrong, but I don't think the location of AC_SUBST matters.
Ian.
Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option
Am Mon, 8 Feb 2021 17:48:31 +0000
schrieb Ian Jackson <[email protected]>:
Quoted 3 lines
Quoted 1 line
It seems the assignments for
CONFIG_DIRandXEN_CONFIG_DIRcan be moved up, because $sysconfdir is expected to be set already. As a result the new AC_ARG_WITH= can continue to use "$XEN_CONFIG_DIR/scripts" for the default case.That would be best I think.
I will split this into individual changes and send a separate series.
Olaf
Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option
Olaf Hering writes ("Re: [PATCH v20210111 05/39] tools: add with-xen-scriptdir configure option"):
Quoted 7 lines
Am Mon, 8 Feb 2021 17:48:31 +0000
schrieb Ian Jackson <[email protected]>:Quoted 3 lines
Quoted 1 line
It seems the assignments for
CONFIG_DIRandXEN_CONFIG_DIRcan be moved up, because $sysconfdir is expected to be set already. As a result the new AC_ARG_WITH= can continue to use "$XEN_CONFIG_DIR/scripts" for the default case.That would be best I think.
I will split this into individual changes and send a separate series.
OK, thanks, that will help with de-risking this from a release PoV.
Ian.
PATCH v20210111 6/39 Use XEN_SCRIPT_DIR to refer to /etc/xen/scripts
In reply to Olaf HeringReplace all hardcoded paths to use XEN_SCRIPT_DIR to expand the actual location.
Update .gitignore.
---
.gitignore | 3 +++
docs/configure.ac | 3 +++
...k-configuration.5.pod => xl-disk-configuration.5.pod.in} | 2 +-
...onfiguration.5.pod => xl-network-configuration.5.pod.in} | 4 ++--
docs/man/xl.1.pod.in | 2 +-
docs/man/{xl.conf.5.pod => xl.conf.5.pod.in} | 6 +++---
docs/misc/block-scripts.txt | 2 +-
tools/xl/xl_cmdtable.c | 2 +-
8 files changed, 15 insertions(+), 9 deletions(-)
rename docs/man/{xl-disk-configuration.5.pod => xl-disk-configuration.5.pod.in} (99%)
rename docs/man/{xl-network-configuration.5.pod => xl-network-configuration.5.pod.in} (98%)
rename docs/man/{xl.conf.5.pod => xl.conf.5.pod.in} (97%).gitignore +3 -0
@@ -48,7 +48,10 @@ dist/* docs/tmp.* docs/html/ docs/man/xl.cfg.5.pod +docs/man/xl-disk-configuration.5.pod +docs/man/xl-network-configuration.5.pod docs/man/xl.1.pod +docs/man/xl.conf.5.pod docs/man1/ docs/man5/ docs/man7/
docs/configure.ac +3 -0
@@ -9,6 +9,9 @@ AC_CONFIG_FILES([ ../config/Docs.mk man/xl.cfg.5.pod man/xl.1.pod +man/xl-disk-configuration.5.pod +man/xl-network-configuration.5.pod +man/xl.conf.5.pod ]) AC_CONFIG_AUX_DIR([../])
docs/man/xl-disk-configuration.5.pod.in renamed from docs/man/xl-disk-configuration.5.pod +1 -1
@@ -257,7 +257,7 @@ automatically determine the most suitable backend. Specifies that B<target> is not a normal host path, but rather information to be interpreted by the executable program I<SCRIPT>, -(looked for in F</etc/xen/scripts>, if it doesn't contain a slash). +(looked for in F<@XEN_SCRIPT_DIR@>, if it doesn't contain a slash). These scripts are normally called "block-I<SCRIPT>".
docs/man/xl-network-configuration.5.pod.in renamed from docs/man/xl-network-configuration.5.pod +2 -2
@@ -169,8 +169,8 @@ number. Likewise the default tap name is C<vifDOMID.DEVID-emu>. Specifies the hotplug script to run to configure this device (e.g. to add it to the relevant bridge). Defaults to -C<XEN_SCRIPT_DIR/vif-bridge> but can be set to any script. Some example -scripts are installed in C<XEN_SCRIPT_DIR>. +C<@XEN_SCRIPT_DIR@/vif-bridge> but can be set to any script. Some example +scripts are installed in C<@XEN_SCRIPT_DIR@>. =head2 ip
docs/man/xl.1.pod.in +1 -1
@@ -571,7 +571,7 @@ See the corresponding option of the I<create> subcommand. =item B<-N> I<netbufscript> Use <netbufscript> to setup network buffering instead of the -default script (/etc/xen/scripts/remus-netbuf-setup). +default script (@XEN_SCRIPT_DIR@/remus-netbuf-setup). =item B<-F>
docs/man/xl.conf.5.pod.in renamed from docs/man/xl.conf.5.pod +3 -3
@@ -107,7 +107,7 @@ Configures the default hotplug script used by virtual network devices. The old B<vifscript> option is deprecated and should not be used. -Default: C</etc/xen/scripts/vif-bridge> +Default: C<@XEN_SCRIPT_DIR@/vif-bridge> =item B<vif.default.bridge="NAME">
@@ -133,13 +133,13 @@ Default: C<None> Configures the default script used by Remus to setup network buffering. -Default: C</etc/xen/scripts/remus-netbuf-setup> +Default: C<@XEN_SCRIPT_DIR@/remus-netbuf-setup> =item B<colo.default.proxyscript="PATH"> Configures the default script used by COLO to setup colo-proxy. -Default: C</etc/xen/scripts/colo-proxy-setup> +Default: C<@XEN_SCRIPT_DIR@/colo-proxy-setup> =item B<output_format="json|sxp">
docs/misc/block-scripts.txt +1 -1
@@ -18,7 +18,7 @@ Setup It is highly recommended that custom hotplug scripts as much as possible include and use the common Xen functionality. If the script -is run from the normal block script location (/etc/xen/scripts by +is run from the normal block script location (/usr/lib/xen/scripts by default), then this can be done by adding the following to the top of the script:
tools/xl/xl_cmdtable.c +1 -1
@@ -521,7 +521,7 @@ struct cmd_spec cmd_table[] = {"-e Do not wait in the background (on <host>) for the death\n" " of the domain.\n" "-N <netbufscript> Use netbufscript to setup network buffering instead of the\n" -" default script (/etc/xen/scripts/remus-netbuf-setup).\n" +" default script (" XEN_SCRIPT_DIR "/remus-netbuf-setup).\n" "-F Enable unsafe configurations [-b|-n|-d flags]. Use this option\n" " with caution as failover may not work as intended.\n" "-b Replicate memory checkpoints to /dev/null (blackhole).\n"
Re: [PATCH v20210111 06/39] Use XEN_SCRIPT_DIR to refer to /etc/xen/scripts
Olaf Hering writes ("[PATCH v20210111 06/39] Use XEN_SCRIPT_DIR to refer to /etc/xen/scripts"):
Quoted 3 lines
Replace all hardcoded paths to use
XEN_SCRIPT_DIRto expand the actual location.Update .gitignore.
Although I would have preferred these to be separate patches.
I see no reason not to commit this now so I will queue it.
Ian.
Re: [PATCH v20210111 06/39] Use XEN_SCRIPT_DIR to refer to /etc/xen/scripts
Ian Jackson writes ("Re: [PATCH v20210111 06/39] Use XEN_SCRIPT_DIR to refer to /etc/xen/scripts"):
Quoted 11 lines
Olaf Hering writes ("[PATCH v20210111 06/39] Use
XEN_SCRIPT_DIRto refer to/etc/xen/scripts"):Quoted 3 lines
Replace all hardcoded paths to use
XEN_SCRIPT_DIRto expand the actual location.Update .gitignore.
Reviewed-by Ian Jackson <[email protected]>Release-Acked-by Ian Jackson <[email protected]>Although I would have preferred these to be separate patches.
I see no reason not to commit this now so I will queue it.
I split the patch up.
I also dropped the hunk for docs/misc/block-scripts.txt which seems to have been left over from v1. Thanks to Andy for spotting that.
Ian.
PATCH v20210111 7/39 xl: optionally print timestamps during xl migrate
In reply to Olaf HeringDuring 'xl -v.. migrate domU host' a large amount of debug is generated. It is difficult to map each line to the sending and receiving side. Also the time spent for migration is not reported.
With 'xl migrate -T domU host' both sides will print timestamps and also the pid of the invoked xl process to make it more obvious which side produced a given log line.
--- docs/man/xl.1.pod.in | 4 ++++ tools/xl/xl_cmdtable.c | 1 + tools/xl/xl_migrate.c | 25 +++++++++++++++++++++---- 3 files changed, 26 insertions(+), 4 deletions(-)
docs/man/xl.1.pod.in +4 -0
@@ -475,6 +475,10 @@ domain. See the corresponding option of the I<create> subcommand. Send the specified <config> file instead of the file used on creation of the domain. +=item B<-T> + +Include timestamps in output. + =item B<--debug> Display huge (!) amount of debug information during the migration process.
tools/xl/xl_cmdtable.c +1 -0
@@ -167,6 +167,7 @@ struct cmd_spec cmd_table[] = {" migrate-receive [-d -e]\n" "-e Do not wait in the background (on <host>) for the death\n" " of the domain.\n" +"-T Show timestamps during the migration process.\n" "--debug Print huge (!) amount of debug during the migration process.\n" "-p Do not unpause domain after migrating it.\n" "-D Preserve the domain id"
tools/xl/xl_migrate.c +21 -4
@@ -32,6 +32,8 @@#ifndef LIBXL_HAVE_NO_SUSPEND_RESUME +static bool timestamps; +static pid_t create_migration_child (const char *rune,int *send_fd,int *recv_fd) {
@@ -187,6 +189,7 @@ static void migrate_domain(uint32_t domid, int preserve_domid,char rc_buf;uint8_t *config_data;int config_len, flags =LIBXL_SUSPEND_LIVE ; +unsigned xtl_flags =XTL_STDIOSTREAM_HIDE_PROGRESS ;save_domain_core_begin (domid, preserve_domid, override_config_file, &config_data, &config_len);
@@ -202,7 +205,9 @@ static void migrate_domain(uint32_t domid, int preserve_domid,migrate_do_preamble (send_fd, recv_fd, child, config_data, config_len, rune); -xtl_stdiostream_adjust_flags (logger,XTL_STDIOSTREAM_HIDE_PROGRESS ,0 ); +if (timestamps) + xtl_flags |=XTL_STDIOSTREAM_SHOW_DATE |XTL_STDIOSTREAM_SHOW_PID ; +xtl_stdiostream_adjust_flags (logger, xtl_flags,0 );if (debug) flags |=LIBXL_SUSPEND_DEBUG ;
@@ -328,6 +333,11 @@ static void migrate_receive(int debug, int daemonize, int monitor,char rc_buf;char *migration_domname;struct domain_create dom_info; +unsigned xtl_flags =0 ; + +if (timestamps) + xtl_flags |=XTL_STDIOSTREAM_SHOW_DATE |XTL_STDIOSTREAM_SHOW_PID ; +xtl_stdiostream_adjust_flags (logger, xtl_flags,0 );signal (SIGPIPE ,SIG_IGN );/* if we get SIGPIPE we'd rather just have it as an error */
@@ -491,7 +501,7 @@ int main_migrate_receive(int argc, char **argv)COMMON_LONG_OPTS }; - SWITCH_FOREACH_OPT(opt, "Fedrp", opts, "migrate-receive", 0) { + SWITCH_FOREACH_OPT(opt, "FedrpT", opts, "migrate-receive", 0) {case 'F' : daemonize =0 ;break ;
@@ -517,6 +527,9 @@ int main_migrate_receive(int argc, char **argv)case 'p' : pause_after_migration =1 ;break ; +case 'T' : + timestamps =1 ; +break ; }if (argc-optind !=0 ) {
@@ -545,7 +558,7 @@ int main_migrate(int argc, char **argv)COMMON_LONG_OPTS }; - SWITCH_FOREACH_OPT(opt, "FC:s:epD", opts, "migrate", 2) { + SWITCH_FOREACH_OPT(opt, "FC:s:eTpD", opts, "migrate", 2) {case 'C' : config_filename = optarg;break ;
@@ -559,6 +572,9 @@ int main_migrate(int argc, char **argv) daemonize =0 ; monitor =0 ;break ; +case 'T' : + timestamps =1 ; +break ;case 'p' : pause_after_migration =1 ;break ;
@@ -592,11 +608,12 @@ int main_migrate(int argc, char **argv) }else { verbose_len = (minmsglevel_default - minmsglevel) +2 ; } -xasprintf (&rune,"exec %s %s xl%s%.*s migrate-receive%s%s%s" , +xasprintf (&rune,"exec %s %s xl%s%.*s migrate-receive%s%s%s%s" , ssh_command, host, pass_tty_arg ?" -t" :"" , verbose_len, verbose_buf, daemonize ?"" :" -e" , + timestamps ?" -T" :"" , debug ?" -d" :"" , pause_after_migration ?" -p" :"" ); }
Re: [PATCH v20210111 07/39] xl: optionally print timestamps during xl migrate
Olaf Hering writes ("[PATCH v20210111 07/39] xl: optionally print timestamps during xl migrate"):
Quoted 9 lines
During 'xl -v.. migrate domU host' a large amount of debug is generated. It is difficult to map each line to the sending and receiving side. Also the time spent for migration is not reported.
With 'xl migrate -T domU host' both sides will print timestamps and also the pid of the invoked xl process to make it more obvious which side produced a given log line.
Signed-off-by Olaf Hering <[email protected]>
This is from October and ought to have been reviewed much sooner. Sorry.
With my maintainer hat on: this is a useful development but I am not sure about the choice of -T, and the choice to make this a migrate-specific option. Most unix things that have an option to print timestamps use -t. But we have already stolen -t as a global option for "prinht CR as well as LF". Hrmf.
Under the circumstances -T for migrate seems a plausible and conservative choice. I think maybe we should maybe add a global -T later.
Now I have to decide what to do about it for 4.15.
The downside risks I see from reading the patch are:
* The CLI API choice is being made in a hurry.
* We might break something. This actually seems quite unlikely since I have reviewed the code changes in detail.
I wonder if I can get a quick second option from someone on the API question. Using up a single letter option is something I don't want to just rush into.
Ian.
Re: [PATCH v20210111 07/39] xl: optionally print timestamps during xl migrate
Am Mon, 8 Feb 2021 16:22:54 +0000
schrieb Ian Jackson <[email protected]>:
Quoted 3 lines
I wonder if I can get a quick second option from someone on the API question. Using up a single letter option is something I don't want to just rush into.
Maybe put "-T" or "-t" into xl.c:main() instead, so that every command prints timestamp+pid?
For everything beside "migrate" it will only be useful along with a couple of "-v" to enable global debug output.
Olaf
Re: [PATCH v20210111 07/39] xl: optionally print timestamps during xl migrate
Olaf Hering writes ("Re: [PATCH v20210111 07/39] xl: optionally print timestamps during xl migrate"):
Quoted 8 lines
Am Mon, 8 Feb 2021 16:22:54 +0000
schrieb Ian Jackson <[email protected]>:Quoted 3 lines
I wonder if I can get a quick second option from someone on the API question. Using up a single letter option is something I don't want to just rush into.
Maybe put "-T" or "-t" into xl.c:main() instead, so that every command prints timestamp+pid?
Yes, I think following some informal irc discussions that that would be best.
Quoted 1 line
For everything beside "migrate" it will only be useful along with a couple of "-v" to enable global debug output.
Right.
Ian.
Re: [PATCH v20210111 07/39] xl: optionally print timestamps during xl migrate
In reply to Ian JacksonAm Mon, 8 Feb 2021 16:22:54 +0000
schrieb Ian Jackson <[email protected]>:
Quoted 5 lines
With my maintainer hat on: this is a useful development but I am not sure about the choice of -T, and the choice to make this a migrate-specific option. Most unix things that have an option to print timestamps use
-t. But we have already stolen-tas a global option for "prinht CR as well as LF". Hrmf.
Was 'xl -t' intentionally left out of xl.c:help()? It is only mentioned in the xl man page.
If yes, my change will also omit it.
If no, I will add it along with the global -T option.
Olaf
PATCH v20210111 8/39 xl: fix description of migrate --debug
In reply to Olaf Heringxl migrate --debug used to track every pfn in every batch of pages. But these times are gone. Adjust the help text to tell what --debug is supposed to do today.
--- docs/man/xl.1.pod.in | 4 +++- tools/xl/xl_cmdtable.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-)
docs/man/xl.1.pod.in +3 -1
@@ -481,7 +481,9 @@ Include timestamps in output. =item B<--debug> -Display huge (!) amount of debug information during the migration process. +Verify transferred domU page data. All memory will be transferred one more +time to the destination host while the domU is paused, and compared with +the result of the inital transfer while the domU was still running. =item B<-p>
tools/xl/xl_cmdtable.c +1 -1
@@ -168,7 +168,7 @@ struct cmd_spec cmd_table[] = {"-e Do not wait in the background (on <host>) for the death\n" " of the domain.\n" "-T Show timestamps during the migration process.\n" -"--debug Print huge (!) amount of debug during the migration process.\n" +"--debug Verify transferred domU page data.\n" "-p Do not unpause domain after migrating it.\n" "-D Preserve the domain id" },
Re: [PATCH v20210111 08/39] xl: fix description of migrate --debug
Olaf Hering writes ("[PATCH v20210111 08/39] xl: fix description of migrate --debug"):
Quoted 3 lines
xl migrate --debug used to track every pfn in every batch of pages. But these times are gone. Adjust the help text to tell what --debug is supposed to do today.
...
docs/man/xl.1.pod.in Include timestamps in output. · line 486
Quoted 4 lines
docs/man/xl.1.pod.in +3
-1-Display huge (!) amount of debug information during the migration process. +Verify transferred domU page data. All memory will be transferred one more +time to the destination host while the domU is paused, and compared with +the result of the inital transfer while the domU was still running.
Andy, as our expert on migration, can you confirm whether this is accurate ?
Docs (including usage message) so
Thanks,
Ian.
Re: [PATCH v20210111 08/39] xl: fix description of migrate --debug
Quoted 12 lines · Ian Jackson
Olaf Hering writes ("[PATCH v20210111 08/39] xl: fix description of migrate --debug"):
Quoted 3 lines
xl migrate --debug used to track every pfn in every batch of pages. But these times are gone. Adjust the help text to tell what --debug is supposed to do today.
...
docs/man/xl.1.pod.in Include timestamps in output. · line 486
Quoted 4 lines
docs/man/xl.1.pod.in +3
-1-Display huge (!) amount of debug information during the migration process. +Verify transferred domU page data. All memory will be transferred one more +time to the destination host while the domU is paused, and compared with +the result of the inital transfer while the domU was still running.Andy, as our expert on migration, can you confirm whether this is accurate ?
Quoted 3 lines
Docs (including usage message) so
Release-Acked-by Ian Jackson <[email protected]>
Yeah - I totally changed what --debug did in migration v2, and this is stale. The legacy behaviour was unhelpful on any non-trivial guest.
It is possibly worth noting that you typically do see changed data when using --debug, because of how the front/back pairs work. This was a bit of a curveball during development.
~Andrew
Re: [PATCH v20210111 08/39] xl: fix description of migrate --debug
Am Mon, 8 Feb 2021 16:39:01 +0000
schrieb Andrew Cooper <[email protected]>:
Quoted 3 lines
It is possibly worth noting that you typically do see changed data when using --debug, because of how the front/back pairs work. This was a bit of a curveball during development.
I just noticed "migrate --debug" is a noop, "verify" works just for remus or colo, per send_domain_memory_live():
- libxl_domain_remus_start sets checkpointed_stream to COLO/REMUS
- libxl_domain_suspend sets checkpointed_stream to NONE.
- external callers can not influence this internal state.
- main_migrate_receive sets it based on the command line option.
In case we want a "verify" functionality also for migration, the "stream_type" check could be removed to make it work everywhere. The domU is suspended, it should not make much difference how often its memory is passed around in this suspended state. But this would be a separate thing to explore.
Having a "LIBXL_SUSPEND_DEBUG/XCFLAGS_DEBUG" might be useful, but in its current state the flags should have "STREAM_VERIFY" in their name.
So instead of changing the help string I suggest to remove "--debug" altogether from the xl UI.
Olaf
PATCH v20210111 9/39 tools: add readv_exact to libxenctrl
In reply to Olaf HeringRead a batch of iovec's.
In the common case of short reads, finish individual iov's with read_exact.
--- tools/libs/ctrl/xc_private.c | 55 +++++++++++++++++++++++++++++++++++- tools/libs/ctrl/xc_private.h | 1 + 2 files changed, 55 insertions(+), 1 deletion(-)
tools/libs/ctrl/xc_private.c +54 -1
@@ -659,8 +659,23 @@ int write_exact(int fd, const void *data, size_t size)#if defined(__MINIOS__)/* -* MiniOS's libc doesn't know about writev(). Implement it as multiple write()s. +* MiniOS's libc doesn't know about readv/writev(). +* Implement it as multiple read/write()s. */ +int readv_exact (int fd,const struct iovec *iov,int iovcnt) +{ +int rc, i; + +for ( i =0 ; i < iovcnt; ++i ) + { + rc =read_exact (fd, iov[i].iov_base, iov[i].iov_len); +if ( rc ) +return rc; + } + +return 0 ; +} +int writev_exact (int fd,const struct iovec *iov,int iovcnt) {int rc, i;
@@ -675,6 +690,44 @@ int writev_exact(int fd, const struct iovec *iov, int iovcnt)return 0 ; }#else +int readv_exact (int fd,const struct iovec *iov,int iovcnt) +{ +int rc =0 , idx =0 ; +ssize_t len; + +while ( idx < iovcnt ) + { + len =readv (fd, &iov[idx],min (iovcnt - idx,IOV_MAX )); +if ( len ==-1 && errno ==EINTR ) +continue ; +if ( len <=0 ) + { + rc =-1 ; + goto out; + } +while ( len >0 && idx < iovcnt ) + { +if ( len >= iov[idx].iov_len ) + { + len -= iov[idx].iov_len; + } +else + { +void *p = iov[idx].iov_base + len; +size_t l = iov[idx].iov_len - len; + + rc =read_exact (fd, p, l); +if ( rc ) + goto out; + len =0 ; + } + idx++; + } + } +out: +return rc; +} +int writev_exact (int fd,const struct iovec *iov,int iovcnt) {struct iovec *local_iov =NULL ;
tools/libs/ctrl/xc_private.h +1 -0
@@ -441,6 +441,7 @@ int xc_flush_mmu_updates(xc_interface *xch, struct xc_mmu *mmu);/* Return 0 on success; -1 on error setting errno. */ int read_exact (int fd,void *data,size_t size);/* EOF => -1, errno=0 */ +int readv_exact (int fd,const struct iovec *iov,int iovcnt);int write_exact (int fd,const void *data,size_t size);int writev_exact (int fd,const struct iovec *iov,int iovcnt);
PATCH v20210111 10/39 tools: add xc_is_known_page_type to libxenctrl
In reply to Olaf HeringUsers of xc_get_pfn_type_batch may want to sanity check the data returned by Xen. Add a simple helper for this purpose.
--- tools/libs/ctrl/xc_private.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)
tools/libs/ctrl/xc_private.h +33 -0
@@ -421,6 +421,39 @@ void *xc_map_foreign_ranges(xc_interface *xch, uint32_t dom,int xc_get_pfn_type_batch (xc_interface *xch,uint32_t dom,unsigned int num,xen_pfn_t *); +/* Sanitiy check for types returned by Xen */ +static inline bool xc_is_known_page_type (xen_pfn_t type) +{ +bool ret; + +switch (type) + { +case XEN_DOMCTL_PFINFO_NOTAB : + +case XEN_DOMCTL_PFINFO_L1TAB : +case XEN_DOMCTL_PFINFO_L1TAB |XEN_DOMCTL_PFINFO_LPINTAB : + +case XEN_DOMCTL_PFINFO_L2TAB : +case XEN_DOMCTL_PFINFO_L2TAB |XEN_DOMCTL_PFINFO_LPINTAB : + +case XEN_DOMCTL_PFINFO_L3TAB : +case XEN_DOMCTL_PFINFO_L3TAB |XEN_DOMCTL_PFINFO_LPINTAB : + +case XEN_DOMCTL_PFINFO_L4TAB : +case XEN_DOMCTL_PFINFO_L4TAB |XEN_DOMCTL_PFINFO_LPINTAB : + +case XEN_DOMCTL_PFINFO_XTAB : +case XEN_DOMCTL_PFINFO_XALLOC : +case XEN_DOMCTL_PFINFO_BROKEN : + ret = true; +break ; +default : + ret = false; +break ; + } +return ret; +} +void bitmap_64_to_byte (uint8_t *bp,const uint64_t *lp,int nbits);void bitmap_byte_to_64 (uint64_t *lp,const uint8_t *bp,int nbits);
PATCH v20210111 11/39 tools: use xc_is_known_page_type
In reply to Olaf HeringVerify pfn type on sending side, also verify incoming batch of pfns.
--- tools/libs/guest/xg_sr_restore.c | 3 +-- tools/libs/guest/xg_sr_save.c | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-)
tools/libs/guest/xg_sr_restore.c +1 -2
@@ -406,8 +406,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec) } type = (pages->pfn[i] &PAGE_DATA_TYPE_MASK ) >>32 ; -if ( ((type >>XEN_DOMCTL_PFINFO_LTAB_SHIFT ) >=5 ) && - ((type >>XEN_DOMCTL_PFINFO_LTAB_SHIFT ) <=8 ) ) +if (xc_is_known_page_type (type) == false ) {ERROR ("Invalid type %#" PRIx32" for pfn %#" PRIpfn" (index %u)" , type, pfn, i);
tools/libs/guest/xg_sr_save.c +6 -0
@@ -147,6 +147,12 @@ static int write_batch(struct xc_sr_context *ctx)for ( i =0 ; i < nr_pfns; ++i ) { +if (xc_is_known_page_type (types[i]) == false ) + { +ERROR ("Wrong type %#" PRIpfn" for pfn %#" PRIpfn, types[i], mfns[i]); + goto err; + } +switch ( types[i] ) {case XEN_DOMCTL_PFINFO_BROKEN :
PATCH v20210111 12/39 tools: unify type checking for data pfns in migration stream
In reply to Olaf HeringIntroduce a helper which decides if a given pfn type has data for the migration stream.
No change in behavior intended.
--- tools/libs/guest/xg_sr_common.h | 17 ++++++++++++++++ tools/libs/guest/xg_sr_restore.c | 34 +++++--------------------------- tools/libs/guest/xg_sr_save.c | 14 ++----------- 3 files changed, 24 insertions(+), 41 deletions(-)
tools/libs/guest/xg_sr_common.h +17 -0
@@ -455,6 +455,23 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,/* Handle a STATIC_DATA_END record. */ int handle_static_data_end (struct xc_sr_context *ctx); +static inline bool page_type_has_stream_data (uint32_t type) +{ +bool ret; + +switch (type) + { +case XEN_DOMCTL_PFINFO_XTAB : +case XEN_DOMCTL_PFINFO_XALLOC : +case XEN_DOMCTL_PFINFO_BROKEN : + ret = false; +break ; +default : + ret = true; +break ; + } +return ret; +}#endif /* * Local variables:
tools/libs/guest/xg_sr_restore.c +5 -29
@@ -152,9 +152,8 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,for ( i =0 ; i < count; ++i ) { -if ( (!types || (types && - (types[i] !=XEN_DOMCTL_PFINFO_XTAB && - types[i] !=XEN_DOMCTL_PFINFO_BROKEN ))) && +if ( (!types || + (types &&page_type_has_stream_data (types[i]) == true)) && !pfn_is_populated (ctx, original_pfns[i]) ) { rc =pfn_set_populated (ctx, original_pfns[i]);
@@ -233,25 +232,8 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count, { ctx->restore.ops.set_page_type (ctx, pfns[i], types[i]); -switch ( types[i] ) - { -case XEN_DOMCTL_PFINFO_NOTAB : - -case XEN_DOMCTL_PFINFO_L1TAB : -case XEN_DOMCTL_PFINFO_L1TAB |XEN_DOMCTL_PFINFO_LPINTAB : - -case XEN_DOMCTL_PFINFO_L2TAB : -case XEN_DOMCTL_PFINFO_L2TAB |XEN_DOMCTL_PFINFO_LPINTAB : - -case XEN_DOMCTL_PFINFO_L3TAB : -case XEN_DOMCTL_PFINFO_L3TAB |XEN_DOMCTL_PFINFO_LPINTAB : - -case XEN_DOMCTL_PFINFO_L4TAB : -case XEN_DOMCTL_PFINFO_L4TAB |XEN_DOMCTL_PFINFO_LPINTAB : - +if (page_type_has_stream_data (types[i]) == true ) mfns[nr_pages++] = ctx->restore.ops.pfn_to_gfn (ctx, pfns[i]); -break ; - } }/* Nothing to do? */
@@ -271,14 +253,8 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,for ( i =0 , j =0 ; i < count; ++i ) { -switch ( types[i] ) - { -case XEN_DOMCTL_PFINFO_XTAB : -case XEN_DOMCTL_PFINFO_BROKEN : -case XEN_DOMCTL_PFINFO_XALLOC : -/* No page data to deal with. */ +if (page_type_has_stream_data (types[i]) == false )continue ; - }if ( map_errs[j] ) {
@@ -413,7 +389,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec) goto err; } -if ( type <XEN_DOMCTL_PFINFO_BROKEN ) +if (page_type_has_stream_data (type) == true )/* NOTAB and all L1 through L4 tables (including pinned) should * have a page worth of data in the record. */ pages_of_data++;
tools/libs/guest/xg_sr_save.c +2 -12
@@ -153,13 +153,8 @@ static int write_batch(struct xc_sr_context *ctx) goto err; } -switch ( types[i] ) - { -case XEN_DOMCTL_PFINFO_BROKEN : -case XEN_DOMCTL_PFINFO_XALLOC : -case XEN_DOMCTL_PFINFO_XTAB : +if (page_type_has_stream_data (types[i]) == false )continue ; - } mfns[nr_pages++] = mfns[i]; }
@@ -177,13 +172,8 @@ static int write_batch(struct xc_sr_context *ctx)for ( i =0 , p =0 ; i < nr_pfns; ++i ) { -switch ( types[i] ) - { -case XEN_DOMCTL_PFINFO_BROKEN : -case XEN_DOMCTL_PFINFO_XALLOC : -case XEN_DOMCTL_PFINFO_XTAB : +if (page_type_has_stream_data (types[i]) == false )continue ; - }if ( errors[p] ) {
PATCH v20210111 13/39 tools: show migration transfer rate in send_dirty_pages
In reply to Olaf HeringShow how fast domU pages are transferred in each iteration.
The relevant data is how fast the pfns travel, not so much how much protocol overhead exists. So the reported MiB/sec is just for pfns.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -238,6 +238,8 @@ struct xc_sr_contextbool debug;unsigned long p2m_size; +size_t pages_sent; +size_t overhead_sent;struct precopy_stats stats;
tools/libs/guest/xg_sr_save.c +47 -0
79 lines not on this page · raw message
PATCH v20210111 14/39 tools/guest: prepare to allocate arrays once
In reply to Olaf HeringThe hotpath 'send_dirty_pages' is supposed to do just one thing: sending. The other end 'handle_page_data' is supposed to do just receiving.
But instead both do other costly work like memory allocations and data moving. Do the allocations once, the array sizes are a compiletime constant. Avoid unneeded copying of data by receiving data directly into mapped guest memory.
This patch is just prepartion, subsequent changes will populate the arrays.
Once all changes are applied, migration of a busy HVM domU changes like that:
Without this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_testing):
2020-10-29 10:23:10.711+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 55.324905335 sec, 203 MiB/sec: Internal error 2020-10-29 10:23:35.115+0000: xc: show_transfer_rate: 16829632 bytes + 2097552 pages in 24.401179720 sec, 335 MiB/sec: Internal error 2020-10-29 10:23:59.436+0000: xc: show_transfer_rate: 16829032 bytes + 2097478 pages in 24.319025928 sec, 336 MiB/sec: Internal error 2020-10-29 10:24:23.844+0000: xc: show_transfer_rate: 16829024 bytes + 2097477 pages in 24.406992500 sec, 335 MiB/sec: Internal error 2020-10-29 10:24:48.292+0000: xc: show_transfer_rate: 16828912 bytes + 2097463 pages in 24.446489027 sec, 335 MiB/sec: Internal error 2020-10-29 10:25:01.816+0000: xc: show_transfer_rate: 16836080 bytes + 2098356 pages in 13.447091818 sec, 609 MiB/sec: Internal error
With this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_unstable):
2020-10-28 21:26:05.074+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 52.564054368 sec, 213 MiB/sec: Internal error 2020-10-28 21:26:23.527+0000: xc: show_transfer_rate: 16830040 bytes + 2097603 pages in 18.450592015 sec, 444 MiB/sec: Internal error 2020-10-28 21:26:41.926+0000: xc: show_transfer_rate: 16830944 bytes + 2097717 pages in 18.397862306 sec, 445 MiB/sec: Internal error 2020-10-28 21:27:00.339+0000: xc: show_transfer_rate: 16829176 bytes + 2097498 pages in 18.411973339 sec, 445 MiB/sec: Internal error 2020-10-28 21:27:18.643+0000: xc: show_transfer_rate: 16828592 bytes + 2097425 pages in 18.303326695 sec, 447 MiB/sec: Internal error 2020-10-28 21:27:26.289+0000: xc: show_transfer_rate: 16835952 bytes + 2098342 pages in 7.579846749 sec, 1081 MiB/sec: Internal error
--- tools/libs/guest/xg_sr_common.h | 8 ++++++++ tools/libs/guest/xg_sr_restore.c | 8 ++++++++ tools/libs/guest/xg_sr_save.c | 4 +++- 3 files changed, 19 insertions(+), 1 deletion(-)
tools/libs/guest/xg_sr_common.h +8 -0
@@ -211,6 +211,12 @@ static inline int update_blob(struct xc_sr_blob *blob,return 0 ; } +struct xc_sr_save_arrays { +}; + +struct xc_sr_restore_arrays { +}; +struct xc_sr_context {xc_interface *xch;
@@ -248,6 +254,7 @@ struct xc_sr_contextunsigned long *deferred_pages;unsigned long nr_deferred_pages;xc_hypercall_buffer_t dirty_bitmap_hbuf; +struct xc_sr_save_arrays *m; } save; struct /* Restore data. */
@@ -299,6 +306,7 @@ struct xc_sr_context/* Sender has invoked verify mode on the stream. */ bool verify; +struct xc_sr_restore_arrays *m; } restore; };
tools/libs/guest/xg_sr_restore.c +8 -0
@@ -739,6 +739,13 @@ static int setup(struct xc_sr_context *ctx) } ctx->restore.allocated_rec_num =DEFAULT_BUF_RECORDS ; + ctx->restore.m =malloc (sizeof (*ctx->restore.m)); +if ( !ctx->restore.m ) { +ERROR ("Unable to allocate memory for arrays" ); + rc =-1 ; + goto err; + } + err:return rc; }
@@ -757,6 +764,7 @@ static void cleanup(struct xc_sr_context *ctx)xc_hypercall_buffer_free_pages ( xch, dirty_bitmap,NRPAGES (bitmap_size (ctx->restore.p2m_size))); +free (ctx->restore.m);free (ctx->restore.buffered_records);free (ctx->restore.populated_pfns);
tools/libs/guest/xg_sr_save.c +3 -1
@@ -853,8 +853,9 @@ static int setup(struct xc_sr_context *ctx) ctx->save.batch_pfns =malloc (MAX_BATCH_SIZE *sizeof (*ctx->save.batch_pfns)); ctx->save.deferred_pages =bitmap_alloc (ctx->save.p2m_size); + ctx->save.m =malloc (sizeof (*ctx->save.m)); -if ( !ctx->save.batch_pfns || !dirty_bitmap || !ctx->save.deferred_pages ) +if ( !ctx->save.m || !ctx->save.batch_pfns || !dirty_bitmap || !ctx->save.deferred_pages ) {ERROR ("Unable to allocate memory for dirty bitmaps, batch pfns and" " deferred pages" );
@@ -886,6 +887,7 @@ static void cleanup(struct xc_sr_context *ctx) NRPAGES(bitmap_size(ctx->save.p2m_size)));free (ctx->save.deferred_pages);free (ctx->save.batch_pfns); +free (ctx->save.m); }/*
PATCH v20210111 15/39 tools/guest: save: move batch_pfns
In reply to Olaf HeringThe batch_pfns array is already allocated in advance. Move it into the preallocated area.
--- tools/libs/guest/xg_sr_common.h | 2 +- tools/libs/guest/xg_sr_save.c | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-)
tools/libs/guest/xg_sr_common.h +1 -1
@@ -212,6 +212,7 @@ static inline int update_blob(struct xc_sr_blob *blob, }struct xc_sr_save_arrays { +xen_pfn_t batch_pfns[MAX_BATCH_SIZE ]; };struct xc_sr_restore_arrays {
@@ -249,7 +250,6 @@ struct xc_sr_contextstruct precopy_stats stats; -xen_pfn_t *batch_pfns;unsigned int nr_batch_pfns;unsigned long *deferred_pages;unsigned long nr_deferred_pages;
tools/libs/guest/xg_sr_save.c +11 -14
@@ -77,7 +77,7 @@ static int write_checkpoint_record(struct xc_sr_context *ctx)/* * Writes a batch of memory as a PAGE_DATA record into the stream. The batch -* is constructed in ctx->save.batch_pfns. +* is constructed in ctx->save.m->batch_pfns. * * This function: * - gets the types for each pfn in the batch.
@@ -128,12 +128,12 @@ static int write_batch(struct xc_sr_context *ctx)for ( i =0 ; i < nr_pfns; ++i ) { types[i] = mfns[i] = ctx->save.ops.pfn_to_gfn (ctx, - ctx->save.batch_pfns[i]); + ctx->save.m->batch_pfns[i]);/* Likely a ballooned page. */ if ( mfns[i] ==INVALID_MFN ) { -set_bit (ctx->save.batch_pfns[i], ctx->save.deferred_pages); +set_bit (ctx->save.m->batch_pfns[i], ctx->save.deferred_pages); ++ctx->save.nr_deferred_pages; } }
@@ -179,7 +179,7 @@ static int write_batch(struct xc_sr_context *ctx)if ( errors[p] ) {ERROR ("Mapping of pfn %#" PRIpfn" (mfn %#" PRIpfn") failed %d" , - ctx->save.batch_pfns[i], mfns[p], errors[p]); + ctx->save.m->batch_pfns[i], mfns[p], errors[p]); goto err; }
@@ -193,7 +193,7 @@ static int write_batch(struct xc_sr_context *ctx) {if ( rc ==-1 && errno ==EAGAIN ) { -set_bit (ctx->save.batch_pfns[i], ctx->save.deferred_pages); +set_bit (ctx->save.m->batch_pfns[i], ctx->save.deferred_pages); ++ctx->save.nr_deferred_pages; types[i] =XEN_DOMCTL_PFINFO_XTAB ; --nr_pages;
@@ -224,7 +224,7 @@ static int write_batch(struct xc_sr_context *ctx) rec.length += nr_pages *PAGE_SIZE ;for ( i =0 ; i < nr_pfns; ++i ) - rec_pfns[i] = ((uint64_t )(types[i]) <<32 ) | ctx->save.batch_pfns[i]; + rec_pfns[i] = ((uint64_t )(types[i]) <<32 ) | ctx->save.m->batch_pfns[i]; iov[0 ].iov_base = &rec.type; iov[0 ].iov_len =sizeof (rec.type);
@@ -296,9 +296,9 @@ static int flush_batch(struct xc_sr_context *ctx)if ( !rc ) { -VALGRIND_MAKE_MEM_UNDEFINED (ctx->save.batch_pfns, +VALGRIND_MAKE_MEM_UNDEFINED (ctx->save.m->batch_pfns,MAX_BATCH_SIZE * -sizeof (*ctx->save.batch_pfns)); +sizeof (*ctx->save.m->batch_pfns)); }return rc;
@@ -315,7 +315,7 @@ static int add_to_batch(struct xc_sr_context *ctx, xen_pfn_t pfn) rc =flush_batch (ctx);if ( rc ==0 ) - ctx->save.batch_pfns[ctx->save.nr_batch_pfns++] = pfn; + ctx->save.m->batch_pfns[ctx->save.nr_batch_pfns++] = pfn;return rc; }
@@ -850,14 +850,12 @@ static int setup(struct xc_sr_context *ctx) dirty_bitmap =xc_hypercall_buffer_alloc_pages ( xch, dirty_bitmap,NRPAGES (bitmap_size (ctx->save.p2m_size))); - ctx->save.batch_pfns =malloc (MAX_BATCH_SIZE * -sizeof (*ctx->save.batch_pfns)); ctx->save.deferred_pages =bitmap_alloc (ctx->save.p2m_size); ctx->save.m =malloc (sizeof (*ctx->save.m)); -if ( !ctx->save.m || !ctx->save.batch_pfns || !dirty_bitmap || !ctx->save.deferred_pages ) +if ( !ctx->save.m || !dirty_bitmap || !ctx->save.deferred_pages ) { -ERROR ("Unable to allocate memory for dirty bitmaps, batch pfns and" +ERROR ("Unable to allocate memory for dirty bitmaps and" " deferred pages" ); rc =-1 ; errno =ENOMEM ;
@@ -886,7 +884,6 @@ static void cleanup(struct xc_sr_context *ctx)xc_hypercall_buffer_free_pages (xch, dirty_bitmap,NRPAGES (bitmap_size (ctx->save.p2m_size)));free (ctx->save.deferred_pages); -free (ctx->save.batch_pfns);free (ctx->save.m); }
Re: [PATCH v20210111 15/39] tools/guest: save: move batch_pfns
Olaf Hering writes ("[PATCH v20210111 15/39] tools/guest: save: move batch_pfns"):
Quoted 2 lines
The
batch_pfnsarray is already allocated in advance. Move it into the preallocated area.
I think these patche really need a review from someone who understands the migration code. Ideally, Andy, who unfortunately has been very busy.
I doubt this is going to make it for 4.15 :-( but maybe Andy or someone else has an opinion.
Thanks,
Ian.
PATCH v20210111 16/39 tools/guest: save: move mfns array
In reply to Olaf HeringRemove allocation from hotpath, move mfns array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -213,6 +213,8 @@ static inline int update_blob(struct xc_sr_blob *blob,struct xc_sr_save_arrays {xen_pfn_t batch_pfns[MAX_BATCH_SIZE ]; +/* write_batch: Mfns of the batch pfns. */ +xen_pfn_t mfns[MAX_BATCH_SIZE ]; };struct xc_sr_restore_arrays {
tools/libs/guest/xg_sr_save.c +2 -5
@@ -88,7 +88,7 @@ static int write_checkpoint_record(struct xc_sr_context *ctx)static int write_batch (struct xc_sr_context *ctx) {xc_interface *xch = ctx->xch; -xen_pfn_t *mfns =NULL , *types =NULL ; +xen_pfn_t *mfns = ctx->save.m->mfns, *types =NULL ;void *guest_mapping =NULL ;void **guest_data =NULL ;void **local_pages =NULL ;
@@ -105,8 +105,6 @@ static int write_batch(struct xc_sr_context *ctx)assert (nr_pfns !=0 ); -/* Mfns of the batch pfns. */ - mfns =malloc (nr_pfns *sizeof (*mfns));/* Types of the batch pfns. */ types =malloc (nr_pfns *sizeof (*types));/* Errors from attempting to map the gfns. */
@@ -118,7 +116,7 @@ static int write_batch(struct xc_sr_context *ctx)/* iovec[] for writev(). */ iov =malloc ((nr_pfns +4 ) *sizeof (*iov)); -if ( !mfns || !types || !errors || !guest_data || !local_pages || !iov ) +if ( !types || !errors || !guest_data || !local_pages || !iov ) {ERROR ("Unable to allocate arrays for a batch of %u pages" , nr_pfns);
@@ -277,7 +275,6 @@ static int write_batch(struct xc_sr_context *ctx)free (guest_data);free (errors);free (types); -free (mfns);return rc; }
PATCH v20210111 17/39 tools/guest: save: move types array
In reply to Olaf HeringRemove allocation from hotpath, move types array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -215,6 +215,8 @@ struct xc_sr_save_arrays {xen_pfn_t batch_pfns[MAX_BATCH_SIZE ];/* write_batch: Mfns of the batch pfns. */ xen_pfn_t mfns[MAX_BATCH_SIZE ]; +/* write_batch: Types of the batch pfns. */ +xen_pfn_t types[MAX_BATCH_SIZE ]; };struct xc_sr_restore_arrays {
tools/libs/guest/xg_sr_save.c +2 -5
@@ -88,7 +88,7 @@ static int write_checkpoint_record(struct xc_sr_context *ctx)static int write_batch (struct xc_sr_context *ctx) {xc_interface *xch = ctx->xch; -xen_pfn_t *mfns = ctx->save.m->mfns, *types =NULL ; +xen_pfn_t *mfns = ctx->save.m->mfns, *types = ctx->save.m->types;void *guest_mapping =NULL ;void **guest_data =NULL ;void **local_pages =NULL ;
@@ -105,8 +105,6 @@ static int write_batch(struct xc_sr_context *ctx)assert (nr_pfns !=0 ); -/* Types of the batch pfns. */ - types =malloc (nr_pfns *sizeof (*types));/* Errors from attempting to map the gfns. */ errors =malloc (nr_pfns *sizeof (*errors));/* Pointers to page data to send. Mapped gfns or local allocations. */
@@ -116,7 +114,7 @@ static int write_batch(struct xc_sr_context *ctx)/* iovec[] for writev(). */ iov =malloc ((nr_pfns +4 ) *sizeof (*iov)); -if ( !types || !errors || !guest_data || !local_pages || !iov ) +if ( !errors || !guest_data || !local_pages || !iov ) {ERROR ("Unable to allocate arrays for a batch of %u pages" , nr_pfns);
@@ -274,7 +272,6 @@ static int write_batch(struct xc_sr_context *ctx)free (local_pages);free (guest_data);free (errors); -free (types);return rc; }
PATCH v20210111 18/39 tools/guest: save: move errors array
In reply to Olaf HeringRemove allocation from hotpath, move errors array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -217,6 +217,8 @@ struct xc_sr_save_arrays {xen_pfn_t mfns[MAX_BATCH_SIZE ];/* write_batch: Types of the batch pfns. */ xen_pfn_t types[MAX_BATCH_SIZE ]; +/* write_batch: Errors from attempting to map the gfns. */ +int errors[MAX_BATCH_SIZE ]; };struct xc_sr_restore_arrays {
tools/libs/guest/xg_sr_save.c +2 -5
@@ -92,7 +92,7 @@ static int write_batch(struct xc_sr_context *ctx)void *guest_mapping =NULL ;void **guest_data =NULL ;void **local_pages =NULL ; -int *errors =NULL , rc =-1 ; +int *errors = ctx->save.m->errors, rc =-1 ;unsigned int i, p, nr_pages =0 , nr_pages_mapped =0 ;unsigned int nr_pfns = ctx->save.nr_batch_pfns;void *page, *orig_page;
@@ -105,8 +105,6 @@ static int write_batch(struct xc_sr_context *ctx)assert (nr_pfns !=0 ); -/* Errors from attempting to map the gfns. */ - errors =malloc (nr_pfns *sizeof (*errors));/* Pointers to page data to send. Mapped gfns or local allocations. */ guest_data =calloc (nr_pfns,sizeof (*guest_data));/* Pointers to locally allocated pages. Need freeing. */
@@ -114,7 +112,7 @@ static int write_batch(struct xc_sr_context *ctx)/* iovec[] for writev(). */ iov =malloc ((nr_pfns +4 ) *sizeof (*iov)); -if ( !errors || !guest_data || !local_pages || !iov ) +if ( !guest_data || !local_pages || !iov ) {ERROR ("Unable to allocate arrays for a batch of %u pages" , nr_pfns);
@@ -271,7 +269,6 @@ static int write_batch(struct xc_sr_context *ctx)free (iov);free (local_pages);free (guest_data); -free (errors);return rc; }
PATCH v20210111 19/39 tools/guest: save: move iov array
In reply to Olaf HeringRemove allocation from hotpath, move iov array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -219,6 +219,8 @@ struct xc_sr_save_arrays {xen_pfn_t types[MAX_BATCH_SIZE ];/* write_batch: Errors from attempting to map the gfns. */ int errors[MAX_BATCH_SIZE ]; +/* write_batch: iovec[] for writev(). */ +struct iovec iov[MAX_BATCH_SIZE +4 ]; };struct xc_sr_restore_arrays {
tools/libs/guest/xg_sr_save.c +2 -5
@@ -97,7 +97,7 @@ static int write_batch(struct xc_sr_context *ctx)unsigned int nr_pfns = ctx->save.nr_batch_pfns;void *page, *orig_page;uint64_t *rec_pfns =NULL ; -struct iovec *iov =NULL ;int iovcnt =0 ; +struct iovec *iov = ctx->save.m->iov;int iovcnt =0 ;struct xc_sr_rec_page_data_header hdr = {0 };struct xc_sr_record rec = { .type =REC_TYPE_PAGE_DATA ,
@@ -109,10 +109,8 @@ static int write_batch(struct xc_sr_context *ctx) guest_data =calloc (nr_pfns,sizeof (*guest_data));/* Pointers to locally allocated pages. Need freeing. */ local_pages =calloc (nr_pfns,sizeof (*local_pages)); -/* iovec[] for writev(). */ - iov =malloc ((nr_pfns +4 ) *sizeof (*iov)); -if ( !guest_data || !local_pages || !iov ) +if ( !guest_data || !local_pages ) {ERROR ("Unable to allocate arrays for a batch of %u pages" , nr_pfns);
@@ -266,7 +264,6 @@ static int write_batch(struct xc_sr_context *ctx)xenforeignmemory_unmap (xch->fmem, guest_mapping, nr_pages_mapped);for ( i =0 ; local_pages && i < nr_pfns; ++i )free (local_pages[i]); -free (iov);free (local_pages);free (guest_data);
PATCH v20210111 20/39 tools/guest: save: move rec_pfns array
In reply to Olaf HeringRemove allocation from hotpath, move rec_pfns array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 11 +---------- 2 files changed, 3 insertions(+), 10 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -221,6 +221,8 @@ struct xc_sr_save_arrays {int errors[MAX_BATCH_SIZE ];/* write_batch: iovec[] for writev(). */ struct iovec iov[MAX_BATCH_SIZE +4 ]; +/* write_batch */ +uint64_t rec_pfns[MAX_BATCH_SIZE ]; };struct xc_sr_restore_arrays {
tools/libs/guest/xg_sr_save.c +1 -10
@@ -96,7 +96,7 @@ static int write_batch(struct xc_sr_context *ctx)unsigned int i, p, nr_pages =0 , nr_pages_mapped =0 ;unsigned int nr_pfns = ctx->save.nr_batch_pfns;void *page, *orig_page; -uint64_t *rec_pfns =NULL ; +uint64_t *rec_pfns = ctx->save.m->rec_pfns;struct iovec *iov = ctx->save.m->iov;int iovcnt =0 ;struct xc_sr_rec_page_data_header hdr = {0 };struct xc_sr_record rec = {
@@ -201,14 +201,6 @@ static int write_batch(struct xc_sr_context *ctx) } } - rec_pfns =malloc (nr_pfns *sizeof (*rec_pfns)); -if ( !rec_pfns ) - { -ERROR ("Unable to allocate %zu bytes of memory for page data pfn list" , - nr_pfns *sizeof (*rec_pfns)); - goto err; - } - hdr.count = nr_pfns; rec.length =sizeof (hdr);
@@ -259,7 +251,6 @@ static int write_batch(struct xc_sr_context *ctx) rc = ctx->save.nr_batch_pfns =0 ; err: -free (rec_pfns);if ( guest_mapping )xenforeignmemory_unmap (xch->fmem, guest_mapping, nr_pages_mapped);for ( i =0 ; local_pages && i < nr_pfns; ++i )
PATCH v20210111 21/39 tools/guest: save: move guest_data array
In reply to Olaf HeringRemove allocation from hotpath, move guest_data array into preallocated space.
Because this was allocated with calloc:
Adjust the loop to clear unused entries as needed.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_save.c | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -223,6 +223,8 @@ struct xc_sr_save_arrays {struct iovec iov[MAX_BATCH_SIZE +4 ];/* write_batch */ uint64_t rec_pfns[MAX_BATCH_SIZE ]; +/* write_batch: Pointers to page data to send. Mapped gfns or local allocations. */ +void *guest_data[MAX_BATCH_SIZE ]; };struct xc_sr_restore_arrays {
tools/libs/guest/xg_sr_save.c +6 -5
@@ -90,7 +90,7 @@ static int write_batch(struct xc_sr_context *ctx)xc_interface *xch = ctx->xch;xen_pfn_t *mfns = ctx->save.m->mfns, *types = ctx->save.m->types;void *guest_mapping =NULL ; -void **guest_data =NULL ; +void **guest_data = ctx->save.m->guest_data;void **local_pages =NULL ;int *errors = ctx->save.m->errors, rc =-1 ;unsigned int i, p, nr_pages =0 , nr_pages_mapped =0 ;
@@ -105,12 +105,10 @@ static int write_batch(struct xc_sr_context *ctx)assert (nr_pfns !=0 ); -/* Pointers to page data to send. Mapped gfns or local allocations. */ - guest_data =calloc (nr_pfns,sizeof (*guest_data));/* Pointers to locally allocated pages. Need freeing. */ local_pages =calloc (nr_pfns,sizeof (*local_pages)); -if ( !guest_data || !local_pages ) +if ( !local_pages ) {ERROR ("Unable to allocate arrays for a batch of %u pages" , nr_pfns);
@@ -166,7 +164,10 @@ static int write_batch(struct xc_sr_context *ctx)for ( i =0 , p =0 ; i < nr_pfns; ++i ) {if (page_type_has_stream_data (types[i]) == false ) + { + guest_data[i] =NULL ;continue ; + }if ( errors[p] ) {
@@ -183,6 +184,7 @@ static int write_batch(struct xc_sr_context *ctx)if ( rc ) { + guest_data[i] =NULL ;if ( rc ==-1 && errno ==EAGAIN ) {set_bit (ctx->save.m->batch_pfns[i], ctx->save.deferred_pages);
@@ -256,7 +258,6 @@ static int write_batch(struct xc_sr_context *ctx)for ( i =0 ; local_pages && i < nr_pfns; ++i )free (local_pages[i]);free (local_pages); -free (guest_data);return rc; }
PATCH v20210111 22/39 tools/guest: save: move local_pages array
In reply to Olaf HeringRemove allocation from hotpath, move local_pages array into preallocated space.
Adjust the code to use the src page as is in case of HVM.
In case of PV the page may need to be normalised, use an private memory area for this purpose.
--- tools/libs/guest/xg_sr_common.h | 22 ++++++++++--------- tools/libs/guest/xg_sr_save.c | 25 +++------------------ tools/libs/guest/xg_sr_save_x86_hvm.c | 5 +++-- tools/libs/guest/xg_sr_save_x86_pv.c | 31 ++++++++++++++++++--------- 4 files changed, 39 insertions(+), 44 deletions(-)
tools/libs/guest/xg_sr_common.h +12 -10
@@ -33,16 +33,12 @@ struct xc_sr_save_ops* Optionally transform the contents of a page from being specific to the * sending environment, to being generic for the stream. * -* The page of data at the end of 'page' may be a read-only mapping of a -* running guest; it must not be modified. If no transformation is -* required, the callee should leave '*pages' untouched. +* The page of data '*src' may be a read-only mapping of a running guest; +* it must not be modified. If no transformation is required, the callee +* should leave '*src' untouched, and return it via '**ptr'. * -* If a transformation is required, the callee should allocate themselves -* a local page using malloc() and return it via '*page'. -* -* The caller shall free() '*page' in all cases. In the case that the -* callee encounters an error, it should *NOT* free() the memory it -* allocated for '*page'. +* If a transformation is required, the callee should provide the +* transformed page in a private buffer and return it via '**ptr'. * * It is valid to fail with EAGAIN if the transformation is not able to be * completed at this point. The page shall be retried later.
@@ -50,7 +46,7 @@ struct xc_sr_save_ops* @returns 0 for success, -1 for failure, with errno appropriately set. */ int (*normalise_page)(struct xc_sr_context *ctx,xen_pfn_t type, -void **page); +void *src,unsigned int idx,void **ptr);/** * Set up local environment to save a domain. (Typically querying
@@ -371,6 +367,12 @@ struct xc_sr_contextunion { +struct + { +/* Used by write_batch for modified pages. */ +void *normalised_pages; + } save; +struct {/* State machine for the order of received records. */
tools/libs/guest/xg_sr_save.c +3 -22
@@ -91,11 +91,10 @@ static int write_batch(struct xc_sr_context *ctx)xen_pfn_t *mfns = ctx->save.m->mfns, *types = ctx->save.m->types;void *guest_mapping =NULL ;void **guest_data = ctx->save.m->guest_data; -void **local_pages =NULL ;int *errors = ctx->save.m->errors, rc =-1 ;unsigned int i, p, nr_pages =0 , nr_pages_mapped =0 ;unsigned int nr_pfns = ctx->save.nr_batch_pfns; -void *page, *orig_page; +void *src;uint64_t *rec_pfns = ctx->save.m->rec_pfns;struct iovec *iov = ctx->save.m->iov;int iovcnt =0 ;struct xc_sr_rec_page_data_header hdr = {0 };
@@ -105,16 +104,6 @@ static int write_batch(struct xc_sr_context *ctx)assert (nr_pfns !=0 ); -/* Pointers to locally allocated pages. Need freeing. */ - local_pages =calloc (nr_pfns,sizeof (*local_pages)); - -if ( !local_pages ) - { -ERROR ("Unable to allocate arrays for a batch of %u pages" , - nr_pfns); - goto err; - } -for ( i =0 ; i < nr_pfns; ++i ) { types[i] = mfns[i] = ctx->save.ops.pfn_to_gfn(ctx,
@@ -176,11 +165,8 @@ static int write_batch(struct xc_sr_context *ctx) goto err; } - orig_page = page = guest_mapping + (p *PAGE_SIZE ); - rc = ctx->save.ops.normalise_page (ctx, types[i], &page); - -if ( orig_page != page ) - local_pages[i] = page; + src = guest_mapping + (p *PAGE_SIZE ); + rc = ctx->save.ops.normalise_page (ctx, types[i], src, i, &guest_data[i]);if ( rc ) {
@@ -195,8 +181,6 @@ static int write_batch(struct xc_sr_context *ctx)else goto err; } -else - guest_data[i] = page; rc =-1 ; ++p;
@@ -255,9 +239,6 @@ static int write_batch(struct xc_sr_context *ctx) err:if ( guest_mapping )xenforeignmemory_unmap (xch->fmem, guest_mapping, nr_pages_mapped); -for ( i =0 ; local_pages && i < nr_pfns; ++i ) -free (local_pages[i]); -free (local_pages);return rc; }
tools/libs/guest/xg_sr_save_x86_hvm.c +3 -2
@@ -129,9 +129,10 @@ static xen_pfn_t x86_hvm_pfn_to_gfn(const struct xc_sr_context *ctx,return pfn; } -static int x86_hvm_normalise_page (struct xc_sr_context *ctx, -xen_pfn_t type,void **page) +static int x86_hvm_normalise_page (struct xc_sr_context *ctx,xen_pfn_t type, +void *src,unsigned int idx,void **ptr) { + *ptr = src;return 0 ; }
tools/libs/guest/xg_sr_save_x86_pv.c +21 -10
@@ -999,29 +999,31 @@ static xen_pfn_t x86_pv_pfn_to_gfn(const struct xc_sr_context *ctx,* save_ops function. Performs pagetable normalisation on appropriate pages. */ static int x86_pv_normalise_page (struct xc_sr_context *ctx,xen_pfn_t type, -void **page) +void *src,unsigned int idx,void **ptr) {xc_interface *xch = ctx->xch; -void *local_page;int rc; +void *dst; type &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ;if ( type <XEN_DOMCTL_PFINFO_L1TAB || type >XEN_DOMCTL_PFINFO_L4TAB ) + { + *ptr = src;return 0 ; + } - local_page =malloc (PAGE_SIZE ); -if ( !local_page ) +if ( idx >=MAX_BATCH_SIZE ) { -ERROR ("Unable to allocate scratch page" ); - rc =-1 ; - goto out; +ERROR ("idx %u out of range" , idx); + errno =ERANGE ; +return -1 ; } - rc =normalise_pagetable (ctx, *page, local_page, type); - *page = local_page; + dst = ctx->x86.pv.save.normalised_pages + idx *PAGE_SIZE ; + rc =normalise_pagetable (ctx, src, dst, type); + *ptr = dst; - out:return rc; }
@@ -1031,8 +1033,16 @@ static int x86_pv_normalise_page(struct xc_sr_context *ctx, xen_pfn_t type,*/ static int x86_pv_setup (struct xc_sr_context *ctx) { +xc_interface *xch = ctx->xch;int rc; + ctx->x86.pv.save.normalised_pages =malloc (MAX_BATCH_SIZE *PAGE_SIZE ); +if ( !ctx->x86.pv.save.normalised_pages ) + { +PERROR ("Failed to allocate normalised_pages" ); +return -1 ; + } + rc =x86_pv_domain_info (ctx);if ( rc )return rc;
@@ -1118,6 +1128,7 @@ static int x86_pv_check_vm_state(struct xc_sr_context *ctx)static int x86_pv_cleanup (struct xc_sr_context *ctx) { +free (ctx->x86.pv.save.normalised_pages);free (ctx->x86.pv.p2m_pfns);if ( ctx->x86.pv.p2m )
PATCH v20210111 23/39 tools/guest: restore: move pfns array
In reply to Olaf HeringRemove allocation from hotpath, move pfns array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_restore.c | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -224,6 +224,8 @@ struct xc_sr_save_arrays { };struct xc_sr_restore_arrays { +/* handle_page_data */ +xen_pfn_t pfns[MAX_BATCH_SIZE ]; };struct xc_sr_context
tools/libs/guest/xg_sr_restore.c +2 -4
@@ -315,7 +315,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)unsigned int i, pages_of_data =0 ;int rc =-1 ; -xen_pfn_t *pfns =NULL , pfn; +xen_pfn_t *pfns = ctx->restore.m->pfns, pfn;uint32_t *types =NULL , type;/*
@@ -363,9 +363,8 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec) goto err; } - pfns =malloc (pages->count *sizeof (*pfns)); types =malloc (pages->count *sizeof (*types)); -if ( !pfns || !types ) +if ( !types ) {ERROR ("Unable to allocate enough memory for %u pfns" , pages->count);
@@ -412,7 +411,6 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec) &pages->pfn[pages->count]); err:free (types); -free (pfns);return rc; }
PATCH v20210111 24/39 tools/guest: restore: move types array
In reply to Olaf HeringRemove allocation from hotpath, move types array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 1 + tools/libs/guest/xg_sr_restore.c | 12 +----------- 2 files changed, 2 insertions(+), 11 deletions(-)
tools/libs/guest/xg_sr_common.h +1 -0
@@ -226,6 +226,7 @@ struct xc_sr_save_arrays {struct xc_sr_restore_arrays {/* handle_page_data */ xen_pfn_t pfns[MAX_BATCH_SIZE ]; +uint32_t types[MAX_BATCH_SIZE ]; };struct xc_sr_context
tools/libs/guest/xg_sr_restore.c +1 -11
@@ -316,7 +316,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)int rc =-1 ;xen_pfn_t *pfns = ctx->restore.m->pfns, pfn; -uint32_t *types =NULL , type; +uint32_t *types = ctx->restore.m->types, type;/* * v2 compatibility only exists for x86 streams. This is a bit of a
@@ -363,14 +363,6 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec) goto err; } - types =malloc (pages->count *sizeof (*types)); -if ( !types ) - { -ERROR ("Unable to allocate enough memory for %u pfns" , - pages->count); - goto err; - } -for ( i =0 ; i < pages->count; ++i ) { pfn = pages->pfn[i] &PAGE_DATA_PFN_MASK ;
@@ -410,8 +402,6 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec) rc =process_page_data (ctx, pages->count, pfns, types, &pages->pfn[pages->count]); err: -free (types); -return rc; }
PATCH v20210111 25/39 tools/guest: restore: move mfns array
In reply to Olaf HeringRemove allocation from hotpath, move mfns array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_restore.c | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -227,6 +227,8 @@ struct xc_sr_restore_arrays {/* handle_page_data */ xen_pfn_t pfns[MAX_BATCH_SIZE ];uint32_t types[MAX_BATCH_SIZE ]; +/* process_page_data */ +xen_pfn_t mfns[MAX_BATCH_SIZE ]; };struct xc_sr_context
tools/libs/guest/xg_sr_restore.c +2 -3
@@ -205,7 +205,7 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count, xen_pfn_t *pfns, uint32_t *types, void *page_data) {xc_interface *xch = ctx->xch; -xen_pfn_t *mfns =malloc (count *sizeof (*mfns)); +xen_pfn_t *mfns = ctx->restore.m->mfns;int *map_errs =malloc (count *sizeof (*map_errs));int rc;void *mapping =NULL , *guest_page =NULL ;
@@ -213,7 +213,7 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count, j,/* j indexes the subset of pfns we decide to map. */ nr_pages =0 ; -if ( !mfns || !map_errs ) +if ( !map_errs ) { rc =-1 ; ERROR("Failed to allocate %zu bytes to process page data",
@@ -299,7 +299,6 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,xenforeignmemory_unmap (xch->fmem, mapping, nr_pages);free (map_errs); -free (mfns);return rc; }
PATCH v20210111 26/39 tools/guest: restore: move map_errs array
In reply to Olaf HeringRemove allocation from hotpath, move map_errs array into preallocated space.
--- tools/libs/guest/xg_sr_common.h | 1 + tools/libs/guest/xg_sr_restore.c | 12 +----------- 2 files changed, 2 insertions(+), 11 deletions(-)
tools/libs/guest/xg_sr_common.h +1 -0
@@ -229,6 +229,7 @@ struct xc_sr_restore_arrays {uint32_t types[MAX_BATCH_SIZE ];/* process_page_data */ xen_pfn_t mfns[MAX_BATCH_SIZE ]; +int map_errs[MAX_BATCH_SIZE ]; };struct xc_sr_context
tools/libs/guest/xg_sr_restore.c +1 -11
@@ -206,21 +206,13 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count, {xc_interface *xch = ctx->xch;xen_pfn_t *mfns = ctx->restore.m->mfns; -int *map_errs =malloc (count *sizeof (*map_errs)); +int *map_errs = ctx->restore.m->map_errs;int rc;void *mapping =NULL , *guest_page =NULL ;unsigned int i,/* i indexes the pfns from the record. */ j,/* j indexes the subset of pfns we decide to map. */ nr_pages =0 ; -if ( !map_errs ) - { - rc =-1 ; -ERROR ("Failed to allocate %zu bytes to process page data" , - count * (sizeof (*mfns) +sizeof (*map_errs))); - goto err; - } - rc =populate_pfns (ctx, count, pfns, types);if ( rc ) {
@@ -298,8 +290,6 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,if ( mapping )xenforeignmemory_unmap (xch->fmem, mapping, nr_pages); -free (map_errs); -return rc; }
PATCH v20210111 27/39 tools/guest: restore: move mfns array in populate_pfns
In reply to Olaf HeringRemove allocation from hotpath, move populate_pfns mfns array into preallocated space. Use some prefix to avoid conflict with an array used in handle_page_data.
--- tools/libs/guest/xg_sr_common.h | 2 ++ tools/libs/guest/xg_sr_restore.c | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -0
@@ -230,6 +230,8 @@ struct xc_sr_restore_arrays {/* process_page_data */ xen_pfn_t mfns[MAX_BATCH_SIZE ];int map_errs[MAX_BATCH_SIZE ]; +/* populate_pfns */ +xen_pfn_t pp_mfns[MAX_BATCH_SIZE ]; };struct xc_sr_context
tools/libs/guest/xg_sr_restore.c +2 -3
@@ -138,12 +138,12 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count, const xen_pfn_t *original_pfns, const uint32_t *types) { xc_interface *xch = ctx->xch; -xen_pfn_t *mfns =malloc (count *sizeof (*mfns)), +xen_pfn_t *mfns = ctx->restore.m->pp_mfns, *pfns =malloc (count *sizeof (*pfns));unsigned int i, nr_pfns =0 ;int rc =-1 ; -if ( !mfns || !pfns ) +if ( !pfns ) {ERROR ("Failed to allocate %zu bytes for populating the physmap" ,2 * count *sizeof (*mfns));
@@ -191,7 +191,6 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count, err:free (pfns); -free (mfns);return rc; }
PATCH v20210111 28/39 tools/guest: restore: move pfns array in populate_pfns
In reply to Olaf HeringRemove allocation from hotpath, move populate_pfns' pfns array into preallocated space. Use some prefix to avoid conflict with an array used in handle_page_data.
--- tools/libs/guest/xg_sr_common.h | 1 + tools/libs/guest/xg_sr_restore.c | 11 +---------- 2 files changed, 2 insertions(+), 10 deletions(-)
tools/libs/guest/xg_sr_common.h +1 -0
@@ -232,6 +232,7 @@ struct xc_sr_restore_arrays {int map_errs[MAX_BATCH_SIZE ];/* populate_pfns */ xen_pfn_t pp_mfns[MAX_BATCH_SIZE ]; +xen_pfn_t pp_pfns[MAX_BATCH_SIZE ]; };struct xc_sr_context
tools/libs/guest/xg_sr_restore.c +1 -10
@@ -139,17 +139,10 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count, {xc_interface *xch = ctx->xch;xen_pfn_t *mfns = ctx->restore.m->pp_mfns, - *pfns =malloc (count *sizeof (*pfns)); + *pfns = ctx->restore.m->pp_pfns;unsigned int i, nr_pfns =0 ;int rc =-1 ; -if ( !pfns ) - { -ERROR ("Failed to allocate %zu bytes for populating the physmap" , -2 * count *sizeof (*mfns)); - goto err; - } -for ( i =0 ; i < count; ++i ) { if ( (!types ||
@@ -190,8 +183,6 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count, rc =0 ; err: -free (pfns); -return rc; }
PATCH v20210111 29/39 tools/guest: restore: split record processing
In reply to Olaf Heringhandle_page_data must be able to read directly into mapped guest memory. This will avoid unneccesary memcpy calls for data which can be consumed verbatim.
Rearrange the code to allow decisions based on the incoming record.
This change is preparation for future changes in handle_page_data, no change in behavior is intended.
--- tools/libs/guest/xg_sr_common.c | 33 ++++++++++++--------- tools/libs/guest/xg_sr_common.h | 4 ++- tools/libs/guest/xg_sr_restore.c | 49 ++++++++++++++++++++++---------- tools/libs/guest/xg_sr_save.c | 7 ++++- 4 files changed, 63 insertions(+), 30 deletions(-)
tools/libs/guest/xg_sr_common.c +20 -13
@@ -91,26 +91,33 @@ int write_split_record(struct xc_sr_context *ctx, struct xc_sr_record *rec,return -1 ; } -int read_record (struct xc_sr_context *ctx,int fd,struct xc_sr_record *rec) +int read_record_header (struct xc_sr_context *ctx,int fd,struct xc_sr_rhdr *rhdr) {xc_interface *xch = ctx->xch; -struct xc_sr_rhdr rhdr; -size_t datasz; -if (read_exact (fd, &rhdr,sizeof (rhdr)) ) +if (read_exact (fd, rhdr,sizeof (*rhdr)) ) {PERROR ("Failed to read Record Header from stream" );return -1 ; } -if ( rhdr.length >REC_LENGTH_MAX ) +if ( rhdr->length >REC_LENGTH_MAX ) { -ERROR ("Record (0x%08x, %s) length %#x exceeds max (%#x)" , rhdr.type, -rec_type_to_str (rhdr.type), rhdr.length,REC_LENGTH_MAX ); +ERROR ("Record (0x%08x, %s) length %#x exceeds max (%#x)" , rhdr->type, +rec_type_to_str (rhdr->type), rhdr->length,REC_LENGTH_MAX );return -1 ; } - datasz =ROUNDUP (rhdr.length,REC_ALIGN_ORDER ); +return 0 ; +} + +int read_record_data (struct xc_sr_context *ctx,int fd,struct xc_sr_rhdr *rhdr, +struct xc_sr_record *rec) +{ +xc_interface *xch = ctx->xch; +size_t datasz; + + datasz =ROUNDUP (rhdr->length,REC_ALIGN_ORDER );if ( datasz ) {
@@ -119,7 +126,7 @@ int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec)if ( !rec->data ) {ERROR ("Unable to allocate %zu bytes for record data (0x%08x, %s)" , - datasz, rhdr.type,rec_type_to_str (rhdr.type)); + datasz, rhdr->type,rec_type_to_str (rhdr->type));return -1 ; }
@@ -128,18 +135,18 @@ int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec)free (rec->data); rec->data =NULL ;PERROR ("Failed to read %zu bytes of data for record (0x%08x, %s)" , - datasz, rhdr.type,rec_type_to_str (rhdr.type)); + datasz, rhdr->type,rec_type_to_str (rhdr->type));return -1 ; } }else rec->data =NULL ; - rec->type = rhdr.type; - rec->length = rhdr.length; + rec->type = rhdr->type; + rec->length = rhdr->length;return 0 ; -}; +}static void __attribute__((unused))build_assertions (void ) {
tools/libs/guest/xg_sr_common.h +3 -1
@@ -475,7 +475,9 @@ static inline int write_record(struct xc_sr_context *ctx,* * On failure, the contents of the record structure are undefined. */ -int read_record (struct xc_sr_context *ctx,int fd,struct xc_sr_record *rec); +int read_record_header (struct xc_sr_context *ctx,int fd,struct xc_sr_rhdr *rhdr); +int read_record_data (struct xc_sr_context *ctx,int fd,struct xc_sr_rhdr *rhdr, +struct xc_sr_record *rec);/* * This would ideally be private in restore.c, but is needed by
tools/libs/guest/xg_sr_restore.c +34 -15
125 lines not on this page · raw message
tools/libs/guest/xg_sr_save.c +6 -1
@@ -590,6 +590,7 @@ static int send_memory_live(struct xc_sr_context *ctx)static int colo_merge_secondary_dirty_bitmap (struct xc_sr_context *ctx) {xc_interface *xch = ctx->xch; +struct xc_sr_rhdr rhdr;struct xc_sr_record rec;uint64_t *pfns =NULL ;uint64_t pfn;
@@ -598,7 +599,11 @@ static int colo_merge_secondary_dirty_bitmap(struct xc_sr_context *ctx) DECLARE_HYPERCALL_BUFFER_SHADOW(unsigned long, dirty_bitmap, &ctx->save.dirty_bitmap_hbuf); - rc =read_record (ctx, ctx->save.recv_fd, &rec); + rc =read_record_header (ctx, ctx->save.recv_fd, &rhdr); +if ( rc ) + goto err; + + rc =read_record_data (ctx, ctx->save.recv_fd, &rhdr, &rec);if ( rc ) goto err;
PATCH v20210111 30/39 tools/guest: restore: split handle_page_data
In reply to Olaf Heringhandle_page_data must be able to read directly into mapped guest memory. This will avoid unneccesary memcpy calls for data that can be consumed verbatim.
Split the various steps of record processing:
- move processing to handle_buffered_page_data
- adjust xenforeignmemory_map to set errno in case of failure
- adjust verify mode to set errno in case of failure
This change is preparation for future changes in handle_page_data, no change in behavior is intended.
--- tools/libs/guest/xg_sr_common.h | 9 + tools/libs/guest/xg_sr_restore.c | 343 ++++++++++++++++++++----------- 2 files changed, 231 insertions(+), 121 deletions(-)
tools/libs/guest/xg_sr_common.h +9 -0
@@ -230,9 +230,14 @@ struct xc_sr_restore_arrays {/* process_page_data */ xen_pfn_t mfns[MAX_BATCH_SIZE ];int map_errs[MAX_BATCH_SIZE ]; +void *guest_data[MAX_BATCH_SIZE ]; +/* populate_pfns */ xen_pfn_t pp_mfns[MAX_BATCH_SIZE ];xen_pfn_t pp_pfns[MAX_BATCH_SIZE ]; + +/* Must be the last member */ +struct xc_sr_rec_page_data_header pages; };struct xc_sr_context
@@ -323,7 +328,11 @@ struct xc_sr_context/* Sender has invoked verify mode on the stream. */ bool verify; +void *verify_buf; +struct xc_sr_restore_arrays *m; +void *guest_mapping; +uint32_t nr_mapped_pages; } restore; };
tools/libs/guest/xg_sr_restore.c +222 -121
412 lines not on this page · raw message
PATCH v20210111 31/39 tools/guest: restore: write data directly into guest
In reply to Olaf HeringRead incoming migration stream directly into the guest memory. This avoids the memory allocation and copying, and the resulting performance penalty.
--- tools/libs/guest/xg_sr_common.h | 1 + tools/libs/guest/xg_sr_restore.c | 132 ++++++++++++++++++++++++++++++- 2 files changed, 129 insertions(+), 4 deletions(-)
tools/libs/guest/xg_sr_common.h +1 -0
@@ -231,6 +231,7 @@ struct xc_sr_restore_arrays {xen_pfn_t mfns[MAX_BATCH_SIZE ];int map_errs[MAX_BATCH_SIZE ];void *guest_data[MAX_BATCH_SIZE ]; +struct iovec iov[MAX_BATCH_SIZE ];/* populate_pfns */ xen_pfn_t pp_mfns[MAX_BATCH_SIZE ];
tools/libs/guest/xg_sr_restore.c +128 -4
145 lines not on this page · raw message
PATCH v20210111 32/39 tools: remove tabs from code produced by libxl_save_msgs_gen.pl
In reply to Olaf Hering--- tools/libs/light/libxl_save_msgs_gen.pl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
tools/libs/light/libxl_save_msgs_gen.pl +10 -10
@@ -120,8 +120,8 @@ sub typeid ($) { my ($t) = @_; $t =~ s/\W/_/; return $t; }; $out_body{'callout'} .= <<END; static int bytes_get(const unsigned char **msg, - const unsigned char *const endmsg, - void *result, int rlen) + const unsigned char *const endmsg, + void *result, int rlen) { if (endmsg - *msg < rlen) return 0; memcpy(result, *msg, rlen);
@@ -132,11 +132,11 @@ static int bytes_get(const unsigned char **msg, END $out_body{'helper'} .= <<END; static void bytes_put(unsigned char *const buf, int *len, - const void *value, int vlen) + const void *value, int vlen) { assert(vlen < INT_MAX/2 - *len); if (buf) - memcpy(buf + *len, value, vlen); + memcpy(buf + *len, value, vlen); *len += vlen; }
@@ -155,7 +155,7 @@ static int ${typeid}_get(const unsigned char **msg, END $out_body{'helper'} .= <<END; static void ${typeid}_put(unsigned char *const buf, int *len, - const $simpletype value) + const $simpletype value) { bytes_put(buf, len, &value, sizeof(value)); }
@@ -192,15 +192,15 @@ END $out_body{'helper'} .= <<END; static void BLOCK_put(unsigned char *const buf, int *len, - const uint8_t *bytes, uint32_t size) + const uint8_t *bytes, uint32_t size) { uint32_t_put(buf, len, size); bytes_put(buf, len, bytes, size); } static void STRING_put(unsigned char *const buf, - int *len, - const char *string) + int *len, + const char *string) { size_t slen = strlen(string); assert(slen < INT_MAX / 4);
@@ -216,7 +216,7 @@ foreach my $sr (qw(save restore)) { "(void *data)"); f_decl("${receiveds}_${sr}", 'callout', 'int', - "(const unsigned char *msg, uint32_t len, void *user)"); + "(const unsigned char *msg, uint32_t len, void *user)"); f_decl("${enumcallbacks}_${sr}", 'callout', 'unsigned', "(const ".cbtype($sr)." *cbs)");
@@ -301,7 +301,7 @@ END_ALWAYS $c_callback_args .= "$c_args, "; $c_recv.= " if (!${typeid}_get(&msg, endmsg, $c_get_args)) return 0;\n"; - f_more("${encode}_$name", " ${typeid}_put(buf, &len, $c_args);\n"); + f_more("${encode}_$name", " ${typeid}_put(buf, &len, $c_args);\n"); } $f_more_sr->($c_recv); $c_decl .= "void *user)";
Re: [PATCH v20210111 32/39] tools: remove tabs from code produced by libxl_save_msgs_gen.pl
Olaf Hering writes ("[PATCH v20210111 32/39] tools: remove tabs from code produced by libxl_save_msgs_gen.pl"):
Quoted 1 line
Signed-off-by Olaf Hering <[email protected]>
Personally I think this is busywork but I am happy to take the patch.
Release risk is clearly negligible since it's a whitespace change.
PATCH v20210111 33/39 tools: recognize LIBXL_API_VERSION for 4.15
In reply to Olaf HeringThis is required by upcoming API changes.
--- tools/include/libxl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
tools/include/libxl.h +2 -1
@@ -690,7 +690,8 @@ typedef struct libxl__ctx libxl_ctx;#if LIBXL_API_VERSION !=0x040200 &&LIBXL_API_VERSION !=0x040300 && \LIBXL_API_VERSION !=0x040400 &&LIBXL_API_VERSION !=0x040500 && \LIBXL_API_VERSION !=0x040700 &&LIBXL_API_VERSION !=0x040800 && \ -LIBXL_API_VERSION !=0x041300 &&LIBXL_API_VERSION !=0x041400 +LIBXL_API_VERSION !=0x041300 &&LIBXL_API_VERSION !=0x041400 && \ +LIBXL_API_VERSION !=0x041500 #error Unknown LIBXL_API_VERSION#endif #endif
PATCH v20210111 34/39 tools: adjust libxl_domain_suspend to receive a struct props
In reply to Olaf HeringUpcoming changes will pass more knobs down to xc_domain_save. Adjust the libxl_domain_suspend API to allow easy adding of additional knobs.
No change in behavior intented.
--- tools/include/libxl.h | 26 +++++++++++++++++++++++--- tools/libs/light/libxl_domain.c | 7 ++++--- tools/ocaml/libs/xl/xenlight_stubs.c | 3 ++- tools/xl/xl_migrate.c | 9 ++++++--- tools/xl/xl_saverestore.c | 3 ++- 5 files changed, 37 insertions(+), 11 deletions(-)
tools/include/libxl.h +23 -3
@@ -1667,12 +1667,32 @@ static inline int libxl_retrieve_domain_configuration_0x041200( libxl_retrieve_domain_configuration_0x041200#endif +/* +* LIBXL_HAVE_DOMAIN_SUSPEND_PROPS indicates that the +* libxl_domain_suspend_props() function takes a props struct. +*/ +#define LIBXL_HAVE_DOMAIN_SUSPEND_PROPS 1 + +typedef struct { +uint32_t flags;/* LIBXL_SUSPEND_* */ +}libxl_domain_suspend_props ; +#define LIBXL_SUSPEND_DEBUG 1 +#define LIBXL_SUSPEND_LIVE 2 + int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ + libxl_domain_suspend_props *props, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; -#define LIBXL_SUSPEND_DEBUG 1 -#define LIBXL_SUSPEND_LIVE 2 +#if defined(LIBXL_API_VERSION ) &&LIBXL_API_VERSION <0x041500 +static inline int libxl_domain_suspend_0x041400 (libxl_ctx *ctx,uint32_t domid, +int fd,int flags,/* LIBXL_SUSPEND_* */ +const libxl_asyncop_how *ao_how) +{ +libxl_domain_suspend_props props = { .flags = flags, }; +return libxl_domain_suspend (ctx, domid, fd, &props, ao_how); +} +#define libxl_domain_suspend libxl_domain_suspend_0x041400 +#endif /* * Only suspend domain, do not save its state to file, do not destroy it.
tools/libs/light/libxl_domain.c +4 -3
@@ -505,7 +505,8 @@ static void domain_suspend_cb(libxl__egc *egc, } -int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +libxl_domain_suspend_props *props,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);
@@ -526,8 +527,8 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags, dss->domid = domid; dss->fd = fd; dss->type = type; - dss->live = flags &LIBXL_SUSPEND_LIVE ; - dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->live = props->flags &LIBXL_SUSPEND_LIVE ; + dss->debug = props->flags &LIBXL_SUSPEND_DEBUG ; dss->checkpointed_stream =LIBXL_CHECKPOINTED_STREAM_NONE ; rc = libxl__fd_flags_modify_save(gc, dss->fd,
tools/ocaml/libs/xl/xenlight_stubs.c +2 -1
@@ -614,10 +614,11 @@ value stub_libxl_domain_suspend(value ctx, value domid, value fd, value async, vint ret;uint32_t c_domid =Int_val (domid);int c_fd =Int_val (fd); +libxl_domain_suspend_props props = {};libxl_asyncop_how *ao_how =aohow_val (async);caml_enter_blocking_section (); - ret =libxl_domain_suspend (CTX , c_domid, c_fd,0 , ao_how); + ret =libxl_domain_suspend (CTX , c_domid, c_fd, &props, ao_how);caml_leave_blocking_section ();free (ao_how);
tools/xl/xl_migrate.c +6 -3
@@ -188,7 +188,10 @@ static void migrate_domain(uint32_t domid, int preserve_domid,char *away_domname;char rc_buf;uint8_t *config_data; -int config_len, flags =LIBXL_SUSPEND_LIVE ; +int config_len; +libxl_domain_suspend_props props = { + .flags =LIBXL_SUSPEND_LIVE , + };unsigned xtl_flags =XTL_STDIOSTREAM_HIDE_PROGRESS ; save_domain_core_begin(domid, preserve_domid, override_config_file,
@@ -210,8 +213,8 @@ static void migrate_domain(uint32_t domid, int preserve_domid,xtl_stdiostream_adjust_flags (logger, xtl_flags,0 );if (debug) - flags |=LIBXL_SUSPEND_DEBUG ; - rc =libxl_domain_suspend (ctx, domid, send_fd, flags,NULL ); + props.flags |=LIBXL_SUSPEND_DEBUG ; + rc =libxl_domain_suspend (ctx, domid, send_fd, &props,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
tools/xl/xl_saverestore.c +2 -1
@@ -130,6 +130,7 @@ static int save_domain(uint32_t domid, int preserve_domid,int fd;uint8_t *config_data;int config_len; +libxl_domain_suspend_props props = {};save_domain_core_begin (domid, preserve_domid, override_config_file, &config_data, &config_len);
@@ -146,7 +147,7 @@ static int save_domain(uint32_t domid, int preserve_domid,save_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd, &props,NULL );close (fd);if (rc <0 ) {
Re: [PATCH v20210111 34/39] tools: adjust libxl_domain_suspend to receive a struct props
Quoted 2 lines
Upcoming changes will pass more knobs down to
xc_domain_save. Adjust thelibxl_domain_suspendAPI to allow easy adding of additional knobs.
________________________________________
From: Olaf Hering <[email protected]>
Sent: 11 January 2021 17:42
To: [email protected]
Cc: Olaf Hering; Ian Jackson; Wei Liu; Anthony Perard; Christian Lindig; David Scott
Subject: [PATCH v20210111 34/39] tools: adjust libxl_domain_suspend to receive a struct props
Upcoming changes will pass more knobs down to xc_domain_save. Adjust the libxl_domain_suspend API to allow easy adding of additional knobs.
No change in behavior intented.
--- tools/include/libxl.h | 26 +++++++++++++++++++++++--- tools/libs/light/libxl_domain.c | 7 ++++--- tools/ocaml/libs/xl/xenlight_stubs.c | 3 ++- tools/xl/xl_migrate.c | 9 ++++++--- tools/xl/xl_saverestore.c | 3 ++- 5 files changed, 37 insertions(+), 11 deletions(-)
tools/include/libxl.h +23 -3
@@ -1667,12 +1667,32 @@ static inline int libxl_retrieve_domain_configuration_0x041200( libxl_retrieve_domain_configuration_0x041200#endif +/* +* LIBXL_HAVE_DOMAIN_SUSPEND_PROPS indicates that the +* libxl_domain_suspend_props() function takes a props struct. +*/ +#define LIBXL_HAVE_DOMAIN_SUSPEND_PROPS 1 + +typedef struct { +uint32_t flags;/* LIBXL_SUSPEND_* */ +}libxl_domain_suspend_props ; +#define LIBXL_SUSPEND_DEBUG 1 +#define LIBXL_SUSPEND_LIVE 2 + int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, - int flags, /* LIBXL_SUSPEND_* */ + libxl_domain_suspend_props *props, const libxl_asyncop_how *ao_how) LIBXL_EXTERNAL_CALLERS_ONLY; -#define LIBXL_SUSPEND_DEBUG 1 -#define LIBXL_SUSPEND_LIVE 2 +#if defined(LIBXL_API_VERSION ) &&LIBXL_API_VERSION <0x041500 +static inline int libxl_domain_suspend_0x041400 (libxl_ctx *ctx,uint32_t domid, +int fd,int flags,/* LIBXL_SUSPEND_* */ +const libxl_asyncop_how *ao_how) +{ +libxl_domain_suspend_props props = { .flags = flags, }; +return libxl_domain_suspend (ctx, domid, fd, &props, ao_how); +} +#define libxl_domain_suspend libxl_domain_suspend_0x041400 +#endif /* * Only suspend domain, do not save its state to file, do not destroy it.
tools/libs/light/libxl_domain.c +4 -3
@@ -505,7 +505,8 @@ static void domain_suspend_cb(libxl__egc *egc, } -int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd,int flags, +int libxl_domain_suspend (libxl_ctx *ctx,uint32_t domid,int fd, +libxl_domain_suspend_props *props,const libxl_asyncop_how *ao_how) {AO_CREATE (ctx, domid, ao_how);
@@ -526,8 +527,8 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags, dss->domid = domid; dss->fd = fd; dss->type = type; - dss->live = flags &LIBXL_SUSPEND_LIVE ; - dss->debug = flags &LIBXL_SUSPEND_DEBUG ; + dss->live = props->flags &LIBXL_SUSPEND_LIVE ; + dss->debug = props->flags &LIBXL_SUSPEND_DEBUG ; dss->checkpointed_stream =LIBXL_CHECKPOINTED_STREAM_NONE ; rc = libxl__fd_flags_modify_save(gc, dss->fd,
tools/ocaml/libs/xl/xenlight_stubs.c +2 -1
@@ -614,10 +614,11 @@ value stub_libxl_domain_suspend(value ctx, value domid, value fd, value async, vint ret;uint32_t c_domid =Int_val (domid);int c_fd =Int_val (fd); +libxl_domain_suspend_props props = {};libxl_asyncop_how *ao_how =aohow_val (async);caml_enter_blocking_section (); - ret =libxl_domain_suspend (CTX , c_domid, c_fd,0 , ao_how); + ret =libxl_domain_suspend (CTX , c_domid, c_fd, &props, ao_how);caml_leave_blocking_section ();free (ao_how);
tools/xl/xl_migrate.c +6 -3
@@ -188,7 +188,10 @@ static void migrate_domain(uint32_t domid, int preserve_domid,char *away_domname;char rc_buf;uint8_t *config_data; -int config_len, flags =LIBXL_SUSPEND_LIVE ; +int config_len; +libxl_domain_suspend_props props = { + .flags =LIBXL_SUSPEND_LIVE , + };unsigned xtl_flags =XTL_STDIOSTREAM_HIDE_PROGRESS ; save_domain_core_begin(domid, preserve_domid, override_config_file,
@@ -210,8 +213,8 @@ static void migrate_domain(uint32_t domid, int preserve_domid,xtl_stdiostream_adjust_flags (logger, xtl_flags,0 );if (debug) - flags |=LIBXL_SUSPEND_DEBUG ; - rc =libxl_domain_suspend (ctx, domid, send_fd, flags,NULL ); + props.flags |=LIBXL_SUSPEND_DEBUG ; + rc =libxl_domain_suspend (ctx, domid, send_fd, &props,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc);
tools/xl/xl_saverestore.c +2 -1
@@ -130,6 +130,7 @@ static int save_domain(uint32_t domid, int preserve_domid,int fd;uint8_t *config_data;int config_len; +libxl_domain_suspend_props props = {};save_domain_core_begin (domid, preserve_domid, override_config_file, &config_data, &config_len);
@@ -146,7 +147,7 @@ static int save_domain(uint32_t domid, int preserve_domid,save_domain_core_writeconfig (fd, filename, config_data, config_len); -int rc =libxl_domain_suspend (ctx, domid, fd,0 ,NULL ); +int rc =libxl_domain_suspend (ctx, domid, fd, &props,NULL );close (fd);if (rc <0 ) {
PATCH v20210111 35/39 tools: change struct precopy_stats to precopy_stats_t
In reply to Olaf HeringThis will help libxl_save_msgs_gen.pl to copy the struct as a region of memory.
No change in behavior intented.
--- tools/include/xenguest.h | 7 +++---- tools/libs/guest/xg_sr_common.h | 2 +- tools/libs/guest/xg_sr_save.c | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-)
tools/include/xenguest.h +3 -4
@@ -435,18 +435,17 @@ static inline xen_pfn_t xc_dom_p2m(struct xc_dom_image *dom, xen_pfn_t pfn)struct xenevtchn_handle ;/* For save's precopy_policy(). */ -struct precopy_stats -{ +typedef struct {unsigned int iteration;unsigned long total_written;long dirty_count;/* -1 if unknown */ -}; +}precopy_stats_t ;/* * A precopy_policy callback may not be running in the same address * space as libxc an so precopy_stats is passed by value. */ -typedef int (*precopy_policy_t )(struct precopy_stats ,void *); +typedef int (*precopy_policy_t )(precopy_stats_t ,void *);/* callbacks provided by xc_domain_save */ struct save_callbacks {
tools/libs/guest/xg_sr_common.h +1 -1
@@ -271,7 +271,7 @@ struct xc_sr_contextsize_t pages_sent;size_t overhead_sent; -struct precopy_stats stats; +precopy_stats_t stats;unsigned int nr_batch_pfns;unsigned long *deferred_pages;
tools/libs/guest/xg_sr_save.c +3 -3
@@ -489,7 +489,7 @@ static int update_progress_string(struct xc_sr_context *ctx, char **str)#define SPP_MAX_ITERATIONS 5#define SPP_TARGET_DIRTY_COUNT 50 -static int simple_precopy_policy (struct precopy_stats stats,void *user) +static int simple_precopy_policy (precopy_stats_t stats,void *user) { return ((stats.dirty_count >= 0 && stats.dirty_count < SPP_TARGET_DIRTY_COUNT) ||
@@ -516,13 +516,13 @@ static int send_memory_live(struct xc_sr_context *ctx)precopy_policy_t precopy_policy = ctx->save.callbacks->precopy_policy;void *data = ctx->save.callbacks->data; -struct precopy_stats *policy_stats; +precopy_stats_t *policy_stats; rc =update_progress_string (ctx, &progress_str);if ( rc ) goto out; - ctx->save.stats = (struct precopy_stats ){ + ctx->save.stats = (precopy_stats_t ){ .dirty_count = ctx->save.p2m_size, }; policy_stats = &ctx->save.stats;
PATCH v20210111 36/39 tools: add callback to libxl for precopy_policy and precopy_stats_t
In reply to Olaf HeringThis duplicates simple_precopy_policy. To recap its purpose:
- do up to 5 iterations of copying dirty domU memory to target, including the initial copying of all domU memory, excluding the final copying while the domU is suspended
- do fewer iterations in case the domU dirtied less than 50 pages
Take the opportunity to also move xen_pfn_t into qw().
--- tools/libs/light/libxl_dom_save.c | 19 +++++++++++++++++++ tools/libs/light/libxl_internal.h | 2 ++ tools/libs/light/libxl_save_msgs_gen.pl | 3 ++- 3 files changed, 23 insertions(+), 1 deletion(-)
tools/libs/light/libxl_dom_save.c +19 -0
@@ -373,6 +373,24 @@ int libxl__save_emulator_xenstore_data(libxl__domain_save_state *dss,return rc; } +static int libxl__domain_save_precopy_policy (precopy_stats_t stats,void *user) +{ +libxl__save_helper_state *shs = user; +libxl__domain_save_state *dss = shs->caller_state; +STATE_AO_GC (dss->ao); + +LOGD (DEBUG , shs->domid,"iteration %u dirty_count %ld total_written %lu" , + stats.iteration, stats.dirty_count, stats.total_written); +if (stats.dirty_count >=0 && stats.dirty_count <LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT ) + goto stop_copy; +if (stats.iteration >=LIBXL_XGS_POLICY_MAX_ITERATIONS ) + goto stop_copy; +return XGS_POLICY_CONTINUE_PRECOPY ; + +stop_copy: +return XGS_POLICY_STOP_AND_COPY ; +} +/*----- main code for saving, in order of execution -----*/ void libxl__domain_save (libxl__egc *egc,libxl__domain_save_state *dss)
@@ -430,6 +448,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss) callbacks->suspend = libxl__domain_suspend_callback; callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty; + callbacks->precopy_policy = libxl__domain_save_precopy_policy; dss->sws.ao = dss->ao; dss->sws.dss = dss;
tools/libs/light/libxl_internal.h +2 -0
@@ -124,6 +124,8 @@#define DOMID_XS_PATH "domid"#define PVSHIM_BASENAME "xen-shim"#define PVSHIM_CMDLINE "pv-shim console=xen,pv" +#define LIBXL_XGS_POLICY_MAX_ITERATIONS 5 +#define LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT 50/* Size macros. */ #define __AC (X ,Y ) (X##Y)
tools/libs/light/libxl_save_msgs_gen.pl +2 -1
@@ -23,6 +23,7 @@ our @msgs = ( STRING doing_what), 'unsigned long', 'done', 'unsigned long', 'total'] ], + [ 'scxW', "precopy_policy", ['precopy_stats_t', 'stats'] ], [ 'srcxA', "suspend", [] ], [ 'srcxA', "postcopy", [] ], [ 'srcxA', "checkpoint", [] ],
@@ -142,7 +143,7 @@ static void bytes_put(unsigned char *const buf, int *len, END -foreach my $simpletype (qw(int uint16_t uint32_t unsigned), 'unsigned long', 'xen_pfn_t') { +foreach my $simpletype (qw(int uint16_t uint32_t unsigned precopy_stats_t xen_pfn_t), 'unsigned long') { my $typeid = typeid($simpletype); $out_body{'callout'} .= <<END; static int ${typeid}_get(const unsigned char **msg,
PATCH v20210111 37/39 tools: add --max_iters to libxl_domain_suspend
In reply to Olaf HeringMigrating a large, and potentially busy, domU will take more time than neccessary due to excessive number of copying iterations.
Allow to host admin to control the number of iterations which copy cumulated domU dirty pages to the target host.
The default remains 5, which means one initial iteration to copy the entire domU memory, and up to 4 additional iterations to copy dirty memory from the still running domU. After the given number of iterations the domU is suspended, remaining dirty memory is copied and the domU is finally moved to the target host.
This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility of the new .max_iters property.
--- docs/man/xl.1.pod.in | 4 ++++ tools/include/libxl.h | 1 + tools/libs/light/libxl_dom_save.c | 2 +- tools/libs/light/libxl_domain.c | 1 + tools/libs/light/libxl_internal.h | 1 + tools/xl/xl_cmdtable.c | 3 ++- tools/xl/xl_migrate.c | 10 +++++++++- 7 files changed, 19 insertions(+), 3 deletions(-)
docs/man/xl.1.pod.in +4 -0
@@ -496,6 +496,10 @@ such that it will be identical on the destination host, unless that configuration is overridden using the B<-C> option. Note that it is not possible to use this option for a 'localhost' migration. +=item B<--max_iters> I<iterations> + +Number of copy iterations before final suspend+move (default: 5) + =back =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/include/libxl.h +1 -0
@@ -1675,6 +1675,7 @@ static inline int libxl_retrieve_domain_configuration_0x041200(typedef struct {uint32_t flags;/* LIBXL_SUSPEND_* */ +uint32_t max_iters; }libxl_domain_suspend_props ;#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2
tools/libs/light/libxl_dom_save.c +1 -1
@@ -383,7 +383,7 @@ static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user) stats.iteration, stats.dirty_count, stats.total_written);if (stats.dirty_count >=0 && stats.dirty_count <LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT ) goto stop_copy; -if (stats.iteration >=LIBXL_XGS_POLICY_MAX_ITERATIONS ) +if (stats.iteration >= dss->max_iters) goto stop_copy;return XGS_POLICY_CONTINUE_PRECOPY ;
tools/libs/light/libxl_domain.c +1 -0
@@ -527,6 +527,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, dss->domid = domid; dss->fd = fd; dss->type = type; + dss->max_iters = props->max_iters ?:LIBXL_XGS_POLICY_MAX_ITERATIONS ; dss->live = props->flags &LIBXL_SUSPEND_LIVE ; dss->debug = props->flags &LIBXL_SUSPEND_DEBUG ; dss->checkpointed_stream =LIBXL_CHECKPOINTED_STREAM_NONE ;
tools/libs/light/libxl_internal.h +1 -0
@@ -3640,6 +3640,7 @@ struct libxl__domain_save_state {int live;int debug;int checkpointed_stream; +uint32_t max_iters;const libxl_domain_remus_info *remus;/* private */ int rc;
tools/xl/xl_cmdtable.c +2 -1
@@ -170,7 +170,8 @@ struct cmd_spec cmd_table[] = {"-T Show timestamps during the migration process.\n" "--debug Verify transferred domU page data.\n" "-p Do not unpause domain after migrating it.\n" -"-D Preserve the domain id" +"-D Preserve the domain id\n" +"--max_iters N Number of copy iterations before final stop+move" }, { "restore", &main_restore, 0, 1,
tools/xl/xl_migrate.c +9 -1
@@ -180,6 +180,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,static void migrate_domain (uint32_t domid,int preserve_domid,const char *rune,int debug, +uint32_t max_iters,const char *override_config_file) {pid_t child =-1 ;
@@ -191,6 +192,7 @@ static void migrate_domain(uint32_t domid, int preserve_domid,int config_len;libxl_domain_suspend_props props = { .flags =LIBXL_SUSPEND_LIVE , + .max_iters = max_iters, };unsigned xtl_flags =XTL_STDIOSTREAM_HIDE_PROGRESS ;
@@ -555,8 +557,10 @@ int main_migrate(int argc, char **argv)char *host;int opt, daemonize =1 , monitor =1 , debug =0 , pause_after_migration =0 ;int preserve_domid =0 ; +uint32_t max_iters =0 ;static struct option opts[] = { {"debug" ,0 ,0 ,0x100 }, + {"max_iters" ,1 ,0 ,0x101 }, {"live" ,0 ,0 ,0x200 },COMMON_LONG_OPTS };
@@ -587,6 +591,9 @@ int main_migrate(int argc, char **argv)case 0x100 :/* --debug */ debug =1 ;break ; +case 0x101 :/* --max_iters */ + max_iters =atoi (optarg); +break ;case 0x200 :/* --live */ /* ignored for compatibility with xm */ break ;
@@ -621,7 +628,8 @@ int main_migrate(int argc, char **argv) pause_after_migration ? " -p" : ""); } -migrate_domain (domid, preserve_domid, rune, debug, config_filename); +migrate_domain (domid, preserve_domid, rune, debug, + max_iters, config_filename);return EXIT_SUCCESS ; }
PATCH v20210111 38/39 tools: add --min_remaining to libxl_domain_suspend
In reply to Olaf HeringThe decision to stop+move a domU to the new host must be based on two factors:
- the available network bandwidth for the migration stream
- the maximum time a workload within a domU can be savely suspended
Both values define how many dirty pages a workload may produce prior the final stop+move.
The default value of 50 pages is much too low with todays network bandwidths. On an idle 1GiB link these 200K will be transferred within ~2ms.
Give the admin a knob to adjust the point when the final stop+move will be done, so he can base this decision on his own needs.
This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility of the new .min_remaining property.
--- docs/man/xl.1.pod.in | 8 ++++++++ tools/include/libxl.h | 1 + tools/libs/light/libxl_dom_save.c | 2 +- tools/libs/light/libxl_domain.c | 1 + tools/libs/light/libxl_internal.h | 1 + tools/xl/xl_cmdtable.c | 25 +++++++++++++------------ tools/xl/xl_migrate.c | 9 ++++++++- 7 files changed, 33 insertions(+), 14 deletions(-)
docs/man/xl.1.pod.in +8 -0
@@ -500,6 +500,14 @@ possible to use this option for a 'localhost' migration. Number of copy iterations before final suspend+move (default: 5) +=item B<--min_remaing> I<pages> + +Number of remaining dirty pages. If the number of dirty pages drops that +low, the guest is suspended and the domU will finally be moved to I<host>. + +This allows the host admin to control for how long the domU will likely +be suspended during transit. + =back =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/include/libxl.h +1 -0
@@ -1676,6 +1676,7 @@ static inline int libxl_retrieve_domain_configuration_0x041200(typedef struct {uint32_t flags;/* LIBXL_SUSPEND_* */ uint32_t max_iters; +uint32_t min_remaining; }libxl_domain_suspend_props ;#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2
tools/libs/light/libxl_dom_save.c +1 -1
@@ -381,7 +381,7 @@ static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user)LOGD (DEBUG , shs->domid,"iteration %u dirty_count %ld total_written %lu" , stats.iteration, stats.dirty_count, stats.total_written); -if (stats.dirty_count >=0 && stats.dirty_count <LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT ) +if (stats.dirty_count >=0 && stats.dirty_count < dss->min_remaining) goto stop_copy;if (stats.iteration >= dss->max_iters) goto stop_copy;
tools/libs/light/libxl_domain.c +1 -0
@@ -528,6 +528,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, dss->fd = fd; dss->type = type; dss->max_iters = props->max_iters ?:LIBXL_XGS_POLICY_MAX_ITERATIONS ; + dss->min_remaining = props->min_remaining ?:LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT ; dss->live = props->flags &LIBXL_SUSPEND_LIVE ; dss->debug = props->flags &LIBXL_SUSPEND_DEBUG ; dss->checkpointed_stream =LIBXL_CHECKPOINTED_STREAM_NONE ;
tools/libs/light/libxl_internal.h +1 -0
@@ -3641,6 +3641,7 @@ struct libxl__domain_save_state {int debug;int checkpointed_stream;uint32_t max_iters; +uint32_t min_remaining;const libxl_domain_remus_info *remus;/* private */ int rc;
tools/xl/xl_cmdtable.c +13 -12
@@ -160,18 +160,19 @@ struct cmd_spec cmd_table[] = { &main_migrate,0 ,1 ,"Migrate a domain to another host" ,"[options] <Domain> <host>" , -"-h Print this help.\n" -"-C <config> Send <config> instead of config file from creation.\n" -"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" -" to sh. If empty, run <host> instead of ssh <host> xl\n" -" migrate-receive [-d -e]\n" -"-e Do not wait in the background (on <host>) for the death\n" -" of the domain.\n" -"-T Show timestamps during the migration process.\n" -"--debug Verify transferred domU page data.\n" -"-p Do not unpause domain after migrating it.\n" -"-D Preserve the domain id\n" -"--max_iters N Number of copy iterations before final stop+move" +"-h Print this help.\n" +"-C <config> Send <config> instead of config file from creation.\n" +"-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n" +" to sh. If empty, run <host> instead of ssh <host> xl\n" +" migrate-receive [-d -e]\n" +"-e Do not wait in the background (on <host>) for the death\n" +" of the domain.\n" +"-T Show timestamps during the migration process.\n" +"--debug Verify transferred domU page data.\n" +"-p Do not unpause domain after migrating it.\n" +"-D Preserve the domain id\n" +"--max_iters N Number of copy iterations before final stop+move\n" +"--min_remaining N Number of remaining dirty pages before final stop+move" }, { "restore", &main_restore, 0, 1,
tools/xl/xl_migrate.c +8 -1
@@ -181,6 +181,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,static void migrate_domain (uint32_t domid,int preserve_domid,const char *rune,int debug,uint32_t max_iters, +uint32_t min_remaining,const char *override_config_file) {pid_t child =-1 ;
@@ -193,6 +194,7 @@ static void migrate_domain(uint32_t domid, int preserve_domid,libxl_domain_suspend_props props = { .flags =LIBXL_SUSPEND_LIVE , .max_iters = max_iters, + .min_remaining = min_remaining, };unsigned xtl_flags =XTL_STDIOSTREAM_HIDE_PROGRESS ;
@@ -558,9 +560,11 @@ int main_migrate(int argc, char **argv)int opt, daemonize =1 , monitor =1 , debug =0 , pause_after_migration =0 ;int preserve_domid =0 ;uint32_t max_iters =0 ; +uint32_t min_remaining =0 ;static struct option opts[] = { {"debug" ,0 ,0 ,0x100 }, {"max_iters" ,1 ,0 ,0x101 }, + {"min_remaining" ,1 ,0 ,0x102 }, {"live" ,0 ,0 ,0x200 },COMMON_LONG_OPTS };
@@ -594,6 +598,9 @@ int main_migrate(int argc, char **argv)case 0x101 :/* --max_iters */ max_iters =atoi (optarg);break ; +case 0x102 :/* --min_remaining */ + min_remaining =atoi (optarg); +break ;case 0x200 :/* --live */ /* ignored for compatibility with xm */ break ;
@@ -629,7 +636,7 @@ int main_migrate(int argc, char **argv) }migrate_domain (domid, preserve_domid, rune, debug, - max_iters, config_filename); + max_iters, min_remaining, config_filename);return EXIT_SUCCESS ; }
PATCH v20210111 39/39 tools: add --abort_if_busy to libxl_domain_suspend
In reply to Olaf HeringProvide a knob to the host admin to abort the live migration of a running domU if the downtime during final transit will be too long for the workload within domU.
Adjust error reporting. Add ERROR_MIGRATION_ABORTED to allow callers of libxl_domain_suspend to distinguish between errors and the requested constraint.
Adjust precopy_policy to simplify reporting of remaining dirty pages. The loop in send_memory_live populates ->dirty_count in a different place than ->iteration. Let it proceeed one more time to provide the desired information before leaving the loop.
This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility of the new .abort_if_busy property.
--- docs/man/xl.1.pod.in | 8 +++++++ tools/include/libxl.h | 1 + tools/libs/light/libxl_dom_save.c | 7 ++++++- tools/libs/light/libxl_domain.c | 1 + tools/libs/light/libxl_internal.h | 2 ++ tools/libs/light/libxl_stream_write.c | 9 +++++++- tools/libs/light/libxl_types.idl | 1 + tools/xl/xl_cmdtable.c | 6 +++++- tools/xl/xl_migrate.c | 30 ++++++++++++++++++++------- 9 files changed, 55 insertions(+), 10 deletions(-)
docs/man/xl.1.pod.in +8 -0
@@ -508,6 +508,14 @@ low, the guest is suspended and the domU will finally be moved to I<host>. This allows the host admin to control for how long the domU will likely be suspended during transit. +=item B<--abort_if_busy> + +Abort migration instead of doing final suspend/move/resume if the +guest produced more than I<min_remaining> dirty pages during th number +of I<max_iters> iterations. +This avoids long periods of time where the guest is suspended, which +may confuse the workload within domU. + =back =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/include/libxl.h +1 -0
@@ -1680,6 +1680,7 @@ typedef struct { } libxl_domain_suspend_props;#define LIBXL_SUSPEND_DEBUG 1#define LIBXL_SUSPEND_LIVE 2 +#define LIBXL_SUSPEND_ABORT_IF_BUSY 4 int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, libxl_domain_suspend_props *props,
tools/libs/light/libxl_dom_save.c +6 -1
@@ -383,11 +383,16 @@ static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user) stats.iteration, stats.dirty_count, stats.total_written);if (stats.dirty_count >=0 && stats.dirty_count < dss->min_remaining) goto stop_copy; -if (stats.iteration >= dss->max_iters) +if (stats.dirty_count >=0 && stats.iteration >= dss->max_iters) goto stop_copy;return XGS_POLICY_CONTINUE_PRECOPY ; stop_copy: +if (dss->abort_if_busy) + { + dss->remaining_dirty_pages = stats.dirty_count; +return XGS_POLICY_ABORT ; + }return XGS_POLICY_STOP_AND_COPY ; }
tools/libs/light/libxl_domain.c +1 -0
@@ -529,6 +529,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, dss->type = type; dss->max_iters = props->max_iters ?:LIBXL_XGS_POLICY_MAX_ITERATIONS ; dss->min_remaining = props->min_remaining ?:LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT ; + dss->abort_if_busy = props->flags &LIBXL_SUSPEND_ABORT_IF_BUSY ; dss->live = props->flags &LIBXL_SUSPEND_LIVE ; dss->debug = props->flags &LIBXL_SUSPEND_DEBUG ; dss->checkpointed_stream =LIBXL_CHECKPOINTED_STREAM_NONE ;
tools/libs/light/libxl_internal.h +2 -0
@@ -3639,9 +3639,11 @@ struct libxl__domain_save_state {libxl_domain_type type;int live;int debug; +int abort_if_busy;int checkpointed_stream;uint32_t max_iters;uint32_t min_remaining; +long remaining_dirty_pages;const libxl_domain_remus_info *remus;/* private */ int rc;
tools/libs/light/libxl_stream_write.c +8 -1
@@ -344,11 +344,18 @@ void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void, goto err;if (retval) { +if (dss->remaining_dirty_pages) { +LOGD (NOTICE , dss->domid,"saving domain: aborted," +" %ld remaining dirty pages." , dss->remaining_dirty_pages); + }else {LOGEVD (ERROR , errnoval, dss->domid,"saving domain: %s" , dss->dsps.guest_responded ?"domain responded to suspend request" :"domain did not respond to suspend request" ); -if (!dss->dsps.guest_responded) + } +if (dss->remaining_dirty_pages) + rc =ERROR_MIGRATION_ABORTED ; +else if (!dss->dsps.guest_responded) rc =ERROR_GUEST_TIMEDOUT ;else if (dss->rc) rc = dss->rc;
tools/libs/light/libxl_types.idl +1 -0
@@ -76,6 +76,7 @@ libxl_error = Enumeration("error", [ (-30, "QMP_DEVICE_NOT_ACTIVE"), # a device has failed to be become active (-31, "QMP_DEVICE_NOT_FOUND"), # the requested device has not been found (-32, "QEMU_API"), # QEMU's replies don't contains expected members + (-33, "MIGRATION_ABORTED"), ], value_namespace = "") libxl_domain_type = Enumeration("domain_type", [
tools/xl/xl_cmdtable.c +5 -1
@@ -172,7 +172,11 @@ struct cmd_spec cmd_table[] = {"-p Do not unpause domain after migrating it.\n" "-D Preserve the domain id\n" "--max_iters N Number of copy iterations before final stop+move\n" -"--min_remaining N Number of remaining dirty pages before final stop+move" +"--min_remaining N Number of remaining dirty pages before final stop+move\n" +"--abort_if_busy Abort migration instead of doing final stop+move,\n" +" if the number of dirty pages is higher than <min_remaining>\n" +" after <max_iters> iterations. Otherwise the amount of memory\n" +" to be transfered would exceed maximum allowed domU downtime." }, { "restore", &main_restore, 0, 1,
tools/xl/xl_migrate.c +23 -7
@@ -179,7 +179,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child, }static void migrate_domain (uint32_t domid,int preserve_domid, -const char *rune,int debug, +const char *rune,int debug,int abort_if_busy,uint32_t max_iters,uint32_t min_remaining,const char *override_config_file)
@@ -218,14 +218,20 @@ static void migrate_domain(uint32_t domid, int preserve_domid,if (debug) props.flags |=LIBXL_SUSPEND_DEBUG ; +if (abort_if_busy) + props.flags |=LIBXL_SUSPEND_ABORT_IF_BUSY ; rc =libxl_domain_suspend (ctx, domid, send_fd, &props,NULL );if (rc) {fprintf (stderr,"migration sender: libxl_domain_suspend failed" " (rc=%d)\n" , rc); -if (rc ==ERROR_GUEST_TIMEDOUT ) - goto failed_suspend; -else - goto failed_resume; +switch (rc) { +case ERROR_GUEST_TIMEDOUT : + goto failed_suspend; +case ERROR_MIGRATION_ABORTED : + goto failed_busy; +default : + goto failed_resume; + } }//fprintf(stderr, "migration sender: Transfer complete.\n");
@@ -307,6 +313,12 @@ static void migrate_domain(uint32_t domid, int preserve_domid,fprintf (stderr,"Migration failed, failed to suspend at sender.\n" );exit (EXIT_FAILURE ); + failed_busy: +close (send_fd); +migration_child_report (recv_fd); +fprintf (stderr,"Migration aborted as requested, domain is too busy.\n" ); +exit (EXIT_FAILURE ); + failed_resume:close (send_fd);migration_child_report (recv_fd);
@@ -558,13 +570,14 @@ int main_migrate(int argc, char **argv)char *rune =NULL ;char *host;int opt, daemonize =1 , monitor =1 , debug =0 , pause_after_migration =0 ; -int preserve_domid =0 ; +int preserve_domid =0 , abort_if_busy =0 ;uint32_t max_iters =0 ;uint32_t min_remaining =0 ;static struct option opts[] = { {"debug" ,0 ,0 ,0x100 }, {"max_iters" ,1 ,0 ,0x101 }, {"min_remaining" ,1 ,0 ,0x102 }, + {"abort_if_busy" ,0 ,0 ,0x103 }, {"live" ,0 ,0 ,0x200 },COMMON_LONG_OPTS };
@@ -601,6 +614,9 @@ int main_migrate(int argc, char **argv)case 0x102 :/* --min_remaining */ min_remaining =atoi (optarg);break ; +case 0x103 :/* --abort_if_busy */ + abort_if_busy =1 ; +break ;case 0x200 :/* --live */ /* ignored for compatibility with xm */ break ;
@@ -635,7 +651,7 @@ int main_migrate(int argc, char **argv) pause_after_migration ? " -p" : ""); } -migrate_domain (domid, preserve_domid, rune, debug, +migrate_domain (domid, preserve_domid, rune, debug, abort_if_busy, max_iters, min_remaining, config_filename);return EXIT_SUCCESS ; }