Skip to contentSkip to conversation

leftover from 2020

PATCH Superseded Olaf Hering 6 participants 86 messages Jul 1, 2021 – Jul 13, 2021

v20210701 of 20210713 Superseded Latest is v20210713 →

lore
Share & tools
lore
#0 lore raw

v20210701 cover leftover from 2020

Various unreviewed changes, rebase to f95b7b37cf.

Olaf Hering (40):
hotplug/Linux: fix starting of xenstored with restarting systemd
tools: add API to work with sevaral bits at once
xl: fix description of migrate --debug
tools: use integer division in convert-legacy-stream
tools: handle libxl__physmap_info.name properly in convert-legacy-stream
tools: fix Python3.4 TypeError in format string
tools: create libxensaverestore
MAINTAINERS: add myself as saverestore maintainer
tools: add readv_exact to libxenctrl
tools: add xc_is_known_page_type to libxenctrl
tools: use sr_is_known_page_type
tools: unify type checking for data pfns in migration stream
tools: unify type checking for data pfns in migration stream
tools: show migration transfer rate in send_dirty_pages
tools: prepare to allocate saverestore arrays once
tools: save: move mfns array
tools: save: move types array
tools: save: move errors array
tools: save: move iov array
tools: save: move rec_pfns array
tools: save: move guest_data array
tools: save: move local_pages array
tools: restore: move types array
tools: restore: move mfns array
tools: restore: move map_errs array
tools: restore: move mfns array in populate_pfns
tools: restore: move pfns array in populate_pfns
tools: restore: split record processing
tools: restore: split handle_page_data
tools: restore: write data directly into guest
tools: recognize LIBXL_API_VERSION for 4.16
tools: adjust libxl_domain_suspend to receive a struct props
tools: change struct precopy_stats to precopy_stats_t
tools: add callback to libxl for precopy_policy and precopy_stats_t
tools: add --max_iters to libxl_domain_suspend
tools: add --min_remaining to libxl_domain_suspend
tools: add --abort_if_busy to libxl_domain_suspend
tools: add API for expandable bitmaps
tools: use xg_sr_bitmap for populated_pfns
tools/libxc: use superpages during restore of HVM guest

 .gitignore                                    |   2 +
 MAINTAINERS                                   |   6 +
 docs/man/xl.1.pod.in                          |  22 +-
 tools/hotplug/Linux/init.d/xencommons.in      |   2 +-
 tools/hotplug/Linux/launch-xenstore.in        |  40 +-
 .../Linux/systemd/xenstored.service.in        |   2 +-
 tools/include/libxl.h                         |  32 +-
 tools/include/xenguest.h                      | 186 -----
 tools/include/xensaverestore.h                | 207 ++++++
 tools/libs/Makefile                           |   1 +
 tools/libs/ctrl/xc_bitops.h                   |  28 +
 tools/libs/ctrl/xc_private.c                  |  57 +-
 tools/libs/ctrl/xc_private.h                  |   1 +
 tools/libs/guest/Makefile                     |  11 -
 tools/libs/guest/xg_dom_x86.c                 |   5 -
 tools/libs/guest/xg_offline_page.c            |   1 -
 tools/libs/guest/xg_private.h                 |   5 +
 tools/libs/guest/xg_sr_restore_x86_hvm.c      | 274 --------
 tools/libs/light/Makefile                     |   4 +-
 tools/libs/light/libxl_dom_save.c             |  24 +
 tools/libs/light/libxl_domain.c               |  10 +-
 tools/libs/light/libxl_internal.h             |   7 +
 tools/libs/light/libxl_save_helper.c          |   1 +
 tools/libs/light/libxl_save_msgs_gen.pl       |   5 +-
 tools/libs/light/libxl_stream_write.c         |   9 +-
 tools/libs/light/libxl_types.idl              |   1 +
 tools/libs/saverestore/Makefile               |  38 ++
 .../xg_sr_common.c => saverestore/common.c}   |  75 +-
 .../xg_sr_common.h => saverestore/common.h}   | 271 +++++++-
 .../common_x86.c}                             |   2 +-
 .../common_x86.h}                             |   2 +-
 .../common_x86_pv.c}                          |   2 +-
 .../common_x86_pv.h}                          |   2 +-
 .../nomigrate.c}                              |   2 +-
 .../xg_sr_restore.c => saverestore/restore.c} | 617 +++++++++--------
 tools/libs/saverestore/restore_x86_hvm.c      | 645 ++++++++++++++++++
 .../restore_x86_pv.c}                         |  70 +-
 .../xg_sr_save.c => saverestore/save.c}       | 165 ++---
 .../save_restore.h}                           |   2 -
 .../save_x86_hvm.c}                           |   7 +-
 .../save_x86_pv.c}                            |  33 +-
 .../stream_format.h}                          |   0
 tools/libs/uselibs.mk                         |   4 +-
 tools/ocaml/libs/xl/xenlight_stubs.c          |   3 +-
 tools/python/scripts/convert-legacy-stream    |  24 +-
 tools/xl/xl_cmdtable.c                        |  26 +-
 tools/xl/xl_migrate.c                         |  54 +-
 tools/xl/xl_saverestore.c                     |   3 +-
 48 files changed, 2037 insertions(+), 953 deletions(-)
 create mode 100644 tools/include/xensaverestore.h
 delete mode 100644 tools/libs/guest/xg_sr_restore_x86_hvm.c
 create mode 100644 tools/libs/saverestore/Makefile
 rename tools/libs/{guest/xg_sr_common.c => saverestore/common.c} (72%)
 rename tools/libs/{guest/xg_sr_common.h => saverestore/common.h} (67%)
 rename tools/libs/{guest/xg_sr_common_x86.c => saverestore/common_x86.c} (99%)
 rename tools/libs/{guest/xg_sr_common_x86.h => saverestore/common_x86.h} (98%)
 rename tools/libs/{guest/xg_sr_common_x86_pv.c => saverestore/common_x86_pv.c} (99%)
 rename tools/libs/{guest/xg_sr_common_x86_pv.h => saverestore/common_x86_pv.h} (98%)
 rename tools/libs/{guest/xg_nomigrate.c => saverestore/nomigrate.c} (98%)
 rename tools/libs/{guest/xg_sr_restore.c => saverestore/restore.c} (66%)
 create mode 100644 tools/libs/saverestore/restore_x86_hvm.c
 rename tools/libs/{guest/xg_sr_restore_x86_pv.c => saverestore/restore_x86_pv.c} (94%)
 rename tools/libs/{guest/xg_sr_save.c => saverestore/save.c} (88%)
 rename tools/libs/{guest/xg_save_restore.h => saverestore/save_restore.h} (98%)
 rename tools/libs/{guest/xg_sr_save_x86_hvm.c => saverestore/save_x86_hvm.c} (96%)
 rename tools/libs/{guest/xg_sr_save_x86_pv.c => saverestore/save_x86_pv.c} (97%)
 rename tools/libs/{guest/xg_sr_stream_format.h => saverestore/stream_format.h} (100%)
#1 lore raw

PATCH v20210701 1/40 hotplug/Linux: fix starting of xenstored with restarting systemd

A hard to trigger race with another unrelated systemd service and xenstored.service unveiled a bug in the way how xenstored is launched with systemd.

launch-xenstore may start either a daemon or a domain. In case a domain is used, systemd-notify was called. If another service triggered a restart of systemd while xenstored.service was executed, systemd may temporary lose track of services with Type=notify. As a result, xenstored.service would be marked as failed and units that depend on it will not be started. This breaks the enire Xen toolstack.

The chain of events is basically: xenstored.service sends the notification to systemd, this is a one-way event. Then systemd may be restarted by the other unit. During this time, xenstored.service is done and exits. Once systemd is done with its restart, it collects the pending notifications and childs. If it does not find the unit which sent the notification it will declare it as failed.

A workaround for this scenario is to leave the child processes running for a short time after sending the "READY=1" notification. If systemd happens to restart it will still find the unit it launched.

Adjust the callers of launch-xenstore to specifiy the init system:
Do not fork xenstored with systemd, preserve pid. This wil also avoid the need for a sleep because the process which sent the "READY=1" (the previously forked child) is still alive.

Remove the --pid-file in the systemd case because the pid of the child is known, and the file had probably little effect anyway due to lack of PidFile= and Type=forking in the unit file.

Be verbose about xenstored startup only with sysv to avoid interleaved output in systemd journal. Do the same also for domain case, even if is not strictly needed because init-xenstore-domain has no output.

The fix for upstream systemd which is supposed to fix it:
575b300b795b6 ("pid1: rework how we dispatch SIGCHLD and other signals")

--
v04:
- do mkdir unconditionally because init-xenstore-domain writes the domid to xenstored.pid
v03:
- remove run_xenstored function, follow style of shell built-in test function v02:
- preserve Type=notify

---
 tools/hotplug/Linux/init.d/xencommons.in      |  2 +-
 tools/hotplug/Linux/launch-xenstore.in        | 40 ++++++++++++++-----
 .../Linux/systemd/xenstored.service.in        |  2 +-
 3 files changed, 31 insertions(+), 13 deletions(-)
tools/hotplug/Linux/init.d/xencommons.in +1 -1
index 7fd6903b98..dcb0ce4b73 100644
@@ -60,7 +60,7 @@ do_start () {
 	mkdir -m700 -p ${XEN_LOCK_DIR}
 	mkdir -p ${XEN_LOG_DIR}
 
-	@XEN_SCRIPT_DIR@/launch-xenstore || exit 1
+	@XEN_SCRIPT_DIR@/launch-xenstore 'sysv' || exit 1
 
 	echo Setting domain 0 name, domid and JSON config...
 	${LIBEXEC_BIN}/xen-init-dom0 ${XEN_DOM0_UUID}
tools/hotplug/Linux/launch-xenstore.in +29 -11
index 019f9d6f4d..d40c66482a 100644
@@ -15,6 +15,17 @@
 # License along with this library; If not, see <http://www.gnu.org/licenses/>.
 #
 
+initd=$1
+
+case "$initd" in
+	sysv) nonl='-n' ;;
+	systemd) nonl= ;;
+	*)
+	echo "first argument must be 'sysv' or 'systemd'"
+	exit 1
+	;;
+esac
+
 XENSTORED=@XENSTORED@
 
 . @XEN_SCRIPT_DIR@/hotplugpath.sh
@@ -44,14 +55,16 @@ timeout_xenstore () {
 	return 0
 }
 
-test_xenstore && exit 0
+mkdir -p @XEN_RUN_DIR@
+
+if test "$initd" = 'sysv' ; then
+	test_xenstore && exit 0
+fi
 
 test -f @CONFIG_DIR@/@CONFIG_LEAF_DIR@/xencommons && . @CONFIG_DIR@/@CONFIG_LEAF_DIR@/xencommons
 
 [ "$XENSTORETYPE" = "" ] && XENSTORETYPE=daemon
 
-/bin/mkdir -p @XEN_RUN_DIR@
-
 [ "$XENSTORETYPE" = "daemon" ] && {
 	[ -z "$XENSTORED_TRACE" ] || XENSTORED_ARGS="$XENSTORED_ARGS -T @XEN_LOG_DIR@/xenstored-trace.log"
 	[ -z "$XENSTORED" ] && XENSTORED=@XENSTORED@
@@ -59,13 +72,15 @@ test -f @CONFIG_DIR@/@CONFIG_LEAF_DIR@/xencommons && . @CONFIG_DIR@/@CONFIG_LEAF
 		echo "No xenstored found"
 		exit 1
 	}
+	[ "$initd" = 'sysv' ] && {
+		echo $nonl Starting $XENSTORED...
+		$XENSTORED --pid-file @XEN_RUN_DIR@/xenstored.pid $XENSTORED_ARGS
+		timeout_xenstore $XENSTORED || exit 1
+		exit 0
+	}
 
-	echo -n Starting $XENSTORED...
-	$XENSTORED --pid-file @XEN_RUN_DIR@/xenstored.pid $XENSTORED_ARGS
-
-	systemd-notify --booted 2>/dev/null || timeout_xenstore $XENSTORED || exit 1
-
-	exit 0
+	exec $XENSTORED -N $XENSTORED_ARGS
+	exit 1
 }
 
 [ "$XENSTORETYPE" = "domain" ] && {
@@ -75,9 +90,12 @@ test -f @CONFIG_DIR@/@CONFIG_LEAF_DIR@/xencommons && . @CONFIG_DIR@/@CONFIG_LEAF
 	XENSTORE_DOMAIN_ARGS="$XENSTORE_DOMAIN_ARGS --memory $XENSTORE_DOMAIN_SIZE"
 	[ -z "$XENSTORE_MAX_DOMAIN_SIZE" ] || XENSTORE_DOMAIN_ARGS="$XENSTORE_DOMAIN_ARGS --maxmem $XENSTORE_MAX_DOMAIN_SIZE"
 
-	echo -n Starting $XENSTORE_DOMAIN_KERNEL...
+	echo $nonl Starting $XENSTORE_DOMAIN_KERNEL...
 	${LIBEXEC_BIN}/init-xenstore-domain $XENSTORE_DOMAIN_ARGS || exit 1
-	systemd-notify --ready 2>/dev/null
+	[ "$initd" = 'systemd' ] && {
+		systemd-notify --ready
+		sleep 9
+	}
 
 	exit 0
 }
tools/hotplug/Linux/systemd/xenstored.service.in +1 -1
index 80c1d408a5..c226eb3635 100644
@@ -11,7 +11,7 @@ Type=notify
 NotifyAccess=all
 RemainAfterExit=true
 ExecStartPre=/bin/grep -q control_d /proc/xen/capabilities
-ExecStart=@XEN_SCRIPT_DIR@/launch-xenstore
+ExecStart=@XEN_SCRIPT_DIR@/launch-xenstore 'systemd'
 
 [Install]
 WantedBy=multi-user.target
#2 lore raw

PATCH v20210701 2/40 tools: add API to work with sevaral bits at once

In reply to Olaf Hering

Introduce new API to test if a fixed number of bits is clear or set, and clear or set them all at once.

The caller has to make sure the input bitnumber is a multiple of BITS_PER_LONG.

This API avoids the loop over each bit in a known range just to see if all of them are either clear or set.

v02:
- change return type from int to bool (jgross)

---
 tools/libs/ctrl/xc_bitops.h | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
tools/libs/ctrl/xc_bitops.h +28 -0
index f0bac4a071..8e8c6efb45 100644
@@ -3,6 +3,7 @@
 
 /* bitmap operations for single threaded access */
 
+#include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -77,4 +78,31 @@ static inline void bitmap_or(void *_dst, const void *_other,
         dst[i] |= other[i];
 }
 
+static inline bool test_bit_long_set(unsigned long nr_base, const void *_addr)
+{
+    const unsigned long *addr = _addr;
+    unsigned long val = addr[nr_base / BITS_PER_LONG];
+
+    return val == ~0;
+}
+
+static inline bool test_bit_long_clear(unsigned long nr_base, const void *_addr)
+{
+    const unsigned long *addr = _addr;
+    unsigned long val = addr[nr_base / BITS_PER_LONG];
+
+    return val == 0;
+}
+
+static inline void clear_bit_long(unsigned long nr_base, void *_addr)
+{
+    unsigned long *addr = _addr;
+    addr[nr_base / BITS_PER_LONG] = 0;
+}
+
+static inline void set_bit_long(unsigned long nr_base, void *_addr)
+{
+    unsigned long *addr = _addr;
+    addr[nr_base / BITS_PER_LONG] = ~0;
+}
 #endif  /* XC_BITOPS_H */
#3 lore raw

PATCH v20210701 3/40 xl: fix description of migrate --debug

In reply to Olaf Hering

xl migrate --debug used to track every pfn in every batch of pages. But these times are gone. The code in xc_domain_save is the consumer of this knob, but it considers it only for the remus and colo case.

Adjust the help text to tell what --debug does today: Nothing.

v02:
- the option has no effect anymore

---
 docs/man/xl.1.pod.in   | 2 +-
 tools/xl/xl_cmdtable.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
docs/man/xl.1.pod.in +1 -1
index e2176bd696..70a6ebf438 100644
@@ -481,7 +481,7 @@ domain.
 
 =item B<--debug>
 
-Display huge (!) amount of debug information during the migration process.
+This option has no effect. It is preserved for compatibility reasons.
 
 =item B<-p>
 
tools/xl/xl_cmdtable.c +1 -1
index 661323d488..ca1dfa3525 100644
@@ -172,7 +172,7 @@ const struct cmd_spec cmd_table[] = {
       "                migrate-receive [-d -e]\n"
       "-e              Do not wait in the background (on <host>) for the death\n"
       "                of the domain.\n"
-      "--debug         Print huge (!) amount of debug during the migration process.\n"
+      "--debug         Ignored.\n"
       "-p              Do not unpause domain after migrating it.\n"
       "-D              Preserve the domain id"
     },
Acked-by on v20210701 3/40
#4 lore raw

Re: [PATCH v20210701 03/40] xl: fix description of migrate --debug

Quoted 11 lines · Olaf Hering

xl migrate --debug used to track every pfn in every batch of pages. But these times are gone. The code in xc_domain_save is the consumer of this knob, but it considers it only for the remus and colo case.

Adjust the help text to tell what --debug does today: Nothing.

v02:
- the option has no effect anymore

Thanks,

Signature · 1 line
Anthony PERARD
#5 lore raw

Re: [PATCH v20210701 03/40] xl: fix description of migrate --debug

In reply to Olaf Hering
Quoted 11 lines · Olaf Hering

xl migrate --debug used to track every pfn in every batch of pages. But these times are gone. The code in xc_domain_save is the consumer of this knob, but it considers it only for the remus and colo case.

Adjust the help text to tell what --debug does today: Nothing.

v02:
- the option has no effect anymore

Since when?  It was absolutely critical to debugging issues during the development of migration v2.

~Andrew

#6 lore raw

Re: [PATCH v20210701 03/40] xl: fix description of migrate --debug

Am Thu, 1 Jul 2021 15:33:47 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 2 lines

Since when?  It was absolutely critical to debugging issues during the development of migration v2.

Well, I can find out if needed.
What could would it enable today?
Last time I looked there was none.

Olaf

#7 lore raw

Re: [PATCH v20210701 03/40] xl: fix description of migrate --debug

Am Thu, 1 Jul 2021 16:40:55 +0200
schrieb Olaf Hering <[email protected]>:

Quoted 1 line

could

"code", sorry.

Olaf

#8 lore raw

Re: [PATCH v20210701 03/40] xl: fix description of migrate --debug

Quoted 5 lines · Olaf Hering

Am Thu, 1 Jul 2021 16:40:55 +0200
schrieb Olaf Hering <[email protected]>:

Quoted 1 line

could

"code", sorry.

c/s 7449fb36c6c81d2ba10a40b59e61a9f420cd8450 was buggy and inverted the condition while making the code transformation.

--debug makes no sense at all in a checkpointed stream.

~Andrew

#9 lore raw

Re: [PATCH v20210701 03/40] xl: fix description of migrate --debug

Am Thu, 1 Jul 2021 15:49:08 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 1 line

--debug makes no sense at all in a checkpointed stream.

This should probably have become "ctx->save.checkpointed == MIG_STREAM_NONE".

But this still leaves the question what value this code branch has when verify_frames does not, and most likely can not, work. I think fixing it requires to expose details like which pages are grant pages.

Olaf

#10 lore raw

PATCH v20210701 4/40 tools: use integer division in convert-legacy-stream

In reply to Olaf Hering

A single slash gives a float, a double slash gives an int.

    bitmap = unpack_exact("Q" * ((max_id/64) + 1))

TypeError: can't multiply sequence by non-int of type 'float'

v02:
- import division to remain compatible with python2.7 (andrew)
- white space in max_id chunk (andrew)

---
 tools/python/scripts/convert-legacy-stream | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
tools/python/scripts/convert-legacy-stream +5 -4
index ca93a93848..66ee3d2f5d 100755
@@ -6,6 +6,7 @@ Convert a legacy migration stream to a v2 stream.
 """
 
 from __future__ import print_function
+from __future__ import division
 
 import sys
 import os, os.path
@@ -163,7 +164,7 @@ def write_libxc_hvm_params(params):
         raise RuntimeError("Expected even length list of hvm parameters")
 
     write_record(libxc.REC_TYPE_hvm_params,
-                 pack(libxc.HVM_PARAMS_FORMAT, len(params) / 2, 0),
+                 pack(libxc.HVM_PARAMS_FORMAT, len(params) // 2, 0),
                  pack("Q" * len(params), *params))
 
 def write_libxc_static_data_end():
@@ -264,8 +265,8 @@ def read_pv_extended_info(vm):
                           (so_far - total_length, ))
 
 def read_pv_p2m_frames(vm):
-    fpp = 4096 / vm.width
-    p2m_frame_len = (vm.p2m_size - 1) / fpp + 1
+    fpp = 4096 // vm.width
+    p2m_frame_len = (vm.p2m_size - 1) // fpp + 1
 
     info("P2M frames: fpp %d, p2m_frame_len %d" % (fpp, p2m_frame_len))
     write_libxc_pv_p2m_frames(vm, unpack_ulongs(p2m_frame_len))
@@ -405,7 +406,7 @@ def read_chunks(vm):
                                   (max_id, legacy.MAX_VCPU_ID))
 
             vm.max_vcpu_id = max_id
-            bitmap = unpack_exact("Q" * ((max_id/64) + 1))
+            bitmap = unpack_exact("Q" * ((max_id // 64) + 1))
 
             for idx, word in enumerate(bitmap):
                 bit_idx = 0
Reviewed-by on v20210701 4/40
#11 lore raw

Re: [PATCH v20210701 04/40] tools: use integer division in convert-legacy-stream

Quoted 6 lines · Olaf Hering

A single slash gives a float, a double slash gives an int.

    bitmap = unpack_exact("Q" * ((max_id/64) + 1))

TypeError: can't multiply sequence by non-int of type 'float'

#12 lore raw

PATCH v20210701 5/40 tools: handle libxl__physmap_info.name properly in convert-legacy-stream

In reply to Olaf Hering

The trailing member name[] in libxl__physmap_info is written as a cstring into the stream. The current code does a sanity check if the last byte is zero. This attempt fails with python3.4 because name[-1] returns a type int. As a result the comparison with byte(\00) fails:

File "/usr/lib/xen/bin/convert-legacy-stream", line 347, in read_libxl_toolstack
raise StreamError("physmap name not NUL terminated")
StreamError: physmap name not NUL terminated

To handle both python variants the cstring is unpacked into the actual string and the trailing nil.

---
 tools/python/scripts/convert-legacy-stream | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
tools/python/scripts/convert-legacy-stream +5 -4
index 66ee3d2f5d..9003ac4f6d 100755
@@ -336,20 +336,21 @@ def read_libxl_toolstack(vm, data):
         if len(data) < namelen:
             raise StreamError("Remaining data too short for physmap name")
 
-        name = data[:namelen]
+        c_string = data[:namelen]
         data = data[namelen:]
 
         # Strip padding off the end of name
         if twidth == 64:
-            name = name[:-4]
+            c_string = c_string[:-4]
 
-        if name[-1] != b'\x00':
+        name, nil = unpack("={0}sB".format(len(c_string) - 1), c_string)
+        if nil != 0:
             raise StreamError("physmap name not NUL terminated")
 
         root = b"physmap/%x" % (phys, )
         kv = [root + b"/start_addr", b"%x" % (start, ),
               root + b"/size",       b"%x" % (size, ),
-              root + b"/name",       name[:-1]]
+              root + b"/name",       name]
 
         for key, val in zip(kv[0::2], kv[1::2]):
             info("    '%s' = '%s'" % (key.decode(), val.decode()))
#13 lore raw

Re: [PATCH v20210701 05/40] tools: handle libxl__physmap_info.name properly in convert-legacy-stream

Quoted 9 lines · Olaf Hering
tools/python/scripts/convert-legacy-stream +0 -1
index 66ee3d2f5d..9003ac4f6d 100755
@@ -336,20 +336,21 @@ def read_libxl_toolstack(vm, data):
         if len(data) < namelen:
             raise StreamError("Remaining data too short for physmap name")
 
-        name = data[:namelen]

tools/python/scripts/convert-legacy-stream def read_libxl_toolstack(vm, data): · line 346

Quoted 10 lines
tools/python/scripts/convert-legacy-stream +3 -2
+        c_string = data[:namelen]
         data = data[namelen:]
 
         # Strip padding off the end of name
         if twidth == 64:
-            name = name[:-4]
+            c_string = c_string[:-4]
 
-        if name[-1] != b'\x00':
+        name, nil = unpack("={0}sB".format(len(c_string) - 1), c_string)

This is rather invasive.  How about simply:

(quoted diff) +0 -0

b/tools/python/scripts/convert-legacy-stream
index ca93a93848ec..d4ae94c02f21 100755

tools/python/scripts/convert-legacy-stream +0 -0
@@ -342,7 +342,7 @@ def read_libxl_toolstack(vm, data):

         if twidth == 64:
             name = name[:-4]

-        if name[-1] != b'\x00':
+        if bytearray(name)[-1] != 0:
             raise StreamError("physmap name not NUL terminated")
         root = b"physmap/%x" % (phys, )

which is rather more contained, and looks to work from Py2.6 and later?

~Andrew

#14 lore raw

PATCH v20210701 6/40 tools: fix Python3.4 TypeError in format string

In reply to Olaf Hering

Using the first element of a tuple for a format specifier fails with python3.4 as included in SLE12:

    b = b"string/%x" % (i, )

TypeError: unsupported operand type(s) for %: 'bytes' and 'tuple'

It happens to work with python 2.7 and 3.6. Use a syntax that is handled by all three variants.

---
 tools/python/scripts/convert-legacy-stream | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
tools/python/scripts/convert-legacy-stream +3 -3
index 9003ac4f6d..235b922ff5 100755
@@ -347,9 +347,9 @@ def read_libxl_toolstack(vm, data):
         if nil != 0:
             raise StreamError("physmap name not NUL terminated")
 
-        root = b"physmap/%x" % (phys, )
-        kv = [root + b"/start_addr", b"%x" % (start, ),
-              root + b"/size",       b"%x" % (size, ),
+        root = bytes(("physmap/%x" % phys).encode('utf-8'))
+        kv = [root + b"/start_addr", bytes(("%x" % start).encode('utf-8')),
+              root + b"/size",       bytes(("%x" % size).encode('utf-8')),
               root + b"/name",       name]
 
         for key, val in zip(kv[0::2], kv[1::2]):
#15 lore raw

Re: [PATCH v20210701 06/40] tools: fix Python3.4 TypeError in format string

Quoted 18 lines · Olaf Hering

Using the first element of a tuple for a format specifier fails with python3.4 as included in SLE12:

    b = b"string/%x" % (i, )

TypeError: unsupported operand type(s) for %: 'bytes' and 'tuple'

It happens to work with python 2.7 and 3.6. Use a syntax that is handled by all three variants.

---
 tools/python/scripts/convert-legacy-stream | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
tools/python/scripts/convert-legacy-stream +0 -0
index 9003ac4f6d..235b922ff5 100755

tools/python/scripts/convert-legacy-stream def read_libxl_toolstack(vm, data): · line 352

Quoted 9 lines
tools/python/scripts/convert-legacy-stream +3 -3
@@ -347,9 +347,9 @@ def read_libxl_toolstack(vm, data):
         if nil != 0:
             raise StreamError("physmap name not NUL terminated")
 
-        root = b"physmap/%x" % (phys, )
-        kv = [root + b"/start_addr", b"%x" % (start, ),
-              root + b"/size",       b"%x" % (size, ),
+        root = bytes(("physmap/%x" % phys).encode('utf-8'))
+        kv = [root + b"/start_addr", bytes(("%x" % start).encode('utf-8')),
+              root + b"/size",       bytes(("%x" % size).encode('utf-8')),

Why bytes()? Encode does already return bytes type.

Quoted 3 lines
tools/python/scripts/convert-legacy-stream +0 -0
               root + b"/name",       name]
 
         for key, val in zip(kv[0::2], kv[1::2]):
Signature · 3 lines
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab
#16 lore raw

Re: [PATCH v20210701 06/40] tools: fix Python3.4 TypeError in format string

Quoted 22 lines · Marek Marczykowski-Górecki
Quoted 21 lines · Olaf Hering

Using the first element of a tuple for a format specifier fails with python3.4 as included in SLE12:

    b = b"string/%x" % (i, )

TypeError: unsupported operand type(s) for %: 'bytes' and 'tuple'

It happens to work with python 2.7 and 3.6. Use a syntax that is handled by all three variants.

---
 tools/python/scripts/convert-legacy-stream | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
tools/python/scripts/convert-legacy-stream +0 -0
index 9003ac4f6d..235b922ff5 100755
@@ -347,9 +347,9 @@ def read_libxl_toolstack(vm, data):
         if nil != 0:
             raise StreamError("physmap name not NUL terminated")
 
Quoted 7 lines

tools/python/scripts/convert-legacy-stream def read_libxl_toolstack(vm, data): · line 352

Quoted 6 lines
tools/python/scripts/convert-legacy-stream +3 -3
-        root = b"physmap/%x" % (phys, )
-        kv = [root + b"/start_addr", b"%x" % (start, ),
-              root + b"/size",       b"%x" % (size, ),
+        root = bytes(("physmap/%x" % phys).encode('utf-8'))
+        kv = [root + b"/start_addr", bytes(("%x" % start).encode('utf-8')),
+              root + b"/size",       bytes(("%x" % size).encode('utf-8')),

Why bytes()? Encode does already return bytes type.

Yes - I've just tried this out on various version of python (including
https://www.onlinegdb.com/online_python_interpreter which is the only place I can find Python 3.4 easily available)

.encode() does return bytes (Py3) or str (Py2) so doesn't need the surrounding bytes().

However, the % (phys, ) with the trailing comma is deliberate to work around a common python error, so wants to remain if you're keeping the %-formatting.

~Andrew

#17 lore raw

Re: [PATCH v20210701 06/40] tools: fix Python3.4 TypeError in format string

Am Fri, 2 Jul 2021 17:39:54 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 3 lines

However, the % (phys, ) with the trailing comma is deliberate to work around a common python error, so wants to remain if you're keeping the %-formatting.

What error is that?

Olaf

#18 lore raw

Re: [PATCH v20210701 06/40] tools: fix Python3.4 TypeError in format string

Quoted 9 lines · Olaf Hering

Am Fri, 2 Jul 2021 17:39:54 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 3 lines

However, the % (phys, ) with the trailing comma is deliberate to work around a common python error, so wants to remain if you're keeping the %-formatting.

What error is that?

Quoted 1 line
Quoted 1 line

def p1(arg):

...     print("%s" % arg)

Quoted 1 line
Quoted 1 line
Quoted 1 line

def p2(arg):

...     print("%s" % (arg, ))

Quoted 1 line
Quoted 1 line
Quoted 1 line

p1("foo")

foo

Quoted 1 line
Quoted 1 line
Quoted 1 line

p2("foo")

foo

Quoted 1 line
Quoted 1 line
Quoted 1 line

p1(("foo", "bar"))

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "<stdin>", line 2, in p1
TypeError: not all arguments converted during string formatting

Quoted 1 line
Quoted 1 line
Quoted 1 line

p2(("foo", "bar"))

('foo', 'bar')

The % operator has some type ambiguity with how it works.  (foo, ) forces arg to be a 1-tuple as far as formatting is concerned.

~Andrew

#19 lore raw

Re: [PATCH v20210701 06/40] tools: fix Python3.4 TypeError in format string

In reply to Marek Marczykowski-Górecki

Am Fri, 2 Jul 2021 18:19:39 +0200
schrieb Marek Marczykowski-Górecki <[email protected]>:

Quoted 1 line

Why bytes()? Encode does already return bytes type.

You are right, this works as well:
i = 123
b = ("str/%x" % (i, )).encode('utf-8')

Any preference regarding the "encoding"? I picked UTF8, but 'ascii' might be more correct in this context. In practice it may not matter.

Olaf

#20 lore raw

Re: [PATCH v20210701 06/40] tools: fix Python3.4 TypeError in format string

Quoted 9 lines · Olaf Hering

Am Fri, 2 Jul 2021 18:19:39 +0200
schrieb Marek Marczykowski-Górecki <[email protected]>:

Quoted 1 line

Why bytes()? Encode does already return bytes type.

You are right, this works as well:
i = 123
b = ("str/%x" % (i, )).encode('utf-8')

Any preference regarding the "encoding"? I picked UTF8, but 'ascii' might be more correct in this context. In practice it may not matter.

I suspect you're right and it doesn't matter, but ascii feels like a safer option.

~Andrew

#21 lore raw

PATCH v20210701 7/40 tools: create libxensaverestore

In reply to Olaf Hering

Move all save/restore related code from libxenguest.so into a separate library libxensaverestore.so. The only consumer is libxl-save-helper. There is no need to have the moved code mapped all the time in binaries where libxenguest.so is used.

According to size(1) the change is:
text data bss dec hex filename
187183 4304 48 191535 2ec2f guest/libxenguest.so.4.15.0

 124106	   3376	     48	 127530	  1f22a	guest/libxenguest.so.4.15.0
  67841	   1872	      8	  69721	  11059	saverestore/libxensaverestore.so.4.15.0

While touching the files anyway, take the opportunity to drop the redundant xg_sr_ filename prefix.

v6:
- fix build of nomigrate.c
v5:
- fix spelling in description
v4:
- drop xg_ prefix from filenames (jgross)
- drop sr_ prefix from filenames (jbeulich)
v3:
- repost in time for 4.16
v2:
- copy also license header
- move xg_nomigrate.c
- add size(1) output to commit msg
- remove change from libxl_create.c

---
 .gitignore                                    |   2 +
 tools/include/xenguest.h                      | 186 ----------------
 tools/include/xensaverestore.h                | 208 ++++++++++++++++++
 tools/libs/Makefile                           |   1 +
 tools/libs/guest/Makefile                     |  11 -
 tools/libs/guest/xg_offline_page.c            |   1 -
 tools/libs/light/Makefile                     |   4 +-
 tools/libs/light/libxl_internal.h             |   1 +
 tools/libs/light/libxl_save_helper.c          |   1 +
 tools/libs/light/libxl_save_msgs_gen.pl       |   2 +-
 tools/libs/saverestore/Makefile               |  38 ++++
 .../xg_sr_common.c => saverestore/common.c}   |   2 +-
 .../xg_sr_common.h => saverestore/common.h}   |  16 +-
 .../common_x86.c}                             |   2 +-
 .../common_x86.h}                             |   2 +-
 .../common_x86_pv.c}                          |   2 +-
 .../common_x86_pv.h}                          |   2 +-
 .../nomigrate.c}                              |   2 +-
 .../xg_sr_restore.c => saverestore/restore.c} |   2 +-
 .../restore_x86_hvm.c}                        |   2 +-
 .../restore_x86_pv.c}                         |   2 +-
 .../xg_sr_save.c => saverestore/save.c}       |   2 +-
 .../save_restore.h}                           |   2 -
 .../save_x86_hvm.c}                           |   2 +-
 .../save_x86_pv.c}                            |   2 +-
 .../stream_format.h}                          |   0
 tools/libs/uselibs.mk                         |   4 +-
 27 files changed, 283 insertions(+), 218 deletions(-)
 create mode 100644 tools/include/xensaverestore.h
 create mode 100644 tools/libs/saverestore/Makefile
 rename tools/libs/{guest/xg_sr_common.c => saverestore/common.c} (99%)
 rename tools/libs/{guest/xg_sr_common.h => saverestore/common.h} (98%)
 rename tools/libs/{guest/xg_sr_common_x86.c => saverestore/common_x86.c} (99%)
 rename tools/libs/{guest/xg_sr_common_x86.h => saverestore/common_x86.h} (98%)
 rename tools/libs/{guest/xg_sr_common_x86_pv.c => saverestore/common_x86_pv.c} (99%)
 rename tools/libs/{guest/xg_sr_common_x86_pv.h => saverestore/common_x86_pv.h} (98%)
 rename tools/libs/{guest/xg_nomigrate.c => saverestore/nomigrate.c} (98%)
 rename tools/libs/{guest/xg_sr_restore.c => saverestore/restore.c} (99%)
 rename tools/libs/{guest/xg_sr_restore_x86_hvm.c => saverestore/restore_x86_hvm.c} (99%)
 rename tools/libs/{guest/xg_sr_restore_x86_pv.c => saverestore/restore_x86_pv.c} (99%)
 rename tools/libs/{guest/xg_sr_save.c => saverestore/save.c} (99%)
 rename tools/libs/{guest/xg_save_restore.h => saverestore/save_restore.h} (98%)
 rename tools/libs/{guest/xg_sr_save_x86_hvm.c => saverestore/save_x86_hvm.c} (99%)
 rename tools/libs/{guest/xg_sr_save_x86_pv.c => saverestore/save_x86_pv.c} (99%)
 rename tools/libs/{guest/xg_sr_stream_format.h => saverestore/stream_format.h} (100%)
.gitignore +2 -0
index 38a085e398..08a321e995 100644
@@ -147,6 +147,8 @@ tools/libs/light/test_timedereg
 tools/libs/light/test_fdderegrace
 tools/libs/light/tmp.*
 tools/libs/light/xenlight.pc
+tools/libs/saverestore/libxensaverestore.map
+tools/libs/saverestore/xensaverestore.pc
 tools/libs/stat/_paths.h
 tools/libs/stat/headers.chk
 tools/libs/stat/libxenstat.map
tools/include/xenguest.h +0 -186

198 lines not on this page · raw message

tools/include/xensaverestore.h new file +208 -0

208 lines not on this page · raw message

tools/libs/Makefile +1 -0
index 1afcd12e2b..ca43c66777 100644
@@ -12,6 +12,7 @@ SUBDIRS-y += devicemodel
 SUBDIRS-y += ctrl
 SUBDIRS-y += guest
 SUBDIRS-y += hypfs
+SUBDIRS-y += saverestore
 SUBDIRS-y += store
 SUBDIRS-y += stat
 SUBDIRS-$(CONFIG_Linux) += vchan
tools/libs/guest/Makefile +0 -11
index 2ce92d247e..4cf5459bb1 100644
@@ -11,18 +11,7 @@ SRCS-y += xg_domain.c
 SRCS-y += xg_suspend.c
 SRCS-y += xg_resume.c
 ifeq ($(CONFIG_MIGRATE),y)
-SRCS-y += xg_sr_common.c
-SRCS-$(CONFIG_X86) += xg_sr_common_x86.c
-SRCS-$(CONFIG_X86) += xg_sr_common_x86_pv.c
-SRCS-$(CONFIG_X86) += xg_sr_restore_x86_pv.c
-SRCS-$(CONFIG_X86) += xg_sr_restore_x86_hvm.c
-SRCS-$(CONFIG_X86) += xg_sr_save_x86_pv.c
-SRCS-$(CONFIG_X86) += xg_sr_save_x86_hvm.c
-SRCS-y += xg_sr_restore.c
-SRCS-y += xg_sr_save.c
 SRCS-y += xg_offline_page.c
-else
-SRCS-y += xg_nomigrate.c
 endif
 SRCS-y       += xg_core.c
 SRCS-$(CONFIG_X86) += xg_core_x86.c
tools/libs/guest/xg_offline_page.c +0 -1
index cfe0e2d537..92b65243b1 100644
@@ -29,7 +29,6 @@
 
 #include "xc_private.h"
 #include "xg_private.h"
-#include "xg_save_restore.h"
 
 struct pte_backup_entry
 {
tools/libs/light/Makefile +2 -2
index 7d8c51d492..68e51dd13c 100644
@@ -179,7 +179,7 @@ $(ACPI_OBJS) $(ACPI_PIC_OBJS): CFLAGS += -I. -DLIBACPI_STDUTILS=\"$(CURDIR)/libx
 $(TEST_PROG_OBJS) _libxl.api-for-check: CFLAGS += $(CFLAGS_libxentoollog) $(CFLAGS_libxentoolcore)
 libxl_dom.o libxl_dom.opic: CFLAGS += -I$(XEN_ROOT)/tools  # include libacpi/x86.h
 libxl_x86_acpi.o libxl_x86_acpi.opic: CFLAGS += -I$(XEN_ROOT)/tools
-$(SAVE_HELPER_OBJS): CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenevtchn) $(CFLAGS_libxenguest)
+$(SAVE_HELPER_OBJS): CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenevtchn) $(CFLAGS_libxensaverestore)
 
 testidl.o: CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenlight)
 testidl.c: libxl_types.idl gentest.py $(XEN_INCLUDE)/libxl.h $(AUTOINCS)
@@ -241,7 +241,7 @@ test_%: test_%.o test_common.o libxenlight_test.so
 	$(CC) $(LDFLAGS) -o $@ $^ $(filter-out %libxenlight.so, $(LDLIBS_libxenlight)) $(LDLIBS_libxentoollog) $(LDLIBS_libxentoolcore) -lyajl $(APPEND_LDFLAGS)
 
 libxl-save-helper: $(SAVE_HELPER_OBJS) libxenlight.so
-	$(CC) $(LDFLAGS) -o $@ $(SAVE_HELPER_OBJS) $(LDLIBS_libxentoollog) $(LDLIBS_libxenctrl) $(LDLIBS_libxenguest) $(LDLIBS_libxentoolcore) $(APPEND_LDFLAGS)
+	$(CC) $(LDFLAGS) -o $@ $(SAVE_HELPER_OBJS) $(LDLIBS_libxentoollog) $(LDLIBS_libxenctrl) $(LDLIBS_libxensaverestore) $(LDLIBS_libxentoolcore) $(APPEND_LDFLAGS)
 
 testidl: testidl.o libxenlight.so
 	$(CC) $(LDFLAGS) -o $@ testidl.o $(LDLIBS_libxenlight) $(LDLIBS_libxentoollog) $(LDLIBS_libxentoolcore) $(APPEND_LDFLAGS)
tools/libs/light/libxl_internal.h +1 -0
index 0b4671318c..439c654733 100644
@@ -56,6 +56,7 @@
 #define XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xenguest.h>
+#include <xensaverestore.h>
 #include <xenhypfs.h>
 
 #include <xen-tools/libs.h>
tools/libs/light/libxl_save_helper.c +1 -0
index 65dff389bf..896e845a2f 100644
@@ -48,6 +48,7 @@
 
 #include "xenctrl.h"
 #include "xenguest.h"
+#include "xensaverestore.h"
 #include "_libxl_save_msgs_helper.h"
 
 /*----- logger -----*/
tools/libs/light/libxl_save_msgs_gen.pl +1 -1
index 9d425b1dee..f263ee01bb 100755
@@ -72,7 +72,7 @@ END_BOTH
 END_CALLOUT
 
 #include <xenctrl.h>
-#include <xenguest.h>
+#include <xensaverestore.h>
 #include "_libxl_save_msgs_${ah}.h"
 
 END_HELPER
tools/libs/saverestore/Makefile new file +38 -0
new file mode 100644
index 0000000000..48728b3be2
@@ -0,0 +1,38 @@
+XEN_ROOT = $(CURDIR)/../../..
+include $(XEN_ROOT)/tools/Rules.mk
+
+ifeq ($(CONFIG_MIGRATE),y)
+SRCS-y += common.c
+SRCS-$(CONFIG_X86) += common_x86.c
+SRCS-$(CONFIG_X86) += common_x86_pv.c
+SRCS-$(CONFIG_X86) += restore_x86_pv.c
+SRCS-$(CONFIG_X86) += restore_x86_hvm.c
+SRCS-$(CONFIG_X86) += save_x86_pv.c
+SRCS-$(CONFIG_X86) += save_x86_hvm.c
+SRCS-y += restore.c
+SRCS-y += save.c
+else
+SRCS-y += nomigrate.c
+endif
+
+CFLAGS += -I$(XEN_libxenctrl)
+CFLAGS += -I$(XEN_libxenguest)
+
+-include $(XEN_TARGET_ARCH)/Makefile
+
+CFLAGS   += -Werror -Wmissing-prototypes
+CFLAGS   += -I. -I./include $(CFLAGS_xeninclude)
+CFLAGS   += -D__XEN_TOOLS__
+CFLAGS   += -include $(XEN_ROOT)/tools/config.h
+# Needed for asprintf()
+CFLAGS-$(CONFIG_Linux) += -D_GNU_SOURCE
+
+LIBHEADER := xensaverestore.h
+
+NO_HEADERS_CHK := y
+
+include $(XEN_ROOT)/tools/libs/libs.mk
+
+.PHONY: cleanlocal
+cleanlocal:
+	rm -f libxensaverestore.map
tools/libs/saverestore/common.c renamed from tools/libs/guest/xg_sr_common.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_common.c
rename to tools/libs/saverestore/common.c
index 17567ab133..77128bc747 100644
@@ -1,6 +1,6 @@
 #include <assert.h>
 
-#include "xg_sr_common.h"
+#include "common.h"
 
 #include <xen-tools/libs.h>
 
tools/libs/saverestore/common.h renamed from tools/libs/guest/xg_sr_common.h +14 -2
similarity index 98%
rename from tools/libs/guest/xg_sr_common.h
rename to tools/libs/saverestore/common.h
index e2994e18ac..ca2eb47a4f 100644
@@ -1,13 +1,25 @@
 #ifndef __COMMON__H
 #define __COMMON__H
 
+#include <unistd.h>
+#include <errno.h>
 #include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "xc_private.h"
+#include "xenguest.h"
+#include "xensaverestore.h"
 
 #include "xg_private.h"
-#include "xg_save_restore.h"
+#include "save_restore.h"
 #include "xc_bitops.h"
 
-#include "xg_sr_stream_format.h"
+#include "stream_format.h"
 
 /* String representation of Domain Header types. */
 const char *dhdr_type_to_str(uint32_t type);
tools/libs/saverestore/common_x86.c renamed from tools/libs/guest/xg_sr_common_x86.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_common_x86.c
rename to tools/libs/saverestore/common_x86.c
index 563b4f0168..f1beb234ae 100644
@@ -1,4 +1,4 @@
-#include "xg_sr_common_x86.h"
+#include "common_x86.h"
 
 int write_x86_tsc_info(struct xc_sr_context *ctx)
 {
tools/libs/saverestore/common_x86.h renamed from tools/libs/guest/xg_sr_common_x86.h +1 -1
similarity index 98%
rename from tools/libs/guest/xg_sr_common_x86.h
rename to tools/libs/saverestore/common_x86.h
index b55758c96d..3a2d91dcb8 100644
@@ -1,7 +1,7 @@
 #ifndef __COMMON_X86__H
 #define __COMMON_X86__H
 
-#include "xg_sr_common.h"
+#include "common.h"
 
 /*
  * Obtains a domains TSC information from Xen and writes a X86_TSC_INFO record
tools/libs/saverestore/common_x86_pv.c renamed from tools/libs/guest/xg_sr_common_x86_pv.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_common_x86_pv.c
rename to tools/libs/saverestore/common_x86_pv.c
index c0acf00f90..cfe1b24bed 100644
@@ -1,6 +1,6 @@
 #include <assert.h>
 
-#include "xg_sr_common_x86_pv.h"
+#include "common_x86_pv.h"
 
 xen_pfn_t mfn_to_pfn(struct xc_sr_context *ctx, xen_pfn_t mfn)
 {
tools/libs/saverestore/common_x86_pv.h renamed from tools/libs/guest/xg_sr_common_x86_pv.h +1 -1
similarity index 98%
rename from tools/libs/guest/xg_sr_common_x86_pv.h
rename to tools/libs/saverestore/common_x86_pv.h
index 953b5bfb8d..a9f8c970e3 100644
@@ -1,7 +1,7 @@
 #ifndef __COMMON_X86_PV_H
 #define __COMMON_X86_PV_H
 
-#include "xg_sr_common_x86.h"
+#include "common_x86.h"
 
 /* Virtual address ranges reserved for hypervisor. */
 #define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
tools/libs/saverestore/nomigrate.c renamed from tools/libs/guest/xg_nomigrate.c +1 -1
similarity index 98%
rename from tools/libs/guest/xg_nomigrate.c
rename to tools/libs/saverestore/nomigrate.c
index 6795c62ddc..67e58d353a 100644
@@ -18,7 +18,7 @@
 #include <inttypes.h>
 #include <errno.h>
 #include <xenctrl.h>
-#include <xenguest.h>
+#include <xensaverestore.h>
 
 int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t flags,
                    struct save_callbacks *callbacks,
tools/libs/saverestore/restore.c renamed from tools/libs/guest/xg_sr_restore.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_restore.c
rename to tools/libs/saverestore/restore.c
index b57a787519..be259a1c6b 100644
@@ -2,7 +2,7 @@
 
 #include <assert.h>
 
-#include "xg_sr_common.h"
+#include "common.h"
 
 /*
  * Read and validate the Image and Domain headers.
tools/libs/saverestore/restore_x86_hvm.c renamed from tools/libs/guest/xg_sr_restore_x86_hvm.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_restore_x86_hvm.c
rename to tools/libs/saverestore/restore_x86_hvm.c
index d6ea6f3012..bd63bd2818 100644
@@ -1,7 +1,7 @@
 #include <assert.h>
 #include <arpa/inet.h>
 
-#include "xg_sr_common_x86.h"
+#include "common_x86.h"
 
 /*
  * Process an HVM_CONTEXT record from the stream.
tools/libs/saverestore/restore_x86_pv.c renamed from tools/libs/guest/xg_sr_restore_x86_pv.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_restore_x86_pv.c
rename to tools/libs/saverestore/restore_x86_pv.c
index dc50b0f5a8..96608e5231 100644
@@ -1,6 +1,6 @@
 #include <assert.h>
 
-#include "xg_sr_common_x86_pv.h"
+#include "common_x86_pv.h"
 
 static xen_pfn_t pfn_to_mfn(const struct xc_sr_context *ctx, xen_pfn_t pfn)
 {
tools/libs/saverestore/save.c renamed from tools/libs/guest/xg_sr_save.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_save.c
rename to tools/libs/saverestore/save.c
index 2ba7c3200c..ae3e8797d0 100644
@@ -1,7 +1,7 @@
 #include <assert.h>
 #include <arpa/inet.h>
 
-#include "xg_sr_common.h"
+#include "common.h"
 
 /*
  * Writes an Image header and Domain header into the stream.
tools/libs/saverestore/save_restore.h renamed from tools/libs/guest/xg_save_restore.h +0 -2
similarity index 98%
rename from tools/libs/guest/xg_save_restore.h
rename to tools/libs/saverestore/save_restore.h
index 3dbbc8dcd2..20bd3d30a5 100644
@@ -15,8 +15,6 @@
  * License along with this library; If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "xc_private.h"
-
 #include <xen/foreign/x86_32.h>
 #include <xen/foreign/x86_64.h>
 
tools/libs/saverestore/save_x86_hvm.c renamed from tools/libs/guest/xg_sr_save_x86_hvm.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_save_x86_hvm.c
rename to tools/libs/saverestore/save_x86_hvm.c
index 1634a7bc43..91c2cb99ab 100644
@@ -1,6 +1,6 @@
 #include <assert.h>
 
-#include "xg_sr_common_x86.h"
+#include "common_x86.h"
 
 #include <xen/hvm/params.h>
 
tools/libs/saverestore/save_x86_pv.c renamed from tools/libs/guest/xg_sr_save_x86_pv.c +1 -1
similarity index 99%
rename from tools/libs/guest/xg_sr_save_x86_pv.c
rename to tools/libs/saverestore/save_x86_pv.c
index 4964f1f7b8..92f77fad0f 100644
@@ -1,7 +1,7 @@
 #include <assert.h>
 #include <limits.h>
 
-#include "xg_sr_common_x86_pv.h"
+#include "common_x86_pv.h"
 
 /* Check a 64 bit virtual address for being canonical. */
 static inline bool is_canonical_address(xen_vaddr_t vaddr)
tools/libs/saverestore/stream_format.h renamed from tools/libs/guest/xg_sr_stream_format.h +0 -0
similarity index 100%
rename from tools/libs/guest/xg_sr_stream_format.h
rename to tools/libs/saverestore/stream_format.h
tools/libs/uselibs.mk +3 -1
index efd7a475ba..62a2990b95 100644
@@ -20,6 +20,8 @@ LIBS_LIBS += ctrl
 USELIBS_ctrl := toollog call evtchn gnttab foreignmemory devicemodel
 LIBS_LIBS += guest
 USELIBS_guest := evtchn ctrl
+LIBS_LIBS += saverestore
+USELIBS_saverestore := guest ctrl
 LIBS_LIBS += store
 USELIBS_store := toolcore
 LIBS_LIBS += vchan
@@ -27,7 +29,7 @@ USELIBS_vchan := toollog store gnttab evtchn
 LIBS_LIBS += stat
 USELIBS_stat := ctrl store
 LIBS_LIBS += light
-USELIBS_light := toollog evtchn toolcore ctrl store hypfs guest
+USELIBS_light := toollog evtchn toolcore ctrl store hypfs guest saverestore
 LIBS_LIBS += util
 USELIBS_util := light
 FILENAME_util := xlutil
#22 lore raw

Re: [PATCH v20210701 07/40] tools: create libxensaverestore

Am Thu, 1 Jul 2021 11:56:02 +0200
schrieb Olaf Hering <[email protected]>:

Quoted 1 line

Move all save/restore related code

This is now 6 months old.

What is blocking approval?

Olaf

#23 lore raw

Re: [PATCH v20210701 07/40] tools: create libxensaverestore

Hi Olaf,

Quoted 8 lines · Olaf Hering

Am Thu, 1 Jul 2021 11:56:02 +0200
schrieb Olaf Hering <[email protected]>:

Quoted 1 line

Move all save/restore related code

This is now 6 months old.

What is blocking approval?

There is already an ack from Wei which should be sufficient for the tools part.

But looking at the history of the patch, there seem to have been concern from Andrew. Were they resolved?

Cheers,

Signature · 1 line
Julien Grall
#24 lore raw

Re: [PATCH v20210701 07/40] tools: create libxensaverestore

Am Fri, 9 Jul 2021 10:31:53 +0100
schrieb Julien Grall <[email protected]>:

Quoted 1 line

Were they resolved?

I think so, yes.

Olaf

#25 lore raw

Re: [PATCH v20210701 07/40] tools: create libxensaverestore

In reply to Olaf Hering

Hi Olaf,

Quoted 9 lines · Olaf Hering

Move all save/restore related code from libxenguest.so into a separate library libxensaverestore.so. The only consumer is libxl-save-helper. There is no need to have the moved code mapped all the time in binaries where libxenguest.so is used.

According to size(1) the change is:
text data bss dec hex filename
187183 4304 48 191535 2ec2f guest/libxenguest.so.4.15.0

Quoted 8 lines
  124106	   3376	     48	 127530	  1f22a	guest/libxenguest.so.4.15.0
   67841	   1872	      8	  69721	  11059	saverestore/libxensaverestore.so.4.15.0

While touching the files anyway, take the opportunity to drop the redundant xg_sr_ filename prefix.

The changelog is not very useful to keep in the commit message. We usally add --- on its own line before so they will get stripped when the patch is applied.

This comment applies for the full series. But I can deal with this patch if I happen to commit it.

Cheers,

Signature · 1 line
Julien Grall
#26 lore raw

PATCH v20210701 8/40 MAINTAINERS: add myself as saverestore maintainer

In reply to Olaf Hering

I touched it last.

---
 MAINTAINERS | 6 ++++++
 1 file changed, 6 insertions(+)
MAINTAINERS +6 -0
index 8a52a03969..36dc634958 100644
@@ -381,6 +381,12 @@ R:	Juergen Gross <[email protected]>
 S:	Supported
 F:	tools/libs/
 
+LIBSAVERESTORE:
+M:	Olaf Hering <[email protected]>
+S:	Supported
+F:	tools/include/xensaverestore.h
+F:	tools/libs/saverestore/
+
 LIBXENLIGHT
 M:	Ian Jackson <[email protected]>
 M:	Wei Liu <[email protected]>
#27 lore raw

Re: [PATCH v20210701 08/40] MAINTAINERS: add myself as saverestore maintainer

Quoted 1 line · Olaf Hering

I touched it last.

For my taste, this is too little as a justification.

MAINTAINERS R: Juergen Gross <[email protected]> · line 384

Quoted 7 lines
MAINTAINERS +1 -0
@@ -381,6 +381,12 @@ R:	Juergen Gross <[email protected]>
 S:	Supported
 F:	tools/libs/
 
+LIBSAVERESTORE:

Nit: DYM LIBXENSAVERESTORE (and hence for the new entry to be below LIBXENLIGHT)?

MAINTAINERS R: Juergen Gross <[email protected]> · line 388

Quoted 4 lines
MAINTAINERS +4 -0
+M:	Olaf Hering <[email protected]>
+S:	Supported
+F:	tools/include/xensaverestore.h
+F:	tools/libs/saverestore/

I'm afraid this goes too far: This way you remove all prior (direct) maintainers (see "The meaning of nesting" in ./MAINTAINERS). And I'm sure Andrew, who has written much of this, ought to be considered to become the maintainer of this code then as well.

Personally I think you may want to take a smaller step first and insert yourself as reviewer for this library. See e.g. what we had done a while back for "VM EVENT, MEM ACCESS and MONITOR" when new maintainers had been proposed. I may not have a sufficiently good picture of reviews you've done in the past for this part of the tree, so I'm sorry if I'm missing significant work you've done there, but surely my recent series fixing code in this area could have been a good opportunity to actually do a full round of review, when you have the intention expressed here.

Jan

Quoted 4 lines
MAINTAINERS +0 -0
 LIBXENLIGHT
 M:	Ian Jackson <[email protected]>
 M:	Wei Liu <[email protected]>
#28 lore raw

Re: [PATCH v20210701 08/40] MAINTAINERS: add myself as saverestore maintainer

Am Thu, 1 Jul 2021 12:39:06 +0200
schrieb Jan Beulich <[email protected]>:

Quoted 5 lines

I'm afraid this goes too far: This way you remove all prior (direct) maintainers (see "The meaning of nesting" in ./MAINTAINERS). And I'm sure Andrew, who has written much of this, ought to be considered to become the maintainer of this code then as well.

I think this was copy&paste from some other entry, which would still include the tools/ maintainers when using get_maintainer.pl. I do not remember which one it was.

Also I think if Andrew would have a desire to be in there he would have added himself already.

Olaf

#29 lore raw

Re: [PATCH v20210701 08/40] MAINTAINERS: add myself as saverestore maintainer

Hi Olaf,

Quoted 10 lines · Olaf Hering

Am Thu, 1 Jul 2021 12:39:06 +0200
schrieb Jan Beulich <[email protected]>:

Quoted 5 lines

I'm afraid this goes too far: This way you remove all prior (direct) maintainers (see "The meaning of nesting" in ./MAINTAINERS). And I'm sure Andrew, who has written much of this, ought to be considered to become the maintainer of this code then as well.

I think this was copy&paste from some other entry, which would still include the tools/ maintainers when using get_maintainer.pl. I do not remember which one it was.

You are mixing CCing and actual maintainers. You can be CCed without maintaining a directory.

Jan's point is tools/ maintainers would not be directly maintaining the library. You would be the sole maintainer of the directory and Jan was referring the following paragraph:

1. Under normal circumstances, the Ack of the most specific maintainer is both necessary and sufficient to get a change to a given file committed. So a change to xen/arch/x86/mm/shadow/multi.c requires the the Ack of the xen/arch/x86/mm/shadow maintainer for that part of the patch, but would not require the Ack of the xen/arch/x86 maintainer or the xen/arch/x86/mm maintainer.

Regarding your proposal to maintain the directory. I don't follow much the tools side and therefore can't judge the merit of the proposal.

However... this is not new code per-se and therefore the fact you touch last is not sufficient (otherwise I could claim the same tomorrow if I send a patch to the directory ;)).

For the commit message, I would suggest to provide some information about your contribution (including review) to the area. Also, was this discussed with the tools maintainers?

Cheers,

Signature · 1 line
Julien Grall
#30 lore raw

Re: [PATCH v20210701 08/40] MAINTAINERS: add myself as saverestore maintainer

Am Thu, 1 Jul 2021 12:40:09 +0100
schrieb Julien Grall <[email protected]>:

Quoted 1 line

You would be the sole maintainer of the directory

Yes that is the point, it changes the count from zero to one.

Olaf

#31 lore raw

Re: [PATCH v20210701 08/40] MAINTAINERS: add myself as saverestore maintainer

Hi Olaf,

Quoted 6 lines · Olaf Hering

Am Thu, 1 Jul 2021 12:40:09 +0100
schrieb Julien Grall <[email protected]>:

Quoted 1 line

You would be the sole maintainer of the directory

Yes that is the point, it changes the count from zero to one.

The code you are moving is already maintained by the tools maintainers. So I am guessing you are saying they are unresponsive...

Cheers,

Signature · 1 line
Julien Grall
#32 lore raw

PATCH v20210701 9/40 tools: add readv_exact to libxenctrl

In reply to Olaf Hering

Read a batch of iovec's.

Short reads are the common case, finish the trailing iov with read_exact.

v2:
- add comment to short-read handling

---
 tools/libs/ctrl/xc_private.c | 57 +++++++++++++++++++++++++++++++++++-
 tools/libs/ctrl/xc_private.h |  1 +
 2 files changed, 57 insertions(+), 1 deletion(-)
tools/libs/ctrl/xc_private.c +56 -1
index c0422662f0..bab9a31a70 100644
@@ -698,8 +698,23 @@ int write_exact(int fd, const void *data, size_t size)
 
 #if defined(__MINIOS__)
 /*
- * MiniOS's libc doesn't know about writev(). Implement it as multiple write()s.
+ * MiniOS's libc doesn't know about readv/writev().
+ * Implement it as multiple read/write()s.
  */
+int readv_exact(int fd, const struct iovec *iov, int iovcnt)
+{
+    int rc, i;
+
+    for ( i = 0; i < iovcnt; ++i )
+    {
+        rc = read_exact(fd, iov[i].iov_base, iov[i].iov_len);
+        if ( rc )
+            return rc;
+    }
+
+    return 0;
+}
+
 int writev_exact(int fd, const struct iovec *iov, int iovcnt)
 {
     int rc, i;
@@ -714,6 +729,46 @@ int writev_exact(int fd, const struct iovec *iov, int iovcnt)
     return 0;
 }
 #else
+int readv_exact(int fd, const struct iovec *iov, int iovcnt)
+{
+    int rc = 0, idx = 0;
+    ssize_t len;
+
+    while ( idx < iovcnt )
+    {
+        len = readv(fd, &iov[idx], min(iovcnt - idx, IOV_MAX));
+        if ( len == -1 && errno == EINTR )
+            continue;
+        if ( len <= 0 )
+        {
+            rc = -1;
+            goto out;
+        }
+
+        /* Finish a potential short read in the last iov */
+        while ( len > 0 && idx < iovcnt )
+        {
+            if ( len >= iov[idx].iov_len )
+            {
+                len -= iov[idx].iov_len;
+            }
+            else
+            {
+                void *p = iov[idx].iov_base + len;
+                size_t l = iov[idx].iov_len - len;
+
+                rc = read_exact(fd, p, l);
+                if ( rc )
+                    goto out;
+                len = 0;
+            }
+            idx++;
+        }
+    }
+out:
+    return rc;
+}
+
 int writev_exact(int fd, const struct iovec *iov, int iovcnt)
 {
     struct iovec *local_iov = NULL;
tools/libs/ctrl/xc_private.h +1 -0
index 3e299b943f..66086ef19f 100644
@@ -410,6 +410,7 @@ int xc_flush_mmu_updates(xc_interface *xch, struct xc_mmu *mmu);
 
 /* Return 0 on success; -1 on error setting errno. */
 int read_exact(int fd, void *data, size_t size); /* EOF => -1, errno=0 */
+int readv_exact(int fd, const struct iovec *iov, int iovcnt);
 int write_exact(int fd, const void *data, size_t size);
 int writev_exact(int fd, const struct iovec *iov, int iovcnt);
 
#33 lore raw

PATCH v20210701 10/40 tools: add xc_is_known_page_type to libxenctrl

In reply to Olaf Hering

Users of xc_get_pfn_type_batch may want to sanity check the data returned by Xen. Add a simple helper for this purpose.

v02:
- rename xc_is_known_page_type to sr_is_known_page_type
- move from ctrl/xc_private.h to saverestore/common.h (jgross)

---
 tools/libs/saverestore/common.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
tools/libs/saverestore/common.h +33 -0
index ca2eb47a4f..07c506360c 100644
@@ -467,6 +467,39 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
 /* Handle a STATIC_DATA_END record. */
 int handle_static_data_end(struct xc_sr_context *ctx);
 
+/* Sanitiy check for types returned by Xen */
+static inline bool sr_is_known_page_type(xen_pfn_t type)
+{
+    bool ret;
+
+    switch (type)
+    {
+    case XEN_DOMCTL_PFINFO_NOTAB:
+
+    case XEN_DOMCTL_PFINFO_L1TAB:
+    case XEN_DOMCTL_PFINFO_L1TAB | XEN_DOMCTL_PFINFO_LPINTAB:
+
+    case XEN_DOMCTL_PFINFO_L2TAB:
+    case XEN_DOMCTL_PFINFO_L2TAB | XEN_DOMCTL_PFINFO_LPINTAB:
+
+    case XEN_DOMCTL_PFINFO_L3TAB:
+    case XEN_DOMCTL_PFINFO_L3TAB | XEN_DOMCTL_PFINFO_LPINTAB:
+
+    case XEN_DOMCTL_PFINFO_L4TAB:
+    case XEN_DOMCTL_PFINFO_L4TAB | XEN_DOMCTL_PFINFO_LPINTAB:
+
+    case XEN_DOMCTL_PFINFO_XTAB:
+    case XEN_DOMCTL_PFINFO_XALLOC: /* Synthetic type in Xen 4.2 - 4.5 */
+    case XEN_DOMCTL_PFINFO_BROKEN:
+        ret = true;
+        break;
+    default:
+        ret = false;
+        break;
+    }
+    return ret;
+}
+
 #endif
 /*
  * Local variables:
#34 lore raw

Re: [PATCH v20210701 10/40] tools: add xc_is_known_page_type to libxenctrl

Quoted 4 lines · Olaf Hering

Users of xc_get_pfn_type_batch may want to sanity check the data returned by Xen. Add a simple helper for this purpose.

Subject needs correcting after v2.

However, given that this is in the save/restore common header, does it really need a prefix?  Simply is_known_page_type() seems good enough.

Quoted 13 lines

v02:
- rename xc_is_known_page_type to sr_is_known_page_type
- move from ctrl/xc_private.h to saverestore/common.h (jgross)

---
 tools/libs/saverestore/common.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
tools/libs/saverestore/common.h +0 -0
index ca2eb47a4f..07c506360c 100644

tools/libs/saverestore/common.h int populate_pfns(struct xc_sr_context *ctx, unsigned int count, · line 471

Quoted 5 lines
tools/libs/saverestore/common.h +2 -0
@@ -467,6 +467,39 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
 /* Handle a STATIC_DATA_END record. */
 int handle_static_data_end(struct xc_sr_context *ctx);
 
+/* Sanitiy check for types returned by Xen */
+static inline bool sr_is_known_page_type(xen_pfn_t type)

uint32_t

tools/libs/saverestore/common.h int populate_pfns(struct xc_sr_context *ctx, unsigned int count, · line 473

Quoted 2 lines
tools/libs/saverestore/common.h +2 -0
+{
+    bool ret;

The logic will be rather shorter and cleaner to read by dropping ret and using return directly out of the switch.

tools/libs/saverestore/common.h int populate_pfns(struct xc_sr_context *ctx, unsigned int count, · line 475

Quoted 2 lines
tools/libs/saverestore/common.h +2 -0
+
+    switch (type)

Spaces.

I can fix up everything on commit if you're happy with the suggestions.

~Andrew

#35 lore raw

Re: [PATCH v20210701 10/40] tools: add xc_is_known_page_type to libxenctrl

Am Fri, 2 Jul 2021 20:20:08 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 1 line

Subject needs correcting after v2.

Apparently I missed some places while removing the old "xc_" prefix.

Quoted 2 lines

However, given that this is in the save/restore common header, does it really need a prefix?  Simply is_known_page_type() seems good enough.

Sure, the possibility of clashes is probably low.

Quoted 3 lines

tools/libs/saverestore/common.h int populate_pfns(struct xc_sr_context *ctx, unsigned int count, · line 471

Quoted 2 lines
tools/libs/saverestore/common.h +2 -0
+/* Sanitiy check for types returned by Xen */
+static inline bool sr_is_known_page_type(xen_pfn_t type)  

uint32_t

Why is this better than returning 'bool'?

Quoted 1 line

I can fix up everything on commit if you're happy with the suggestions.

Yes, I'm certainly fine with it.

Olaf

#36 lore raw

Re: [PATCH v20210701 10/40] tools: add xc_is_known_page_type to libxenctrl

Quoted 6 lines · Olaf Hering

Am Fri, 2 Jul 2021 20:20:08 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 3 lines

tools/libs/saverestore/common.h int populate_pfns(struct xc_sr_context *ctx, unsigned int count, · line 471

Quoted 2 lines
tools/libs/saverestore/common.h +2 -0
+/* Sanitiy check for types returned by Xen */
+static inline bool sr_is_known_page_type(xen_pfn_t type)  

uint32_t

Why is this better than returning 'bool'?

For the parameter sorry, not the return type.

All type fields are uniformly uint32_t elsewhere.

Quoted 3 lines
Quoted 2 lines

I can fix up everything on commit if you're happy with the suggestions.

Yes, I'm certainly fine with it.

Ok - I'll put together a branch.

~Andrew

#37 lore raw

Re: [PATCH v20210701 10/40] tools: add xc_is_known_page_type to libxenctrl

Am Mon, 5 Jul 2021 10:51:50 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 1 line

All type fields are uniformly uint32_t elsewhere.

To me it looks like xc_get_pfn_type_batch writes to an array of xen_pfn_t.

Olaf

#38 lore raw

PATCH v20210701 11/40 tools: use sr_is_known_page_type

In reply to Olaf Hering

Verify pfn type on sending side, also verify incoming batch of pfns.

v02:
- use sr_is_known_page_type instead of xc_is_known_page_type

---
 tools/libs/saverestore/restore.c | 3 +--
 tools/libs/saverestore/save.c    | 6 ++++++
 2 files changed, 7 insertions(+), 2 deletions(-)
tools/libs/saverestore/restore.c +1 -2
index be259a1c6b..324b9050e2 100644
@@ -406,8 +406,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
         }
 
         type = (pages->pfn[i] & PAGE_DATA_TYPE_MASK) >> 32;
-        if ( ((type >> XEN_DOMCTL_PFINFO_LTAB_SHIFT) >= 5) &&
-             ((type >> XEN_DOMCTL_PFINFO_LTAB_SHIFT) <= 8) )
+        if ( sr_is_known_page_type(type) == false )
         {
             ERROR("Invalid type %#"PRIx32" for pfn %#"PRIpfn" (index %u)",
                   type, pfn, i);
tools/libs/saverestore/save.c +6 -0
index ae3e8797d0..6f820ea432 100644
@@ -147,6 +147,12 @@ static int write_batch(struct xc_sr_context *ctx)
 
     for ( i = 0; i < nr_pfns; ++i )
     {
+        if ( sr_is_known_page_type(types[i]) == false )
+        {
+            ERROR("Wrong type %#"PRIpfn" for pfn %#"PRIpfn, types[i], mfns[i]);
+            goto err;
+        }
+
         switch ( types[i] )
         {
         case XEN_DOMCTL_PFINFO_BROKEN:
#39 lore raw

Re: [PATCH v20210701 11/40] tools: use sr_is_known_page_type

Quoted 4 lines · Olaf Hering

Verify pfn type on sending side, also verify incoming batch of pfns.

Any reason this isn't folded into the previous patch, like your subsequent two page type helper patches are?

tools/libs/saverestore/save.c static int write_batch(struct xc_sr_context *ctx) · line 152

Quoted 11 lines
tools/libs/saverestore/save.c +3 -0
index ae3e8797d0..6f820ea432 100644
@@ -147,6 +147,12 @@ static int write_batch(struct xc_sr_context *ctx)
 
     for ( i = 0; i < nr_pfns; ++i )
     {
+        if ( sr_is_known_page_type(types[i]) == false )
+        {
+            ERROR("Wrong type %#"PRIpfn" for pfn %#"PRIpfn, types[i], mfns[i]);

"Unknown type" would be more accurate.

~Andrew

#40 lore raw

Re: [PATCH v20210701 11/40] tools: use sr_is_known_page_type

Am Fri, 2 Jul 2021 20:27:21 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 2 lines

Any reason this isn't folded into the previous patch, like your subsequent two page type helper patches are?

I think I wanted to separate this for simpler review, but I forgot to split the followup change as well.

Quoted 2 lines
Quoted 1 line

+ ERROR("Wrong type %#"PRIpfn" for pfn %#"PRIpfn, types[i], mfns[i]);

"Unknown type" would be more accurate.

Yes, this is better. Thanks.

Olaf

#41 lore raw

Re: [PATCH v20210701 11/40] tools: use sr_is_known_page_type

Quoted 6 lines · Olaf Hering

Am Fri, 2 Jul 2021 20:27:21 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 2 lines

Any reason this isn't folded into the previous patch, like your subsequent two page type helper patches are?

I think I wanted to separate this for simpler review, but I forgot to split the followup change as well.

All patches are largely mechanical changes.  It's easier to review together, rather than split, because you can only judge the correctness of the new helper in terms of the code it is replacing.

~Andrew

#42 lore raw

PATCH v20210701 12/40 tools: unify type checking for data pfns in migration stream

In reply to Olaf Hering

Introduce a helper which decides if a given pfn in the migration stream is backed by memory.

This specifically deals with type XEN_DOMCTL_PFINFO_XALLOC, which was a synthetic toolstack-only type used in Xen 4.2 to 4.5. It indicated a dirty page on the sending side for which no data will be send in the initial iteration.

No change in behavior intended.

---
 tools/libs/saverestore/common.h  | 17 +++++++++++++++++
 tools/libs/saverestore/restore.c |  5 ++---
 2 files changed, 19 insertions(+), 3 deletions(-)
tools/libs/saverestore/common.h +17 -0
index 07c506360c..fa242e808d 100644
@@ -500,6 +500,23 @@ static inline bool sr_is_known_page_type(xen_pfn_t type)
     return ret;
 }
 
+static inline bool page_type_to_populate(uint32_t type)
+{
+    bool ret;
+
+    switch (type)
+    {
+    case XEN_DOMCTL_PFINFO_XTAB:
+    case XEN_DOMCTL_PFINFO_BROKEN:
+        ret = false;
+        break;
+    case XEN_DOMCTL_PFINFO_XALLOC:
+    default:
+        ret = true;
+        break;
+    }
+    return ret;
+}
 #endif
 /*
  * Local variables:
tools/libs/saverestore/restore.c +2 -3
index 324b9050e2..477b7527a1 100644
@@ -152,9 +152,8 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
 
     for ( i = 0; i < count; ++i )
     {
-        if ( (!types || (types &&
-                         (types[i] != XEN_DOMCTL_PFINFO_XTAB &&
-                          types[i] != XEN_DOMCTL_PFINFO_BROKEN))) &&
+        if ( (!types ||
+              (types && page_type_to_populate(types[i]) == true)) &&
              !pfn_is_populated(ctx, original_pfns[i]) )
         {
             rc = pfn_set_populated(ctx, original_pfns[i]);
#43 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

Quoted 21 lines · Olaf Hering

Introduce a helper which decides if a given pfn in the migration stream is backed by memory.

This specifically deals with type XEN_DOMCTL_PFINFO_XALLOC, which was a synthetic toolstack-only type used in Xen 4.2 to 4.5. It indicated a dirty page on the sending side for which no data will be send in the initial iteration.

No change in behavior intended.

---
 tools/libs/saverestore/common.h  | 17 +++++++++++++++++
 tools/libs/saverestore/restore.c |  5 ++---
 2 files changed, 19 insertions(+), 3 deletions(-)
tools/libs/saverestore/common.h +0 -0
index 07c506360c..fa242e808d 100644

tools/libs/saverestore/common.h static inline bool sr_is_known_page_type(xen_pfn_t type) · line 508

Quoted 9 lines
tools/libs/saverestore/common.h +6 -0
@@ -500,6 +500,23 @@ static inline bool sr_is_known_page_type(xen_pfn_t type)
     return ret;
 }
 
+static inline bool page_type_to_populate(uint32_t type)
+{
+    bool ret;
+
+    switch (type)
+    {

Same style comments as before.

tools/libs/saverestore/common.h static inline bool sr_is_known_page_type(xen_pfn_t type) · line 516

Quoted 8 lines
tools/libs/saverestore/common.h +8 -0
+    case XEN_DOMCTL_PFINFO_XTAB:
+    case XEN_DOMCTL_PFINFO_BROKEN:
+        ret = false;
+        break;
+    case XEN_DOMCTL_PFINFO_XALLOC:
+    default:
+        ret = true;
+        break;

I know you're replacing the logic as-was, but in hindsight, I'm not sure it was great to begin with.  It defaults the unallocated types to being considered populated, which isn't a clever idea.

Anyone adding a new page type is going to have to audit/edit each of these helpers.  I think it would be better to write all the true cases explicitly.

Quoted 11 lines
tools/libs/saverestore/common.h +3 -0
+    }
+    return ret;
+}
 #endif
 /*
  * Local variables:
tools/libs/saverestore/restore.c +0 -0
index 324b9050e2..477b7527a1 100644

tools/libs/saverestore/restore.c int populate_pfns(struct xc_sr_context *ctx, unsigned int count, · line 156

Quoted 8 lines
tools/libs/saverestore/restore.c +2 -3
@@ -152,9 +152,8 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
 
     for ( i = 0; i < count; ++i )
     {
-        if ( (!types || (types &&
-                         (types[i] != XEN_DOMCTL_PFINFO_XTAB &&
-                          types[i] != XEN_DOMCTL_PFINFO_BROKEN))) &&
+        if ( (!types ||
+              (types && page_type_to_populate(types[i]) == true)) &&

I'm surprised not to have seen a compiler or static analysis complaint about this.

!A || (A && B) is redundant, and simplifies to !A || B.

Clearly need to blame whichever numpty wrote this code to begin with.

~Andrew

#44 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

Am Fri, 2 Jul 2021 20:43:13 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 3 lines

Anyone adding a new page type is going to have to audit/edit each of these helpers.  I think it would be better to write all the true cases explicitly.

You mean the check if a page has data or needs to be populated should look like sr_is_known_page_type, where each known variant is listed?

Olaf

#45 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

Quoted 7 lines · Olaf Hering

Am Fri, 2 Jul 2021 20:43:13 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 3 lines

Anyone adding a new page type is going to have to audit/edit each of these helpers.  I think it would be better to write all the true cases explicitly.

You mean the check if a page has data or needs to be populated should look like sr_is_known_page_type, where each known variant is listed?

Yes.  I think that is a safer approach overall.

~Andrew

#46 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

In reply to Olaf Hering
Quoted 7 lines · Olaf Hering

Introduce a helper which decides if a given pfn in the migration stream is backed by memory.

This specifically deals with type XEN_DOMCTL_PFINFO_XALLOC, which was a synthetic toolstack-only type used in Xen 4.2 to 4.5. It indicated a dirty page on the sending side for which no data will be send in the initial iteration.

What do you mean "This specifically deals with" ?

AFACT, the code before was correct.

~Andrew

#47 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

Quoted 1 line · Andrew Cooper

What do you mean "This specifically deals with" ?

This was a result from Jürgen pointing out that XEN_DOMCTL_PFINFO_XALLOC is not handled. If all the type checking changes go into a single commit, the commig message has to be reworded.

Olaf

#48 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

Quoted 5 lines · Olaf Hering
Quoted 1 line · Andrew Cooper

What do you mean "This specifically deals with" ?

This was a result from Jürgen pointing out that XEN_DOMCTL_PFINFO_XALLOC is not handled.

But it is...

Before the patch, we only don't populate for XTAB or BROKEN.  We populate for every other type, including the unknown/invalid ones.

There is no change in behaviour (for the non-invalid cases) that I can see.

~Andrew

#49 lore raw

Re: [PATCH v20210701 12/40] tools: unify type checking for data pfns in migration stream

Am Mon, 5 Jul 2021 19:54:21 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 1 line

But it is...

It was not handled in an earlier variant of the patch. Meanwhile the original behavior is restored with the current variant.

Olaf

#50 lore raw

PATCH v20210701 13/40 tools: unify type checking for data pfns in migration stream

In reply to Olaf Hering

Introduce a helper which decides if a given pfn type has data in the migration stream.

No change in behavior intended.

---
 tools/libs/saverestore/common.h  | 18 ++++++++++++++++++
 tools/libs/saverestore/restore.c | 29 +++--------------------------
 tools/libs/saverestore/save.c    | 14 ++------------
 3 files changed, 23 insertions(+), 38 deletions(-)
tools/libs/saverestore/common.h +18 -0
index fa242e808d..905b4078f6 100644
@@ -517,6 +517,24 @@ static inline bool page_type_to_populate(uint32_t type)
     }
     return ret;
 }
+
+static inline bool page_type_has_stream_data(uint32_t type)
+{
+    bool ret;
+
+    switch (type)
+    {
+    case XEN_DOMCTL_PFINFO_BROKEN:
+    case XEN_DOMCTL_PFINFO_XALLOC:
+    case XEN_DOMCTL_PFINFO_XTAB:
+        ret = false;
+        break;
+    default:
+        ret = true;
+        break;
+    }
+    return ret;
+}
 #endif
 /*
  * Local variables:
tools/libs/saverestore/restore.c +3 -26
index 477b7527a1..799170c7a1 100644
@@ -232,25 +232,8 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
     {
         ctx->restore.ops.set_page_type(ctx, pfns[i], types[i]);
 
-        switch ( types[i] )
-        {
-        case XEN_DOMCTL_PFINFO_NOTAB:
-
-        case XEN_DOMCTL_PFINFO_L1TAB:
-        case XEN_DOMCTL_PFINFO_L1TAB | XEN_DOMCTL_PFINFO_LPINTAB:
-
-        case XEN_DOMCTL_PFINFO_L2TAB:
-        case XEN_DOMCTL_PFINFO_L2TAB | XEN_DOMCTL_PFINFO_LPINTAB:
-
-        case XEN_DOMCTL_PFINFO_L3TAB:
-        case XEN_DOMCTL_PFINFO_L3TAB | XEN_DOMCTL_PFINFO_LPINTAB:
-
-        case XEN_DOMCTL_PFINFO_L4TAB:
-        case XEN_DOMCTL_PFINFO_L4TAB | XEN_DOMCTL_PFINFO_LPINTAB:
-
+        if ( page_type_has_stream_data(types[i]) == true )
             mfns[nr_pages++] = ctx->restore.ops.pfn_to_gfn(ctx, pfns[i]);
-            break;
-        }
     }
 
     /* Nothing to do? */
@@ -270,14 +253,8 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
 
     for ( i = 0, j = 0; i < count; ++i )
     {
-        switch ( types[i] )
-        {
-        case XEN_DOMCTL_PFINFO_XTAB:
-        case XEN_DOMCTL_PFINFO_BROKEN:
-        case XEN_DOMCTL_PFINFO_XALLOC:
-            /* No page data to deal with. */
+        if ( page_type_has_stream_data(types[i]) == false )
             continue;
-        }
 
         if ( map_errs[j] )
         {
@@ -412,7 +389,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
             goto err;
         }
 
-        if ( type < XEN_DOMCTL_PFINFO_BROKEN )
+        if ( page_type_has_stream_data(type) == true )
             /* NOTAB and all L1 through L4 tables (including pinned) should
              * have a page worth of data in the record. */
             pages_of_data++;
tools/libs/saverestore/save.c +2 -12
index 6f820ea432..12598bd4e2 100644
@@ -153,13 +153,8 @@ static int write_batch(struct xc_sr_context *ctx)
             goto err;
         }
 
-        switch ( types[i] )
-        {
-        case XEN_DOMCTL_PFINFO_BROKEN:
-        case XEN_DOMCTL_PFINFO_XALLOC:
-        case XEN_DOMCTL_PFINFO_XTAB:
+        if ( page_type_has_stream_data(types[i]) == false )
             continue;
-        }
 
         mfns[nr_pages++] = mfns[i];
     }
@@ -177,13 +172,8 @@ static int write_batch(struct xc_sr_context *ctx)
 
         for ( i = 0, p = 0; i < nr_pfns; ++i )
         {
-            switch ( types[i] )
-            {
-            case XEN_DOMCTL_PFINFO_BROKEN:
-            case XEN_DOMCTL_PFINFO_XALLOC:
-            case XEN_DOMCTL_PFINFO_XTAB:
+            if ( page_type_has_stream_data(types[i]) == false )
                 continue;
-            }
 
             if ( errors[p] )
             {
#51 lore raw

Re: [PATCH v20210701 13/40] tools: unify type checking for data pfns in migration stream

Quoted 13 lines · Olaf Hering
tools/libs/saverestore/common.h +5 -0
index fa242e808d..905b4078f6 100644
@@ -517,6 +517,24 @@ static inline bool page_type_to_populate(uint32_t type)
     }
     return ret;
 }
+
+static inline bool page_type_has_stream_data(uint32_t type)
+{
+    bool ret;
+

tools/libs/saverestore/common.h static inline bool page_type_to_populate(uint32_t type) · line 534

Quoted 10 lines
tools/libs/saverestore/common.h +10 -0
+    switch (type)
+    {
+    case XEN_DOMCTL_PFINFO_BROKEN:
+    case XEN_DOMCTL_PFINFO_XALLOC:
+    case XEN_DOMCTL_PFINFO_XTAB:
+        ret = false;
+        break;
+    default:
+        ret = true;
+        break;

As with page_type_to_populate(), we shouldn't really default the unallocated types to having stream data.

Subject to this and the other style concerned, Reviewed-by: Andrew Cooper <[email protected]>

I'm happy to fix up all the issue for the page type helpers on commit, if you're happy.

~Andrew

#52 lore raw

PATCH v20210701 14/40 tools: show migration transfer rate in send_dirty_pages

In reply to Olaf Hering

Show how fast domU pages are transferred in each iteration.

The relevant data is how fast the pfns travel, not so much how much protocol overhead exists. So the reported MiB/sec is just for pfns.

v02:
- rearrange MiB_sec calculation (jgross)

---
 tools/libs/saverestore/common.h |  2 ++
 tools/libs/saverestore/save.c   | 46 +++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
tools/libs/saverestore/common.h +2 -0
index 905b4078f6..252076cf51 100644
@@ -250,6 +250,8 @@ struct xc_sr_context
             bool debug;
 
             unsigned long p2m_size;
+            size_t pages_sent;
+            size_t overhead_sent;
 
             struct precopy_stats stats;
 
tools/libs/saverestore/save.c +46 -0

78 lines not on this page · raw message

#53 lore raw

PATCH v20210701 15/40 tools: prepare to allocate saverestore arrays once

In reply to Olaf Hering

The hotpath 'send_dirty_pages' is supposed to do just one thing: sending. The other end 'handle_page_data' is supposed to do just receiving.

But instead both do other costly work like memory allocations and data moving. Do the allocations once, the array sizes are a compiletime constant. Avoid unneeded copying of data by receiving data directly into mapped guest memory.

This patch is just prepartion, subsequent changes will populate the arrays.

Once all changes are applied, migration of a busy HVM domU changes like that:

Without this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_testing):

2020-10-29 10:23:10.711+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 55.324905335 sec, 203 MiB/sec: Internal error
2020-10-29 10:23:35.115+0000: xc: show_transfer_rate: 16829632 bytes + 2097552 pages in 24.401179720 sec, 335 MiB/sec: Internal error
2020-10-29 10:23:59.436+0000: xc: show_transfer_rate: 16829032 bytes + 2097478 pages in 24.319025928 sec, 336 MiB/sec: Internal error
2020-10-29 10:24:23.844+0000: xc: show_transfer_rate: 16829024 bytes + 2097477 pages in 24.406992500 sec, 335 MiB/sec: Internal error
2020-10-29 10:24:48.292+0000: xc: show_transfer_rate: 16828912 bytes + 2097463 pages in 24.446489027 sec, 335 MiB/sec: Internal error
2020-10-29 10:25:01.816+0000: xc: show_transfer_rate: 16836080 bytes + 2098356 pages in 13.447091818 sec, 609 MiB/sec: Internal error

With this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_unstable):

2020-10-28 21:26:05.074+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 52.564054368 sec, 213 MiB/sec: Internal error
2020-10-28 21:26:23.527+0000: xc: show_transfer_rate: 16830040 bytes + 2097603 pages in 18.450592015 sec, 444 MiB/sec: Internal error
2020-10-28 21:26:41.926+0000: xc: show_transfer_rate: 16830944 bytes + 2097717 pages in 18.397862306 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:00.339+0000: xc: show_transfer_rate: 16829176 bytes + 2097498 pages in 18.411973339 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:18.643+0000: xc: show_transfer_rate: 16828592 bytes + 2097425 pages in 18.303326695 sec, 447 MiB/sec: Internal error
2020-10-28 21:27:26.289+0000: xc: show_transfer_rate: 16835952 bytes + 2098342 pages in 7.579846749 sec, 1081 MiB/sec: Internal error

Note: the performance improvement depends on the used network cards, wirespeed and the host:
- No improvement is expected with a 1G link.
- Improvement can be seen as shown above on a 10G link.
- Just a slight improvment can be seen on a 100G link.

This change also populates sr_save_arrays with "batch_pfns", and sr_restore_arrays with "pfns" to make sure malloc is always called with a non-zero value.

v02:
- rename xc_sr_save_arrays to sr_save_arrays
- rename xc_sr_restore_arrays to sr_restore_arrays
- merge handling of "batch_pfns" and "pfns" to make sure malloc is called with a non-zero size value (jgross)

---
 tools/libs/saverestore/common.h  | 12 +++++++++++-
 tools/libs/saverestore/restore.c | 14 ++++++++++----
 tools/libs/saverestore/save.c    | 27 +++++++++++++--------------
 3 files changed, 34 insertions(+), 19 deletions(-)
tools/libs/saverestore/common.h +11 -1
index 252076cf51..968bb8af13 100644
@@ -223,6 +223,15 @@ static inline int update_blob(struct xc_sr_blob *blob,
     return 0;
 }
 
+struct sr_save_arrays {
+    xen_pfn_t batch_pfns[MAX_BATCH_SIZE];
+};
+
+struct sr_restore_arrays {
+    /* handle_page_data */
+    xen_pfn_t pfns[MAX_BATCH_SIZE];
+};
+
 struct xc_sr_context
 {
     xc_interface *xch;
@@ -255,11 +264,11 @@ struct xc_sr_context
 
             struct precopy_stats stats;
 
-            xen_pfn_t *batch_pfns;
             unsigned int nr_batch_pfns;
             unsigned long *deferred_pages;
             unsigned long nr_deferred_pages;
             xc_hypercall_buffer_t dirty_bitmap_hbuf;
+            struct sr_save_arrays *m;
         } save;
 
         struct /* Restore data. */
@@ -311,6 +320,7 @@ struct xc_sr_context
 
             /* Sender has invoked verify mode on the stream. */
             bool verify;
+            struct sr_restore_arrays *m;
         } restore;
     };
 
tools/libs/saverestore/restore.c +10 -4
index 799170c7a1..c203ce503d 100644
@@ -315,7 +315,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
     unsigned int i, pages_of_data = 0;
     int rc = -1;
 
-    xen_pfn_t *pfns = NULL, pfn;
+    xen_pfn_t *pfns = ctx->restore.m->pfns, pfn;
     uint32_t *types = NULL, type;
 
     /*
@@ -363,9 +363,8 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
         goto err;
     }
 
-    pfns = malloc(pages->count * sizeof(*pfns));
     types = malloc(pages->count * sizeof(*types));
-    if ( !pfns || !types )
+    if ( !types )
     {
         ERROR("Unable to allocate enough memory for %u pfns",
               pages->count);
@@ -412,7 +411,6 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
                            &pages->pfn[pages->count]);
  err:
     free(types);
-    free(pfns);
 
     return rc;
 }
@@ -739,6 +737,13 @@ static int setup(struct xc_sr_context *ctx)
     }
     ctx->restore.allocated_rec_num = DEFAULT_BUF_RECORDS;
 
+    ctx->restore.m = malloc(sizeof(*ctx->restore.m));
+    if ( !ctx->restore.m ) {
+        ERROR("Unable to allocate memory for arrays");
+        rc = -1;
+        goto err;
+    }
+
  err:
     return rc;
 }
@@ -757,6 +762,7 @@ static void cleanup(struct xc_sr_context *ctx)
         xc_hypercall_buffer_free_pages(
             xch, dirty_bitmap, NRPAGES(bitmap_size(ctx->restore.p2m_size)));
 
+    free(ctx->restore.m);
     free(ctx->restore.buffered_records);
     free(ctx->restore.populated_pfns);
 
tools/libs/saverestore/save.c +13 -14
index f8fbe7a742..e29b6e1d66 100644
@@ -77,7 +77,7 @@ static int write_checkpoint_record(struct xc_sr_context *ctx)
 
 /*
  * Writes a batch of memory as a PAGE_DATA record into the stream.  The batch
- * is constructed in ctx->save.batch_pfns.
+ * is constructed in ctx->save.m->batch_pfns.
  *
  * This function:
  * - gets the types for each pfn in the batch.
@@ -128,12 +128,12 @@ static int write_batch(struct xc_sr_context *ctx)
     for ( i = 0; i < nr_pfns; ++i )
     {
         types[i] = mfns[i] = ctx->save.ops.pfn_to_gfn(ctx,
-                                                      ctx->save.batch_pfns[i]);
+                                                      ctx->save.m->batch_pfns[i]);
 
         /* Likely a ballooned page. */
         if ( mfns[i] == INVALID_MFN )
         {
-            set_bit(ctx->save.batch_pfns[i], ctx->save.deferred_pages);
+            set_bit(ctx->save.m->batch_pfns[i], ctx->save.deferred_pages);
             ++ctx->save.nr_deferred_pages;
         }
     }
@@ -179,7 +179,7 @@ static int write_batch(struct xc_sr_context *ctx)
             if ( errors[p] )
             {
                 ERROR("Mapping of pfn %#"PRIpfn" (mfn %#"PRIpfn") failed %d",
-                      ctx->save.batch_pfns[i], mfns[p], errors[p]);
+                      ctx->save.m->batch_pfns[i], mfns[p], errors[p]);
                 goto err;
             }
 
@@ -193,7 +193,7 @@ static int write_batch(struct xc_sr_context *ctx)
             {
                 if ( rc == -1 && errno == EAGAIN )
                 {
-                    set_bit(ctx->save.batch_pfns[i], ctx->save.deferred_pages);
+                    set_bit(ctx->save.m->batch_pfns[i], ctx->save.deferred_pages);
                     ++ctx->save.nr_deferred_pages;
                     types[i] = XEN_DOMCTL_PFINFO_XTAB;
                     --nr_pages;
@@ -224,7 +224,7 @@ static int write_batch(struct xc_sr_context *ctx)
     rec.length += nr_pages * PAGE_SIZE;
 
     for ( i = 0; i < nr_pfns; ++i )
-        rec_pfns[i] = ((uint64_t)(types[i]) << 32) | ctx->save.batch_pfns[i];
+        rec_pfns[i] = ((uint64_t)(types[i]) << 32) | ctx->save.m->batch_pfns[i];
 
     iov[0].iov_base = &rec.type;
     iov[0].iov_len = sizeof(rec.type);
@@ -296,9 +296,9 @@ static int flush_batch(struct xc_sr_context *ctx)
 
     if ( !rc )
     {
-        VALGRIND_MAKE_MEM_UNDEFINED(ctx->save.batch_pfns,
+        VALGRIND_MAKE_MEM_UNDEFINED(ctx->save.m->batch_pfns,
                                     MAX_BATCH_SIZE *
-                                    sizeof(*ctx->save.batch_pfns));
+                                    sizeof(*ctx->save.m->batch_pfns));
     }
 
     return rc;
@@ -315,7 +315,7 @@ static int add_to_batch(struct xc_sr_context *ctx, xen_pfn_t pfn)
         rc = flush_batch(ctx);
 
     if ( rc == 0 )
-        ctx->save.batch_pfns[ctx->save.nr_batch_pfns++] = pfn;
+        ctx->save.m->batch_pfns[ctx->save.nr_batch_pfns++] = pfn;
 
     return rc;
 }
@@ -849,13 +849,12 @@ static int setup(struct xc_sr_context *ctx)
 
     dirty_bitmap = xc_hypercall_buffer_alloc_pages(
         xch, dirty_bitmap, NRPAGES(bitmap_size(ctx->save.p2m_size)));
-    ctx->save.batch_pfns = malloc(MAX_BATCH_SIZE *
-                                  sizeof(*ctx->save.batch_pfns));
     ctx->save.deferred_pages = bitmap_alloc(ctx->save.p2m_size);
+    ctx->save.m = malloc(sizeof(*ctx->save.m));
 
-    if ( !ctx->save.batch_pfns || !dirty_bitmap || !ctx->save.deferred_pages )
+    if ( !ctx->save.m || !dirty_bitmap || !ctx->save.deferred_pages )
     {
-        ERROR("Unable to allocate memory for dirty bitmaps, batch pfns and"
+        ERROR("Unable to allocate memory for dirty bitmaps and"
               " deferred pages");
         rc = -1;
         errno = ENOMEM;
@@ -884,7 +883,7 @@ static void cleanup(struct xc_sr_context *ctx)
     xc_hypercall_buffer_free_pages(xch, dirty_bitmap,
                                    NRPAGES(bitmap_size(ctx->save.p2m_size)));
     free(ctx->save.deferred_pages);
-    free(ctx->save.batch_pfns);
+    free(ctx->save.m);
 }
 
 /*
#54 lore raw

Re: [PATCH v20210701 15/40] tools: prepare to allocate saverestore arrays once

Quoted 6 lines · Olaf Hering

The hotpath 'send_dirty_pages' is supposed to do just one thing: sending. The other end 'handle_page_data' is supposed to do just receiving.

But instead both do other costly work like memory allocations and data moving. Do the allocations once, the array sizes are a compiletime constant. Avoid unneeded copying of data by receiving data directly into mapped guest memory.

There is a reason why the logic was written that way, which was good at the time.  It was so valgrind could spot bugs with the memory handling in these functions (And it did indeed find many bugs during development).

These days, its ASAN is perhaps the preferred tool, but both depend on dynamic allocations to figure out the actual size of various objects.

I agree that the repeated alloc/free of same-sized memory regions on each iteration is a waste.  However, if we are going to fix this by using one-off allocations, then we want to compensate with logic such as the call to VALGRIND_MAKE_MEM_UNDEFINED() in flush_batch(), and I think we still need individual allocations to let the tools work properly.

Quoted 13 lines

This patch is just prepartion, subsequent changes will populate the arrays.

Once all changes are applied, migration of a busy HVM domU changes like that:

Without this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_testing):

2020-10-29 10:23:10.711+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 55.324905335 sec, 203 MiB/sec: Internal error
2020-10-29 10:23:35.115+0000: xc: show_transfer_rate: 16829632 bytes + 2097552 pages in 24.401179720 sec, 335 MiB/sec: Internal error
2020-10-29 10:23:59.436+0000: xc: show_transfer_rate: 16829032 bytes + 2097478 pages in 24.319025928 sec, 336 MiB/sec: Internal error
2020-10-29 10:24:23.844+0000: xc: show_transfer_rate: 16829024 bytes + 2097477 pages in 24.406992500 sec, 335 MiB/sec: Internal error
2020-10-29 10:24:48.292+0000: xc: show_transfer_rate: 16828912 bytes + 2097463 pages in 24.446489027 sec, 335 MiB/sec: Internal error
2020-10-29 10:25:01.816+0000: xc: show_transfer_rate: 16836080 bytes + 2098356 pages in 13.447091818 sec, 609 MiB/sec: Internal error
Quoted 7 lines

With this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_unstable):

2020-10-28 21:26:05.074+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 52.564054368 sec, 213 MiB/sec: Internal error
2020-10-28 21:26:23.527+0000: xc: show_transfer_rate: 16830040 bytes + 2097603 pages in 18.450592015 sec, 444 MiB/sec: Internal error
2020-10-28 21:26:41.926+0000: xc: show_transfer_rate: 16830944 bytes + 2097717 pages in 18.397862306 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:00.339+0000: xc: show_transfer_rate: 16829176 bytes + 2097498 pages in 18.411973339 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:18.643+0000: xc: show_transfer_rate: 16828592 bytes + 2097425 pages in 18.303326695 sec, 447 MiB/sec: Internal error
2020-10-28 21:27:26.289+0000: xc: show_transfer_rate: 16835952 bytes + 2098342 pages in 7.579846749 sec, 1081 MiB/sec: Internal error

These are good numbers, and clearly show that there is some value here, but shouldn't they be in the series header?  They're not terribly relevant to this patch specifically.

Also, while I can believe that the first sample is slower than the later ones (in particular, during the first round, we've got to deal with the non-RAM regions too and therefore spend more time making hypercalls), I'm not sure I believe the final sample.  Given the byte/page count, the substantially smaller elapsed time looks suspicious.

Quoted 5 lines

Note: the performance improvement depends on the used network cards, wirespeed and the host:
- No improvement is expected with a 1G link.
- Improvement can be seen as shown above on a 10G link.
- Just a slight improvment can be seen on a 100G link.

Are these observations with an otherwise idle dom0?

Even if CPU time in dom0 wasn't the bottlekneck with a 1G link, the reduction in CPU time you observe at higher link speeds will still be making a difference at 1G, and will probably be visible if you perform multiple concurrent migrations.

~Andrew

#55 lore raw

Re: [PATCH v20210701 15/40] tools: prepare to allocate saverestore arrays once

Am Mon, 5 Jul 2021 11:44:30 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 6 lines · Olaf Hering

I agree that the repeated alloc/free of same-sized memory regions on each iteration is a waste.  However, if we are going to fix this by using one-off allocations, then we want to compensate with logic such as the call to VALGRIND_MAKE_MEM_UNDEFINED() in flush_batch(), and I think we still need individual allocations to let the tools work properly.

If this is a concern, lets just do a few individual arrays.

Quoted 20 lines
Quoted 12 lines

This patch is just prepartion, subsequent changes will populate the arrays.

Once all changes are applied, migration of a busy HVM domU changes like that:

Without this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_testing):

2020-10-29 10:23:10.711+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 55.324905335 sec, 203 MiB/sec: Internal error
2020-10-29 10:23:35.115+0000: xc: show_transfer_rate: 16829632 bytes + 2097552 pages in 24.401179720 sec, 335 MiB/sec: Internal error
2020-10-29 10:23:59.436+0000: xc: show_transfer_rate: 16829032 bytes + 2097478 pages in 24.319025928 sec, 336 MiB/sec: Internal error
2020-10-29 10:24:23.844+0000: xc: show_transfer_rate: 16829024 bytes + 2097477 pages in 24.406992500 sec, 335 MiB/sec: Internal error
2020-10-29 10:24:48.292+0000: xc: show_transfer_rate: 16828912 bytes + 2097463 pages in 24.446489027 sec, 335 MiB/sec: Internal error
2020-10-29 10:25:01.816+0000: xc: show_transfer_rate: 16836080 bytes + 2098356 pages in 13.447091818 sec, 609 MiB/sec: Internal error
Quoted 7 lines

With this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_unstable):

2020-10-28 21:26:05.074+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 52.564054368 sec, 213 MiB/sec: Internal error
2020-10-28 21:26:23.527+0000: xc: show_transfer_rate: 16830040 bytes + 2097603 pages in 18.450592015 sec, 444 MiB/sec: Internal error
2020-10-28 21:26:41.926+0000: xc: show_transfer_rate: 16830944 bytes + 2097717 pages in 18.397862306 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:00.339+0000: xc: show_transfer_rate: 16829176 bytes + 2097498 pages in 18.411973339 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:18.643+0000: xc: show_transfer_rate: 16828592 bytes + 2097425 pages in 18.303326695 sec, 447 MiB/sec: Internal error
2020-10-28 21:27:26.289+0000: xc: show_transfer_rate: 16835952 bytes + 2098342 pages in 7.579846749 sec, 1081 MiB/sec: Internal error  
Quoted 3 lines

These are good numbers, and clearly show that there is some value here, but shouldn't they be in the series header?  They're not terribly relevant to this patch specifically.

The cover letter is unfortunately not under version control. Perhaps there are ways with git notes, I never use it.

Quoted 5 lines

Also, while I can believe that the first sample is slower than the later ones (in particular, during the first round, we've got to deal with the non-RAM regions too and therefore spend more time making hypercalls), I'm not sure I believe the final sample.  Given the byte/page count, the substantially smaller elapsed time looks suspicious.

The first one is slower because it has to wait for the receiver to allocate pages.
But maybe as you said there are other aspects as well.
The last one is always way faster because apparently map/unmap is less costly with a stopped guest. Right now the code may reach up to 15Gbit/s. The next step is to map the domU just once to reach wirespeed.

Quoted 1 line

Are these observations with an otherwise idle dom0?

Yes. Idle dom0 and a domU busy with touching its memory.

Unfortunately, I'm not able to prove the reported gain with the systems I have today.
I'm waiting for preparation of different hardware, right now I have only a pair of CoyotePass and WilsonCity.

I'm sure there were NUMA effects involved. Last years libvirt was unable to properly pin vcpus. If I pin all the involved memory to node#0 there is some jitter in the logged numbers, but no obvious improvement. The fist iteration is slightly faster, but that is it.

Meanwhile I think this commit message needs to be redone.

Quoted 4 lines

Even if CPU time in dom0 wasn't the bottlekneck with a 1G link, the reduction in CPU time you observe at higher link speeds will still be making a difference at 1G, and will probably be visible if you perform multiple concurrent migrations.

Yes, I will see what numbers I get with two or more migrations running in parallel.

Olaf

#56 lore raw

Re: [PATCH v20210701 15/40] tools: prepare to allocate saverestore arrays once

Quoted 22 lines · Olaf Hering

Am Mon, 5 Jul 2021 11:44:30 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 19 lines
Quoted 19 lines

This patch is just prepartion, subsequent changes will populate the arrays.

Once all changes are applied, migration of a busy HVM domU changes like that:

Without this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_testing):

2020-10-29 10:23:10.711+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 55.324905335 sec, 203 MiB/sec: Internal error
2020-10-29 10:23:35.115+0000: xc: show_transfer_rate: 16829632 bytes + 2097552 pages in 24.401179720 sec, 335 MiB/sec: Internal error
2020-10-29 10:23:59.436+0000: xc: show_transfer_rate: 16829032 bytes + 2097478 pages in 24.319025928 sec, 336 MiB/sec: Internal error
2020-10-29 10:24:23.844+0000: xc: show_transfer_rate: 16829024 bytes + 2097477 pages in 24.406992500 sec, 335 MiB/sec: Internal error
2020-10-29 10:24:48.292+0000: xc: show_transfer_rate: 16828912 bytes + 2097463 pages in 24.446489027 sec, 335 MiB/sec: Internal error
2020-10-29 10:25:01.816+0000: xc: show_transfer_rate: 16836080 bytes + 2098356 pages in 13.447091818 sec, 609 MiB/sec: Internal error

With this series, from sr650 to sr950 (xen-4.15.20201027T173911.16a20963b3 xen_unstable):

2020-10-28 21:26:05.074+0000: xc: show_transfer_rate: 23663128 bytes + 2879563 pages in 52.564054368 sec, 213 MiB/sec: Internal error
2020-10-28 21:26:23.527+0000: xc: show_transfer_rate: 16830040 bytes + 2097603 pages in 18.450592015 sec, 444 MiB/sec: Internal error
2020-10-28 21:26:41.926+0000: xc: show_transfer_rate: 16830944 bytes + 2097717 pages in 18.397862306 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:00.339+0000: xc: show_transfer_rate: 16829176 bytes + 2097498 pages in 18.411973339 sec, 445 MiB/sec: Internal error
2020-10-28 21:27:18.643+0000: xc: show_transfer_rate: 16828592 bytes + 2097425 pages in 18.303326695 sec, 447 MiB/sec: Internal error
2020-10-28 21:27:26.289+0000: xc: show_transfer_rate: 16835952 bytes + 2098342 pages in 7.579846749 sec, 1081 MiB/sec: Internal error  
Quoted 5 lines
Quoted 3 lines

These are good numbers, and clearly show that there is some value here, but shouldn't they be in the series header?  They're not terribly relevant to this patch specifically.

The cover letter is unfortunately not under version control. Perhaps there are ways with git notes, I never use it.

In the end, we'll want some kind of note in the changelog, but that wants to be a single line.  Its probably fine to say "Improve migration performance.  25% better bandwidth when NIC link speed is the bottleneck, due to optimising the data handling logic."

Quoted 8 lines
Quoted 5 lines

Also, while I can believe that the first sample is slower than the later ones (in particular, during the first round, we've got to deal with the non-RAM regions too and therefore spend more time making hypercalls), I'm not sure I believe the final sample.  Given the byte/page count, the substantially smaller elapsed time looks suspicious.

The first one is slower because it has to wait for the receiver to allocate pages. But maybe as you said there are other aspects as well.
The last one is always way faster because apparently map/unmap is less costly with a stopped guest.

That's suspicious.  If true, we've got some very wonky behaviour in the hypervisor...

Quoted 1 line

Right now the code may reach up to 15Gbit/s. The next step is to map the domU just once to reach wirespeed.

We can in principle do that in 64bit toolstacks, for HVM guests.  But not usefully until we've fixed the fact that Xen has no idea what the guest physmap is supposed to look like.

At the moment, the current scheme is a little more resilient to bugs caused by the guest attempting to balloon during the live phase.

Another area to improve, which can be started now, is to avoid bounce buffering hypercall data.  Now that we have /dev/xen/hypercall which you can mmap() regular kernel pages from, what we want is a simple memory allocator which we can allocate permanent hypercall buffers from, rather than the internals of every xc_*() hypercall wrapper bouncing the data in (potentially) both directions.

Quoted 8 lines
Quoted 2 lines

Are these observations with an otherwise idle dom0?

Yes. Idle dom0 and a domU busy with touching its memory.

Unfortunately, I'm not able to prove the reported gain with the systems I have today.
I'm waiting for preparation of different hardware, right now I have only a pair of CoyotePass and WilsonCity.

I'm sure there were NUMA effects involved. Last years libvirt was unable to properly pin vcpus. If I pin all the involved memory to node#0 there is some jitter in the logged numbers, but no obvious improvement. The fist iteration is slightly faster, but that is it.

Oh - so the speedup might not be from reduced data handling?

Avoiding unnecessary data copies is clearly going to improve things, even if it isn't 25%.

~Andrew

#57 lore raw

Re: [PATCH v20210701 15/40] tools: prepare to allocate saverestore arrays once

Am Mon, 5 Jul 2021 14:01:07 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 3 lines
Quoted 1 line

The last one is always way faster because apparently map/unmap is less costly with a stopped guest.

That's suspicious.  If true, we've got some very wonky behaviour in the hypervisor...

At least the transfer rate this last iteration is consistent.
Since the only difference I can see is the fact that the domU is suspended, I suspect the mapping.
I did no investigation where the time is spent, I should probably do that one day to better understand this specific difference.

Quoted 5 lines
Quoted 1 line

Right now the code may reach up to 15Gbit/s. The next step is to map the domU just once to reach wirespeed.

We can in principle do that in 64bit toolstacks, for HVM guests.  But not usefully until we've fixed the fact that Xen has no idea what the guest physmap is supposed to look like.

Why would Xen care?
My attempt last year with a new save/restore code did just 'map' the memory on both sides. The 'unmap' was done in exit().

With this approach I got wirespeed in all iterations with a 10G link.

Quoted 2 lines

At the moment, the current scheme is a little more resilient to bugs caused by the guest attempting to balloon during the live phase.

I did not specifically test how a domU behaves when it claims and releases pages while being migrated. I think this series would handle at least parts of that:
If a page appears or disappears it will be recognized by getpageframeinfo.
If a page disappears between getpageframeinfo and MMAPBATCH I expect an error.
This error is fatal right now, perhaps the code could catch this and move on.
If a page disappears after MMAPBATCH it will be caught by later iterations.

Quoted 6 lines

Another area to improve, which can be started now, is to avoid bounce buffering hypercall data.  Now that we have /dev/xen/hypercall which you can mmap() regular kernel pages from, what we want is a simple memory allocator which we can allocate permanent hypercall buffers from, rather than the internals of every xc_*() hypercall wrapper bouncing the data in (potentially) both directions.

That sounds like a good idea. Not sure how costly the current approach is.

Quoted 1 line

Oh - so the speedup might not be from reduced data handling?

At least not on the systems I have now.

Perhaps I should test how the numbers look like with the NIC and the toolstack in node#0, and the domU in node#1.

Olaf

#58 lore raw

Re: [PATCH v20210701 15/40] tools: prepare to allocate saverestore arrays once

In reply to Andrew Cooper

Am Mon, 5 Jul 2021 14:01:07 +0100
schrieb Andrew Cooper <[email protected]>:

Quoted 9 lines
Quoted 4 lines

Unfortunately, I'm not able to prove the reported gain with the systems I have today.
I'm waiting for preparation of different hardware, right now I have only a pair of CoyotePass and WilsonCity.

I'm sure there were NUMA effects involved. Last years libvirt was unable to properly pin vcpus. If I pin all the involved memory to node#0 there is some jitter in the logged numbers, but no obvious improvement. The fist iteration is slightly faster, but that is it.

Oh - so the speedup might not be from reduced data handling?

Avoiding unnecessary data copies is clearly going to improve things, even if it isn't 25%.

For HVM the only notable improvement is the initial iteration.

On average with 4 migrations of a single domU from A to B and back from B to A, transfer rate goes up from ~490MiB/s to ~677MiB/s. The initial transfer time for the 4194299 domU pages:

with plain staging:
36.800582009
32.145531727
31.827540709
33.009956041
34.951513466
33.416769973
32.128985762
33.201786076

with the series applied:
24.266428156
24.632898175
24.112660134
23.603475994
24.418323859
23.841875914
25.087779229
23.493812677

Migration of a PV domU is much faster, but transfer rate for each iteration varies with or without the patches being applied.

Olaf

#59 lore raw

PATCH v20210701 16/40 tools: save: move mfns array

In reply to Olaf Hering

Remove allocation from hotpath, move mfns array into preallocated space.

---
 tools/libs/saverestore/common.h | 2 ++
 tools/libs/saverestore/save.c   | 7 ++-----
 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/saverestore/common.h +2 -0
index 968bb8af13..1415a182d2 100644
@@ -225,6 +225,8 @@ static inline int update_blob(struct xc_sr_blob *blob,
 
 struct sr_save_arrays {
     xen_pfn_t batch_pfns[MAX_BATCH_SIZE];
+    /* write_batch: Mfns of the batch pfns. */
+    xen_pfn_t mfns[MAX_BATCH_SIZE];
 };
 
 struct sr_restore_arrays {
tools/libs/saverestore/save.c +2 -5
index e29b6e1d66..6b09784be8 100644
@@ -88,7 +88,7 @@ static int write_checkpoint_record(struct xc_sr_context *ctx)
 static int write_batch(struct xc_sr_context *ctx)
 {
     xc_interface *xch = ctx->xch;
-    xen_pfn_t *mfns = NULL, *types = NULL;
+    xen_pfn_t *mfns = ctx->save.m->mfns, *types = NULL;
     void *guest_mapping = NULL;
     void **guest_data = NULL;
     void **local_pages = NULL;
@@ -105,8 +105,6 @@ static int write_batch(struct xc_sr_context *ctx)
 
     assert(nr_pfns != 0);
 
-    /* Mfns of the batch pfns. */
-    mfns = malloc(nr_pfns * sizeof(*mfns));
     /* Types of the batch pfns. */
     types = malloc(nr_pfns * sizeof(*types));
     /* Errors from attempting to map the gfns. */
@@ -118,7 +116,7 @@ static int write_batch(struct xc_sr_context *ctx)
     /* iovec[] for writev(). */
     iov = malloc((nr_pfns + 4) * sizeof(*iov));
 
-    if ( !mfns || !types || !errors || !guest_data || !local_pages || !iov )
+    if ( !types || !errors || !guest_data || !local_pages || !iov )
     {
         ERROR("Unable to allocate arrays for a batch of %u pages",
               nr_pfns);
@@ -277,7 +275,6 @@ static int write_batch(struct xc_sr_context *ctx)
     free(guest_data);
     free(errors);
     free(types);
-    free(mfns);
 
     return rc;
 }
#60 lore raw

PATCH v20210701 17/40 tools: save: move types array

In reply to Olaf Hering

Remove allocation from hotpath, move types array into preallocated space.

---
 tools/libs/saverestore/common.h | 2 ++
 tools/libs/saverestore/save.c   | 7 ++-----
 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/saverestore/common.h +2 -0
index 1415a182d2..5bd2913cb6 100644
@@ -227,6 +227,8 @@ struct sr_save_arrays {
     xen_pfn_t batch_pfns[MAX_BATCH_SIZE];
     /* write_batch: Mfns of the batch pfns. */
     xen_pfn_t mfns[MAX_BATCH_SIZE];
+    /* write_batch: Types of the batch pfns. */
+    xen_pfn_t types[MAX_BATCH_SIZE];
 };
 
 struct sr_restore_arrays {
tools/libs/saverestore/save.c +2 -5
index 6b09784be8..0883c1fac0 100644
@@ -88,7 +88,7 @@ static int write_checkpoint_record(struct xc_sr_context *ctx)
 static int write_batch(struct xc_sr_context *ctx)
 {
     xc_interface *xch = ctx->xch;
-    xen_pfn_t *mfns = ctx->save.m->mfns, *types = NULL;
+    xen_pfn_t *mfns = ctx->save.m->mfns, *types = ctx->save.m->types;
     void *guest_mapping = NULL;
     void **guest_data = NULL;
     void **local_pages = NULL;
@@ -105,8 +105,6 @@ static int write_batch(struct xc_sr_context *ctx)
 
     assert(nr_pfns != 0);
 
-    /* Types of the batch pfns. */
-    types = malloc(nr_pfns * sizeof(*types));
     /* Errors from attempting to map the gfns. */
     errors = malloc(nr_pfns * sizeof(*errors));
     /* Pointers to page data to send.  Mapped gfns or local allocations. */
@@ -116,7 +114,7 @@ static int write_batch(struct xc_sr_context *ctx)
     /* iovec[] for writev(). */
     iov = malloc((nr_pfns + 4) * sizeof(*iov));
 
-    if ( !types || !errors || !guest_data || !local_pages || !iov )
+    if ( !errors || !guest_data || !local_pages || !iov )
     {
         ERROR("Unable to allocate arrays for a batch of %u pages",
               nr_pfns);
@@ -274,7 +272,6 @@ static int write_batch(struct xc_sr_context *ctx)
     free(local_pages);
     free(guest_data);
     free(errors);
-    free(types);
 
     return rc;
 }
#61 lore raw

PATCH v20210701 18/40 tools: save: move errors array

In reply to Olaf Hering

Remove allocation from hotpath, move errors array into preallocated space.

---
 tools/libs/saverestore/common.h | 2 ++
 tools/libs/saverestore/save.c   | 7 ++-----
 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/saverestore/common.h +2 -0
index 5bd2913cb6..25ee8fcb0f 100644
@@ -229,6 +229,8 @@ struct sr_save_arrays {
     xen_pfn_t mfns[MAX_BATCH_SIZE];
     /* write_batch: Types of the batch pfns. */
     xen_pfn_t types[MAX_BATCH_SIZE];
+    /* write_batch: Errors from attempting to map the gfns. */
+    int errors[MAX_BATCH_SIZE];
 };
 
 struct sr_restore_arrays {
tools/libs/saverestore/save.c +2 -5
index 0883c1fac0..9ebbf00ce7 100644
@@ -92,7 +92,7 @@ static int write_batch(struct xc_sr_context *ctx)
     void *guest_mapping = NULL;
     void **guest_data = NULL;
     void **local_pages = NULL;
-    int *errors = NULL, rc = -1;
+    int *errors = ctx->save.m->errors, rc = -1;
     unsigned int i, p, nr_pages = 0, nr_pages_mapped = 0;
     unsigned int nr_pfns = ctx->save.nr_batch_pfns;
     void *page, *orig_page;
@@ -105,8 +105,6 @@ static int write_batch(struct xc_sr_context *ctx)
 
     assert(nr_pfns != 0);
 
-    /* Errors from attempting to map the gfns. */
-    errors = malloc(nr_pfns * sizeof(*errors));
     /* Pointers to page data to send.  Mapped gfns or local allocations. */
     guest_data = calloc(nr_pfns, sizeof(*guest_data));
     /* Pointers to locally allocated pages.  Need freeing. */
@@ -114,7 +112,7 @@ static int write_batch(struct xc_sr_context *ctx)
     /* iovec[] for writev(). */
     iov = malloc((nr_pfns + 4) * sizeof(*iov));
 
-    if ( !errors || !guest_data || !local_pages || !iov )
+    if ( !guest_data || !local_pages || !iov )
     {
         ERROR("Unable to allocate arrays for a batch of %u pages",
               nr_pfns);
@@ -271,7 +269,6 @@ static int write_batch(struct xc_sr_context *ctx)
     free(iov);
     free(local_pages);
     free(guest_data);
-    free(errors);
 
     return rc;
 }
#62 lore raw

PATCH v20210701 19/40 tools: save: move iov array

In reply to Olaf Hering

Remove allocation from hotpath, move iov array into preallocated space.

---
 tools/libs/saverestore/common.h | 2 ++
 tools/libs/saverestore/save.c   | 7 ++-----
 2 files changed, 4 insertions(+), 5 deletions(-)
tools/libs/saverestore/common.h +2 -0
index 25ee8fcb0f..c8a30acf7b 100644
@@ -231,6 +231,8 @@ struct sr_save_arrays {
     xen_pfn_t types[MAX_BATCH_SIZE];
     /* write_batch: Errors from attempting to map the gfns. */
     int errors[MAX_BATCH_SIZE];
+    /* write_batch: iovec[] for writev(). */
+    struct iovec iov[MAX_BATCH_SIZE + 4];
 };
 
 struct sr_restore_arrays {
tools/libs/saverestore/save.c +2 -5
index 9ebbf00ce7..1a5f3d29ea 100644
@@ -97,7 +97,7 @@ static int write_batch(struct xc_sr_context *ctx)
     unsigned int nr_pfns = ctx->save.nr_batch_pfns;
     void *page, *orig_page;
     uint64_t *rec_pfns = NULL;
-    struct iovec *iov = NULL; int iovcnt = 0;
+    struct iovec *iov = ctx->save.m->iov; int iovcnt = 0;
     struct xc_sr_rec_page_data_header hdr = { 0 };
     struct xc_sr_record rec = {
         .type = REC_TYPE_PAGE_DATA,
@@ -109,10 +109,8 @@ static int write_batch(struct xc_sr_context *ctx)
     guest_data = calloc(nr_pfns, sizeof(*guest_data));
     /* Pointers to locally allocated pages.  Need freeing. */
     local_pages = calloc(nr_pfns, sizeof(*local_pages));
-    /* iovec[] for writev(). */
-    iov = malloc((nr_pfns + 4) * sizeof(*iov));
 
-    if ( !guest_data || !local_pages || !iov )
+    if ( !guest_data || !local_pages )
     {
         ERROR("Unable to allocate arrays for a batch of %u pages",
               nr_pfns);
@@ -266,7 +264,6 @@ static int write_batch(struct xc_sr_context *ctx)
         xenforeignmemory_unmap(xch->fmem, guest_mapping, nr_pages_mapped);
     for ( i = 0; local_pages && i < nr_pfns; ++i )
         free(local_pages[i]);
-    free(iov);
     free(local_pages);
     free(guest_data);
 
#63 lore raw

PATCH v20210701 20/40 tools: save: move rec_pfns array

In reply to Olaf Hering

Remove allocation from hotpath, move rec_pfns array into preallocated space.

---
 tools/libs/saverestore/common.h |  2 ++
 tools/libs/saverestore/save.c   | 11 +----------
 2 files changed, 3 insertions(+), 10 deletions(-)
tools/libs/saverestore/common.h +2 -0
index c8a30acf7b..3994ab3844 100644
@@ -233,6 +233,8 @@ struct sr_save_arrays {
     int errors[MAX_BATCH_SIZE];
     /* write_batch: iovec[] for writev(). */
     struct iovec iov[MAX_BATCH_SIZE + 4];
+    /* write_batch */
+    uint64_t rec_pfns[MAX_BATCH_SIZE];
 };
 
 struct sr_restore_arrays {
tools/libs/saverestore/save.c +1 -10
index 1a5f3d29ea..0f02988ff9 100644
@@ -96,7 +96,7 @@ static int write_batch(struct xc_sr_context *ctx)
     unsigned int i, p, nr_pages = 0, nr_pages_mapped = 0;
     unsigned int nr_pfns = ctx->save.nr_batch_pfns;
     void *page, *orig_page;
-    uint64_t *rec_pfns = NULL;
+    uint64_t *rec_pfns = ctx->save.m->rec_pfns;
     struct iovec *iov = ctx->save.m->iov; int iovcnt = 0;
     struct xc_sr_rec_page_data_header hdr = { 0 };
     struct xc_sr_record rec = {
@@ -201,14 +201,6 @@ static int write_batch(struct xc_sr_context *ctx)
         }
     }
 
-    rec_pfns = malloc(nr_pfns * sizeof(*rec_pfns));
-    if ( !rec_pfns )
-    {
-        ERROR("Unable to allocate %zu bytes of memory for page data pfn list",
-              nr_pfns * sizeof(*rec_pfns));
-        goto err;
-    }
-
     hdr.count = nr_pfns;
 
     rec.length = sizeof(hdr);
@@ -259,7 +251,6 @@ static int write_batch(struct xc_sr_context *ctx)
     rc = ctx->save.nr_batch_pfns = 0;
 
  err:
-    free(rec_pfns);
     if ( guest_mapping )
         xenforeignmemory_unmap(xch->fmem, guest_mapping, nr_pages_mapped);
     for ( i = 0; local_pages && i < nr_pfns; ++i )
#64 lore raw

PATCH v20210701 21/40 tools: save: move guest_data array

In reply to Olaf Hering

Remove allocation from hotpath, move guest_data array into preallocated space.

Because this was allocated with calloc:
Adjust the loop to clear unused entries as needed.

---
 tools/libs/saverestore/common.h |  2 ++
 tools/libs/saverestore/save.c   | 11 ++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)
tools/libs/saverestore/common.h +2 -0
index 3994ab3844..c3570e0c9a 100644
@@ -235,6 +235,8 @@ struct sr_save_arrays {
     struct iovec iov[MAX_BATCH_SIZE + 4];
     /* write_batch */
     uint64_t rec_pfns[MAX_BATCH_SIZE];
+    /* write_batch: Pointers to page data to send. Mapped gfns or local allocations. */
+    void *guest_data[MAX_BATCH_SIZE];
 };
 
 struct sr_restore_arrays {
tools/libs/saverestore/save.c +6 -5
index 0f02988ff9..ea04cb1a74 100644
@@ -90,7 +90,7 @@ static int write_batch(struct xc_sr_context *ctx)
     xc_interface *xch = ctx->xch;
     xen_pfn_t *mfns = ctx->save.m->mfns, *types = ctx->save.m->types;
     void *guest_mapping = NULL;
-    void **guest_data = NULL;
+    void **guest_data = ctx->save.m->guest_data;
     void **local_pages = NULL;
     int *errors = ctx->save.m->errors, rc = -1;
     unsigned int i, p, nr_pages = 0, nr_pages_mapped = 0;
@@ -105,12 +105,10 @@ static int write_batch(struct xc_sr_context *ctx)
 
     assert(nr_pfns != 0);
 
-    /* Pointers to page data to send.  Mapped gfns or local allocations. */
-    guest_data = calloc(nr_pfns, sizeof(*guest_data));
     /* Pointers to locally allocated pages.  Need freeing. */
     local_pages = calloc(nr_pfns, sizeof(*local_pages));
 
-    if ( !guest_data || !local_pages )
+    if ( !local_pages )
     {
         ERROR("Unable to allocate arrays for a batch of %u pages",
               nr_pfns);
@@ -166,7 +164,10 @@ static int write_batch(struct xc_sr_context *ctx)
         for ( i = 0, p = 0; i < nr_pfns; ++i )
         {
             if ( page_type_has_stream_data(types[i]) == false )
+            {
+                guest_data[i] = NULL;
                 continue;
+            }
 
             if ( errors[p] )
             {
@@ -183,6 +184,7 @@ static int write_batch(struct xc_sr_context *ctx)
 
             if ( rc )
             {
+                guest_data[i] = NULL;
                 if ( rc == -1 && errno == EAGAIN )
                 {
                     set_bit(ctx->save.m->batch_pfns[i], ctx->save.deferred_pages);
@@ -256,7 +258,6 @@ static int write_batch(struct xc_sr_context *ctx)
     for ( i = 0; local_pages && i < nr_pfns; ++i )
         free(local_pages[i]);
     free(local_pages);
-    free(guest_data);
 
     return rc;
 }
#65 lore raw

PATCH v20210701 22/40 tools: save: move local_pages array

In reply to Olaf Hering

Remove allocation from hotpath, move local_pages array into preallocated space.

Adjust the code to use the src page as is in case of HVM. In case of PV the page may need to be normalised, use a private memory area for this purpose.

---
 tools/libs/saverestore/common.h       | 22 ++++++++++---------
 tools/libs/saverestore/save.c         | 25 +++------------------
 tools/libs/saverestore/save_x86_hvm.c |  5 +++--
 tools/libs/saverestore/save_x86_pv.c  | 31 ++++++++++++++++++---------
 4 files changed, 39 insertions(+), 44 deletions(-)
tools/libs/saverestore/common.h +12 -10
index c3570e0c9a..8089449011 100644
@@ -45,16 +45,12 @@ struct xc_sr_save_ops
      * Optionally transform the contents of a page from being specific to the
      * sending environment, to being generic for the stream.
      *
-     * The page of data at the end of 'page' may be a read-only mapping of a
-     * running guest; it must not be modified.  If no transformation is
-     * required, the callee should leave '*pages' untouched.
+     * The page of data '*src' may be a read-only mapping of a running guest;
+     * it must not be modified. If no transformation is required, the callee
+     * should leave '*src' untouched, and return it via '**ptr'.
      *
-     * If a transformation is required, the callee should allocate themselves
-     * a local page using malloc() and return it via '*page'.
-     *
-     * The caller shall free() '*page' in all cases.  In the case that the
-     * callee encounters an error, it should *NOT* free() the memory it
-     * allocated for '*page'.
+     * If a transformation is required, the callee should provide the
+     * transformed page in a private buffer and return it via '**ptr'.
      *
      * It is valid to fail with EAGAIN if the transformation is not able to be
      * completed at this point.  The page shall be retried later.
@@ -62,7 +58,7 @@ struct xc_sr_save_ops
      * @returns 0 for success, -1 for failure, with errno appropriately set.
      */
     int (*normalise_page)(struct xc_sr_context *ctx, xen_pfn_t type,
-                          void **page);
+                          void *src, unsigned int idx, void **ptr);
 
     /**
      * Set up local environment to save a domain. (Typically querying
@@ -385,6 +381,12 @@ struct xc_sr_context
 
                 union
                 {
+                    struct
+                    {
+                        /* Used by write_batch for modified pages. */
+                        void *normalised_pages;
+                    } save;
+
                     struct
                     {
                         /* State machine for the order of received records. */
tools/libs/saverestore/save.c +3 -22
index ea04cb1a74..fa83648f9a 100644
@@ -91,11 +91,10 @@ static int write_batch(struct xc_sr_context *ctx)
     xen_pfn_t *mfns = ctx->save.m->mfns, *types = ctx->save.m->types;
     void *guest_mapping = NULL;
     void **guest_data = ctx->save.m->guest_data;
-    void **local_pages = NULL;
     int *errors = ctx->save.m->errors, rc = -1;
     unsigned int i, p, nr_pages = 0, nr_pages_mapped = 0;
     unsigned int nr_pfns = ctx->save.nr_batch_pfns;
-    void *page, *orig_page;
+    void *src;
     uint64_t *rec_pfns = ctx->save.m->rec_pfns;
     struct iovec *iov = ctx->save.m->iov; int iovcnt = 0;
     struct xc_sr_rec_page_data_header hdr = { 0 };
@@ -105,16 +104,6 @@ static int write_batch(struct xc_sr_context *ctx)
 
     assert(nr_pfns != 0);
 
-    /* Pointers to locally allocated pages.  Need freeing. */
-    local_pages = calloc(nr_pfns, sizeof(*local_pages));
-
-    if ( !local_pages )
-    {
-        ERROR("Unable to allocate arrays for a batch of %u pages",
-              nr_pfns);
-        goto err;
-    }
-
     for ( i = 0; i < nr_pfns; ++i )
     {
         types[i] = mfns[i] = ctx->save.ops.pfn_to_gfn(ctx,
@@ -176,11 +165,8 @@ static int write_batch(struct xc_sr_context *ctx)
                 goto err;
             }
 
-            orig_page = page = guest_mapping + (p * PAGE_SIZE);
-            rc = ctx->save.ops.normalise_page(ctx, types[i], &page);
-
-            if ( orig_page != page )
-                local_pages[i] = page;
+            src = guest_mapping + (p * PAGE_SIZE);
+            rc = ctx->save.ops.normalise_page(ctx, types[i], src, i, &guest_data[i]);
 
             if ( rc )
             {
@@ -195,8 +181,6 @@ static int write_batch(struct xc_sr_context *ctx)
                 else
                     goto err;
             }
-            else
-                guest_data[i] = page;
 
             rc = -1;
             ++p;
@@ -255,9 +239,6 @@ static int write_batch(struct xc_sr_context *ctx)
  err:
     if ( guest_mapping )
         xenforeignmemory_unmap(xch->fmem, guest_mapping, nr_pages_mapped);
-    for ( i = 0; local_pages && i < nr_pfns; ++i )
-        free(local_pages[i]);
-    free(local_pages);
 
     return rc;
 }
tools/libs/saverestore/save_x86_hvm.c +3 -2
index 91c2cb99ab..26f49ee267 100644
@@ -129,9 +129,10 @@ static xen_pfn_t x86_hvm_pfn_to_gfn(const struct xc_sr_context *ctx,
     return pfn;
 }
 
-static int x86_hvm_normalise_page(struct xc_sr_context *ctx,
-                                  xen_pfn_t type, void **page)
+static int x86_hvm_normalise_page(struct xc_sr_context *ctx, xen_pfn_t type,
+                                  void *src, unsigned int idx, void **ptr)
 {
+    *ptr = src;
     return 0;
 }
 
tools/libs/saverestore/save_x86_pv.c +21 -10
index 92f77fad0f..159ff59480 100644
@@ -999,29 +999,31 @@ static xen_pfn_t x86_pv_pfn_to_gfn(const struct xc_sr_context *ctx,
  * save_ops function.  Performs pagetable normalisation on appropriate pages.
  */
 static int x86_pv_normalise_page(struct xc_sr_context *ctx, xen_pfn_t type,
-                                 void **page)
+                                  void *src, unsigned int idx, void **ptr)
 {
     xc_interface *xch = ctx->xch;
-    void *local_page;
     int rc;
+    void *dst;
 
     type &= XEN_DOMCTL_PFINFO_LTABTYPE_MASK;
 
     if ( type < XEN_DOMCTL_PFINFO_L1TAB || type > XEN_DOMCTL_PFINFO_L4TAB )
+    {
+        *ptr = src;
         return 0;
+    }
 
-    local_page = malloc(PAGE_SIZE);
-    if ( !local_page )
+    if ( idx >= MAX_BATCH_SIZE )
     {
-        ERROR("Unable to allocate scratch page");
-        rc = -1;
-        goto out;
+        ERROR("idx %u out of range", idx);
+        errno = ERANGE;
+        return -1;
     }
 
-    rc = normalise_pagetable(ctx, *page, local_page, type);
-    *page = local_page;
+    dst = ctx->x86.pv.save.normalised_pages + idx * PAGE_SIZE;
+    rc = normalise_pagetable(ctx, src, dst, type);
+    *ptr = dst;
 
- out:
     return rc;
 }
 
@@ -1031,8 +1033,16 @@ static int x86_pv_normalise_page(struct xc_sr_context *ctx, xen_pfn_t type,
  */
 static int x86_pv_setup(struct xc_sr_context *ctx)
 {
+    xc_interface *xch = ctx->xch;
     int rc;
 
+    ctx->x86.pv.save.normalised_pages = malloc(MAX_BATCH_SIZE * PAGE_SIZE);
+    if ( !ctx->x86.pv.save.normalised_pages )
+    {
+        PERROR("Failed to allocate normalised_pages");
+        return -1;
+    }
+
     rc = x86_pv_domain_info(ctx);
     if ( rc )
         return rc;
@@ -1118,6 +1128,7 @@ static int x86_pv_check_vm_state(struct xc_sr_context *ctx)
 
 static int x86_pv_cleanup(struct xc_sr_context *ctx)
 {
+    free(ctx->x86.pv.save.normalised_pages);
     free(ctx->x86.pv.p2m_pfns);
 
     if ( ctx->x86.pv.p2m )
#66 lore raw

PATCH v20210701 23/40 tools: restore: move types array

In reply to Olaf Hering

Remove allocation from hotpath, move types array into preallocated space.

---
 tools/libs/saverestore/common.h  |  1 +
 tools/libs/saverestore/restore.c | 12 +-----------
 2 files changed, 2 insertions(+), 11 deletions(-)
tools/libs/saverestore/common.h +1 -0
index 8089449011..d798b79745 100644
@@ -238,6 +238,7 @@ struct sr_save_arrays {
 struct sr_restore_arrays {
     /* handle_page_data */
     xen_pfn_t pfns[MAX_BATCH_SIZE];
+    uint32_t types[MAX_BATCH_SIZE];
 };
 
 struct xc_sr_context
tools/libs/saverestore/restore.c +1 -11
index c203ce503d..8ea125cf73 100644
@@ -316,7 +316,7 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
     int rc = -1;
 
     xen_pfn_t *pfns = ctx->restore.m->pfns, pfn;
-    uint32_t *types = NULL, type;
+    uint32_t *types = ctx->restore.m->types, type;
 
     /*
      * v2 compatibility only exists for x86 streams.  This is a bit of a
@@ -363,14 +363,6 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
         goto err;
     }
 
-    types = malloc(pages->count * sizeof(*types));
-    if ( !types )
-    {
-        ERROR("Unable to allocate enough memory for %u pfns",
-              pages->count);
-        goto err;
-    }
-
     for ( i = 0; i < pages->count; ++i )
     {
         pfn = pages->pfn[i] & PAGE_DATA_PFN_MASK;
@@ -410,8 +402,6 @@ static int handle_page_data(struct xc_sr_context *ctx, struct xc_sr_record *rec)
     rc = process_page_data(ctx, pages->count, pfns, types,
                            &pages->pfn[pages->count]);
  err:
-    free(types);
-
     return rc;
 }
 
#67 lore raw

PATCH v20210701 24/40 tools: restore: move mfns array

In reply to Olaf Hering

Remove allocation from hotpath, move mfns array into preallocated space.

---
 tools/libs/saverestore/common.h  | 2 ++
 tools/libs/saverestore/restore.c | 5 ++---
 2 files changed, 4 insertions(+), 3 deletions(-)
tools/libs/saverestore/common.h +2 -0
index d798b79745..9d7efff03d 100644
@@ -239,6 +239,8 @@ struct sr_restore_arrays {
     /* handle_page_data */
     xen_pfn_t pfns[MAX_BATCH_SIZE];
     uint32_t types[MAX_BATCH_SIZE];
+    /* process_page_data */
+    xen_pfn_t mfns[MAX_BATCH_SIZE];
 };
 
 struct xc_sr_context
tools/libs/saverestore/restore.c +2 -3
index 8ea125cf73..d7ea52b89e 100644
@@ -205,7 +205,7 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
                              xen_pfn_t *pfns, uint32_t *types, void *page_data)
 {
     xc_interface *xch = ctx->xch;
-    xen_pfn_t *mfns = malloc(count * sizeof(*mfns));
+    xen_pfn_t *mfns = ctx->restore.m->mfns;
     int *map_errs = malloc(count * sizeof(*map_errs));
     int rc;
     void *mapping = NULL, *guest_page = NULL;
@@ -213,7 +213,7 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
         j,          /* j indexes the subset of pfns we decide to map. */
         nr_pages = 0;
 
-    if ( !mfns || !map_errs )
+    if ( !map_errs )
     {
         rc = -1;
         ERROR("Failed to allocate %zu bytes to process page data",
@@ -299,7 +299,6 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
         xenforeignmemory_unmap(xch->fmem, mapping, nr_pages);
 
     free(map_errs);
-    free(mfns);
 
     return rc;
 }
#68 lore raw

PATCH v20210701 25/40 tools: restore: move map_errs array

In reply to Olaf Hering

Remove allocation from hotpath, move map_errs array into preallocated space.

---
 tools/libs/saverestore/common.h  |  1 +
 tools/libs/saverestore/restore.c | 12 +-----------
 2 files changed, 2 insertions(+), 11 deletions(-)
tools/libs/saverestore/common.h +1 -0
index 9d7efff03d..7684c35e22 100644
@@ -241,6 +241,7 @@ struct sr_restore_arrays {
     uint32_t types[MAX_BATCH_SIZE];
     /* process_page_data */
     xen_pfn_t mfns[MAX_BATCH_SIZE];
+    int map_errs[MAX_BATCH_SIZE];
 };
 
 struct xc_sr_context
tools/libs/saverestore/restore.c +1 -11
index d7ea52b89e..578ee1accb 100644
@@ -206,21 +206,13 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
 {
     xc_interface *xch = ctx->xch;
     xen_pfn_t *mfns = ctx->restore.m->mfns;
-    int *map_errs = malloc(count * sizeof(*map_errs));
+    int *map_errs = ctx->restore.m->map_errs;
     int rc;
     void *mapping = NULL, *guest_page = NULL;
     unsigned int i, /* i indexes the pfns from the record. */
         j,          /* j indexes the subset of pfns we decide to map. */
         nr_pages = 0;
 
-    if ( !map_errs )
-    {
-        rc = -1;
-        ERROR("Failed to allocate %zu bytes to process page data",
-              count * (sizeof(*mfns) + sizeof(*map_errs)));
-        goto err;
-    }
-
     rc = populate_pfns(ctx, count, pfns, types);
     if ( rc )
     {
@@ -298,8 +290,6 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,
     if ( mapping )
         xenforeignmemory_unmap(xch->fmem, mapping, nr_pages);
 
-    free(map_errs);
-
     return rc;
 }
 
#69 lore raw

PATCH v20210701 26/40 tools: restore: move mfns array in populate_pfns

In reply to Olaf Hering

Remove allocation from hotpath, move populate_pfns mfns array into preallocated space. Use some prefix to avoid conflict with an array used in handle_page_data.

---
 tools/libs/saverestore/common.h  | 2 ++
 tools/libs/saverestore/restore.c | 5 ++---
 2 files changed, 4 insertions(+), 3 deletions(-)
tools/libs/saverestore/common.h +2 -0
index 7684c35e22..9d2ea96583 100644
@@ -242,6 +242,8 @@ struct sr_restore_arrays {
     /* process_page_data */
     xen_pfn_t mfns[MAX_BATCH_SIZE];
     int map_errs[MAX_BATCH_SIZE];
+    /* populate_pfns */
+    xen_pfn_t pp_mfns[MAX_BATCH_SIZE];
 };
 
 struct xc_sr_context
tools/libs/saverestore/restore.c +2 -3
index 578ee1accb..7418abf1c5 100644
@@ -138,12 +138,12 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
                   const xen_pfn_t *original_pfns, const uint32_t *types)
 {
     xc_interface *xch = ctx->xch;
-    xen_pfn_t *mfns = malloc(count * sizeof(*mfns)),
+    xen_pfn_t *mfns = ctx->restore.m->pp_mfns,
         *pfns = malloc(count * sizeof(*pfns));
     unsigned int i, nr_pfns = 0;
     int rc = -1;
 
-    if ( !mfns || !pfns )
+    if ( !pfns )
     {
         ERROR("Failed to allocate %zu bytes for populating the physmap",
               2 * count * sizeof(*mfns));
@@ -191,7 +191,6 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
 
  err:
     free(pfns);
-    free(mfns);
 
     return rc;
 }
#70 lore raw

PATCH v20210701 27/40 tools: restore: move pfns array in populate_pfns

In reply to Olaf Hering

Remove allocation from hotpath, move populate_pfns' pfns array into preallocated space. Use some prefix to avoid conflict with an array used in handle_page_data.

---
 tools/libs/saverestore/common.h  |  1 +
 tools/libs/saverestore/restore.c | 11 +----------
 2 files changed, 2 insertions(+), 10 deletions(-)
tools/libs/saverestore/common.h +1 -0
index 9d2ea96583..c319148f8f 100644
@@ -244,6 +244,7 @@ struct sr_restore_arrays {
     int map_errs[MAX_BATCH_SIZE];
     /* populate_pfns */
     xen_pfn_t pp_mfns[MAX_BATCH_SIZE];
+    xen_pfn_t pp_pfns[MAX_BATCH_SIZE];
 };
 
 struct xc_sr_context
tools/libs/saverestore/restore.c +1 -10
index 7418abf1c5..2a6ccce847 100644
@@ -139,17 +139,10 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
 {
     xc_interface *xch = ctx->xch;
     xen_pfn_t *mfns = ctx->restore.m->pp_mfns,
-        *pfns = malloc(count * sizeof(*pfns));
+        *pfns = ctx->restore.m->pp_pfns;
     unsigned int i, nr_pfns = 0;
     int rc = -1;
 
-    if ( !pfns )
-    {
-        ERROR("Failed to allocate %zu bytes for populating the physmap",
-              2 * count * sizeof(*mfns));
-        goto err;
-    }
-
     for ( i = 0; i < count; ++i )
     {
         if ( (!types ||
@@ -190,8 +183,6 @@ int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
     rc = 0;
 
  err:
-    free(pfns);
-
     return rc;
 }
 
#71 lore raw

PATCH v20210701 28/40 tools: restore: split record processing

In reply to Olaf Hering

handle_page_data must be able to read directly into mapped guest memory. This will avoid unneccesary memcpy calls for data which can be consumed verbatim.

Rearrange the code to allow decisions based on the incoming record.

This change is preparation for future changes in handle_page_data, no change in behavior is intended.

---
 tools/libs/saverestore/common.c  | 33 ++++++++++++---------
 tools/libs/saverestore/common.h  |  4 ++-
 tools/libs/saverestore/restore.c | 49 ++++++++++++++++++++++----------
 tools/libs/saverestore/save.c    |  7 ++++-
 4 files changed, 63 insertions(+), 30 deletions(-)
tools/libs/saverestore/common.c +20 -13
index 77128bc747..7da7fa4e2c 100644
@@ -91,26 +91,33 @@ int write_split_record(struct xc_sr_context *ctx, struct xc_sr_record *rec,
     return -1;
 }
 
-int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec)
+int read_record_header(struct xc_sr_context *ctx, int fd, struct xc_sr_rhdr *rhdr)
 {
     xc_interface *xch = ctx->xch;
-    struct xc_sr_rhdr rhdr;
-    size_t datasz;
 
-    if ( read_exact(fd, &rhdr, sizeof(rhdr)) )
+    if ( read_exact(fd, rhdr, sizeof(*rhdr)) )
     {
         PERROR("Failed to read Record Header from stream");
         return -1;
     }
 
-    if ( rhdr.length > REC_LENGTH_MAX )
+    if ( rhdr->length > REC_LENGTH_MAX )
     {
-        ERROR("Record (0x%08x, %s) length %#x exceeds max (%#x)", rhdr.type,
-              rec_type_to_str(rhdr.type), rhdr.length, REC_LENGTH_MAX);
+        ERROR("Record (0x%08x, %s) length %#x exceeds max (%#x)", rhdr->type,
+              rec_type_to_str(rhdr->type), rhdr->length, REC_LENGTH_MAX);
         return -1;
     }
 
-    datasz = ROUNDUP(rhdr.length, REC_ALIGN_ORDER);
+    return 0;
+}
+
+int read_record_data(struct xc_sr_context *ctx, int fd, struct xc_sr_rhdr *rhdr,
+                     struct xc_sr_record *rec)
+{
+    xc_interface *xch = ctx->xch;
+    size_t datasz;
+
+    datasz = ROUNDUP(rhdr->length, REC_ALIGN_ORDER);
 
     if ( datasz )
     {
@@ -119,7 +126,7 @@ int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec)
         if ( !rec->data )
         {
             ERROR("Unable to allocate %zu bytes for record data (0x%08x, %s)",
-                  datasz, rhdr.type, rec_type_to_str(rhdr.type));
+                  datasz, rhdr->type, rec_type_to_str(rhdr->type));
             return -1;
         }
 
@@ -128,18 +135,18 @@ int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec)
             free(rec->data);
             rec->data = NULL;
             PERROR("Failed to read %zu bytes of data for record (0x%08x, %s)",
-                   datasz, rhdr.type, rec_type_to_str(rhdr.type));
+                   datasz, rhdr->type, rec_type_to_str(rhdr->type));
             return -1;
         }
     }
     else
         rec->data = NULL;
 
-    rec->type   = rhdr.type;
-    rec->length = rhdr.length;
+    rec->type   = rhdr->type;
+    rec->length = rhdr->length;
 
     return 0;
-};
+}
 
 static void __attribute__((unused)) build_assertions(void)
 {
tools/libs/saverestore/common.h +3 -1
index c319148f8f..580eafacc8 100644
@@ -487,7 +487,9 @@ static inline int write_record(struct xc_sr_context *ctx,
  *
  * On failure, the contents of the record structure are undefined.
  */
-int read_record(struct xc_sr_context *ctx, int fd, struct xc_sr_record *rec);
+int read_record_header(struct xc_sr_context *ctx, int fd, struct xc_sr_rhdr *rhdr);
+int read_record_data(struct xc_sr_context *ctx, int fd, struct xc_sr_rhdr *rhdr,
+                     struct xc_sr_record *rec);
 
 /*
  * This would ideally be private in restore.c, but is needed by
tools/libs/saverestore/restore.c +34 -15

125 lines not on this page · raw message

tools/libs/saverestore/save.c +6 -1
index fa83648f9a..e486bce96f 100644
@@ -589,6 +589,7 @@ static int send_memory_live(struct xc_sr_context *ctx)
 static int colo_merge_secondary_dirty_bitmap(struct xc_sr_context *ctx)
 {
     xc_interface *xch = ctx->xch;
+    struct xc_sr_rhdr rhdr;
     struct xc_sr_record rec;
     uint64_t *pfns = NULL;
     uint64_t pfn;
@@ -597,7 +598,11 @@ static int colo_merge_secondary_dirty_bitmap(struct xc_sr_context *ctx)
     DECLARE_HYPERCALL_BUFFER_SHADOW(unsigned long, dirty_bitmap,
                                     &ctx->save.dirty_bitmap_hbuf);
 
-    rc = read_record(ctx, ctx->save.recv_fd, &rec);
+    rc = read_record_header(ctx, ctx->save.recv_fd, &rhdr);
+    if ( rc )
+        goto err;
+
+    rc = read_record_data(ctx, ctx->save.recv_fd, &rhdr, &rec);
     if ( rc )
         goto err;
 
#72 lore raw

PATCH v20210701 29/40 tools: restore: split handle_page_data

In reply to Olaf Hering

handle_page_data must be able to read directly into mapped guest memory. This will avoid unneccesary memcpy calls for data that can be consumed verbatim.

Split the various steps of record processing:
- move processing to handle_buffered_page_data
- adjust xenforeignmemory_map to set errno in case of failure
- adjust verify mode to set errno in case of failure

This change is preparation for future changes in handle_page_data, no change in behavior is intended.

---
 tools/libs/saverestore/common.h  |   9 +
 tools/libs/saverestore/restore.c | 343 ++++++++++++++++++++-----------
 2 files changed, 231 insertions(+), 121 deletions(-)
tools/libs/saverestore/common.h +9 -0
index 580eafacc8..96bd0ab80e 100644
@@ -242,9 +242,14 @@ struct sr_restore_arrays {
     /* process_page_data */
     xen_pfn_t mfns[MAX_BATCH_SIZE];
     int map_errs[MAX_BATCH_SIZE];
+    void *guest_data[MAX_BATCH_SIZE];
+
     /* populate_pfns */
     xen_pfn_t pp_mfns[MAX_BATCH_SIZE];
     xen_pfn_t pp_pfns[MAX_BATCH_SIZE];
+
+    /* Must be the last member */
+    struct xc_sr_rec_page_data_header pages;
 };
 
 struct xc_sr_context
@@ -335,7 +340,11 @@ struct xc_sr_context
 
             /* Sender has invoked verify mode on the stream. */
             bool verify;
+            void *verify_buf;
+
             struct sr_restore_arrays *m;
+            void *guest_mapping;
+            uint32_t nr_mapped_pages;
         } restore;
     };
 
tools/libs/saverestore/restore.c +222 -121

412 lines not on this page · raw message

#73 lore raw

PATCH v20210701 30/40 tools: restore: write data directly into guest

In reply to Olaf Hering

Read incoming migration stream directly into the guest memory. This avoids the memory allocation and copying, and the resulting performance penalty.

---
 tools/libs/saverestore/common.h  |   1 +
 tools/libs/saverestore/restore.c | 132 ++++++++++++++++++++++++++++++-
 2 files changed, 129 insertions(+), 4 deletions(-)
tools/libs/saverestore/common.h +1 -0
index 96bd0ab80e..3adcf2f83f 100644
@@ -243,6 +243,7 @@ struct sr_restore_arrays {
     xen_pfn_t mfns[MAX_BATCH_SIZE];
     int map_errs[MAX_BATCH_SIZE];
     void *guest_data[MAX_BATCH_SIZE];
+    struct iovec iov[MAX_BATCH_SIZE];
 
     /* populate_pfns */
     xen_pfn_t pp_mfns[MAX_BATCH_SIZE];
tools/libs/saverestore/restore.c +128 -4

145 lines not on this page · raw message

#74 lore raw

PATCH v20210701 31/40 tools: recognize LIBXL_API_VERSION for 4.16

In reply to Olaf Hering

This is required by upcoming API changes.

---
 tools/include/libxl.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
tools/include/libxl.h +2 -1
index ae7fe27c1f..29931626a2 100644
@@ -729,7 +729,8 @@ typedef struct libxl__ctx libxl_ctx;
 #if LIBXL_API_VERSION != 0x040200 && LIBXL_API_VERSION != 0x040300 && \
     LIBXL_API_VERSION != 0x040400 && LIBXL_API_VERSION != 0x040500 && \
     LIBXL_API_VERSION != 0x040700 && LIBXL_API_VERSION != 0x040800 && \
-    LIBXL_API_VERSION != 0x041300 && LIBXL_API_VERSION != 0x041400
+    LIBXL_API_VERSION != 0x041300 && LIBXL_API_VERSION != 0x041400 && \
+    LIBXL_API_VERSION != 0x041600
 #error Unknown LIBXL_API_VERSION
 #endif
 #endif
#75 lore raw

PATCH v20210701 32/40 tools: adjust libxl_domain_suspend to receive a struct props

In reply to Olaf Hering

Upcoming changes will pass more knobs down to xc_domain_save. Adjust the libxl_domain_suspend API to allow easy adding of additional knobs.

No change in behavior intented.

---
 tools/include/libxl.h                | 26 +++++++++++++++++++++++---
 tools/libs/light/libxl_domain.c      |  7 ++++---
 tools/ocaml/libs/xl/xenlight_stubs.c |  3 ++-
 tools/xl/xl_migrate.c                |  9 ++++++---
 tools/xl/xl_saverestore.c            |  3 ++-
 5 files changed, 37 insertions(+), 11 deletions(-)
tools/include/libxl.h +23 -3
index 29931626a2..9a4d7514ed 100644
@@ -1706,12 +1706,32 @@ static inline int libxl_retrieve_domain_configuration_0x041200(
     libxl_retrieve_domain_configuration_0x041200
 #endif
 
+/*
+ * LIBXL_HAVE_DOMAIN_SUSPEND_PROPS indicates that the
+ * libxl_domain_suspend_props() function takes a props struct.
+ */
+#define LIBXL_HAVE_DOMAIN_SUSPEND_PROPS 1
+
+typedef struct {
+    uint32_t flags; /* LIBXL_SUSPEND_* */
+} libxl_domain_suspend_props;
+#define LIBXL_SUSPEND_DEBUG 1
+#define LIBXL_SUSPEND_LIVE 2
+
 int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
-                         int flags, /* LIBXL_SUSPEND_* */
+                         libxl_domain_suspend_props *props,
                          const libxl_asyncop_how *ao_how)
                          LIBXL_EXTERNAL_CALLERS_ONLY;
-#define LIBXL_SUSPEND_DEBUG 1
-#define LIBXL_SUSPEND_LIVE 2
+#if defined(LIBXL_API_VERSION) && LIBXL_API_VERSION < 0x041600
+static inline int libxl_domain_suspend_0x041500(libxl_ctx *ctx, uint32_t domid,
+                         int fd, int flags, /* LIBXL_SUSPEND_* */
+                         const libxl_asyncop_how *ao_how)
+{
+    libxl_domain_suspend_props props = { .flags = flags, };
+    return libxl_domain_suspend(ctx, domid, fd, &props, ao_how);
+}
+#define libxl_domain_suspend libxl_domain_suspend_0x041500
+#endif
 
 /*
  * Only suspend domain, do not save its state to file, do not destroy it.
tools/libs/light/libxl_domain.c +4 -3
index c00c36c928..5dbd27900f 100644
@@ -505,7 +505,8 @@ static void domain_suspend_cb(libxl__egc *egc,
 
 }
 
-int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
+int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
+                         libxl_domain_suspend_props *props,
                          const libxl_asyncop_how *ao_how)
 {
     AO_CREATE(ctx, domid, ao_how);
@@ -526,8 +527,8 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
     dss->domid = domid;
     dss->fd = fd;
     dss->type = type;
-    dss->live = flags & LIBXL_SUSPEND_LIVE;
-    dss->debug = flags & LIBXL_SUSPEND_DEBUG;
+    dss->live = props->flags & LIBXL_SUSPEND_LIVE;
+    dss->debug = props->flags & LIBXL_SUSPEND_DEBUG;
     dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
 
     rc = libxl__fd_flags_modify_save(gc, dss->fd,
tools/ocaml/libs/xl/xenlight_stubs.c +2 -1
index 352a00134d..eaf7bce35a 100644
@@ -614,10 +614,11 @@ value stub_libxl_domain_suspend(value ctx, value domid, value fd, value async, v
 	int ret;
 	uint32_t c_domid = Int_val(domid);
 	int c_fd = Int_val(fd);
+    libxl_domain_suspend_props props = {};
 	libxl_asyncop_how *ao_how = aohow_val(async);
 
 	caml_enter_blocking_section();
-	ret = libxl_domain_suspend(CTX, c_domid, c_fd, 0, ao_how);
+	ret = libxl_domain_suspend(CTX, c_domid, c_fd, &props, ao_how);
 	caml_leave_blocking_section();
 
 	free(ao_how);
tools/xl/xl_migrate.c +6 -3
index b8594f44a5..144890924f 100644
@@ -186,7 +186,10 @@ static void migrate_domain(uint32_t domid, int preserve_domid,
     char *away_domname;
     char rc_buf;
     uint8_t *config_data;
-    int config_len, flags = LIBXL_SUSPEND_LIVE;
+    int config_len;
+    libxl_domain_suspend_props props = {
+        .flags = LIBXL_SUSPEND_LIVE,
+        };
 
     save_domain_core_begin(domid, preserve_domid, override_config_file,
                            &config_data, &config_len);
@@ -205,8 +208,8 @@ static void migrate_domain(uint32_t domid, int preserve_domid,
     xtl_stdiostream_adjust_flags(logger, XTL_STDIOSTREAM_HIDE_PROGRESS, 0);
 
     if (debug)
-        flags |= LIBXL_SUSPEND_DEBUG;
-    rc = libxl_domain_suspend(ctx, domid, send_fd, flags, NULL);
+        props.flags |= LIBXL_SUSPEND_DEBUG;
+    rc = libxl_domain_suspend(ctx, domid, send_fd, &props, NULL);
     if (rc) {
         fprintf(stderr, "migration sender: libxl_domain_suspend failed"
                 " (rc=%d)\n", rc);
tools/xl/xl_saverestore.c +2 -1
index 953d791d1a..476d4d9a6a 100644
@@ -130,6 +130,7 @@ static int save_domain(uint32_t domid, int preserve_domid,
     int fd;
     uint8_t *config_data;
     int config_len;
+    libxl_domain_suspend_props props = {};
 
     save_domain_core_begin(domid, preserve_domid, override_config_file,
                            &config_data, &config_len);
@@ -146,7 +147,7 @@ static int save_domain(uint32_t domid, int preserve_domid,
 
     save_domain_core_writeconfig(fd, filename, config_data, config_len);
 
-    int rc = libxl_domain_suspend(ctx, domid, fd, 0, NULL);
+    int rc = libxl_domain_suspend(ctx, domid, fd, &props, NULL);
     close(fd);
 
     if (rc < 0) {
#76 lore raw

PATCH v20210701 33/40 tools: change struct precopy_stats to precopy_stats_t

In reply to Olaf Hering

This will help libxl_save_msgs_gen.pl to copy the struct as a region of memory.

No change in behavior intented.

---
 tools/include/xensaverestore.h  | 7 +++----
 tools/libs/saverestore/common.h | 2 +-
 tools/libs/saverestore/save.c   | 6 +++---
 3 files changed, 7 insertions(+), 8 deletions(-)
tools/include/xensaverestore.h +3 -4
index 0410f0469e..dca0134605 100644
@@ -23,18 +23,17 @@
 #define XCFLAGS_DEBUG     (1 << 1)
 
 /* For save's precopy_policy(). */
-struct precopy_stats
-{
+typedef struct {
     unsigned int iteration;
     unsigned long total_written;
     long dirty_count; /* -1 if unknown */
-};
+} precopy_stats_t;
 
 /*
  * A precopy_policy callback may not be running in the same address
  * space as libxc an so precopy_stats is passed by value.
  */
-typedef int (*precopy_policy_t)(struct precopy_stats, void *);
+typedef int (*precopy_policy_t)(precopy_stats_t, void *);
 
 /* callbacks provided by xc_domain_save */
 struct save_callbacks {
tools/libs/saverestore/common.h +1 -1
index 3adcf2f83f..bb7e437291 100644
@@ -283,7 +283,7 @@ struct xc_sr_context
             size_t pages_sent;
             size_t overhead_sent;
 
-            struct precopy_stats stats;
+            precopy_stats_t stats;
 
             unsigned int nr_batch_pfns;
             unsigned long *deferred_pages;
tools/libs/saverestore/save.c +3 -3
index e486bce96f..537b977ba8 100644
@@ -488,7 +488,7 @@ static int update_progress_string(struct xc_sr_context *ctx, char **str)
 #define SPP_MAX_ITERATIONS      5
 #define SPP_TARGET_DIRTY_COUNT 50
 
-static int simple_precopy_policy(struct precopy_stats stats, void *user)
+static int simple_precopy_policy(precopy_stats_t stats, void *user)
 {
     return ((stats.dirty_count >= 0 &&
              stats.dirty_count < SPP_TARGET_DIRTY_COUNT) ||
@@ -515,13 +515,13 @@ static int send_memory_live(struct xc_sr_context *ctx)
     precopy_policy_t precopy_policy = ctx->save.callbacks->precopy_policy;
     void *data = ctx->save.callbacks->data;
 
-    struct precopy_stats *policy_stats;
+    precopy_stats_t *policy_stats;
 
     rc = update_progress_string(ctx, &progress_str);
     if ( rc )
         goto out;
 
-    ctx->save.stats = (struct precopy_stats){
+    ctx->save.stats = (precopy_stats_t){
         .dirty_count = ctx->save.p2m_size,
     };
     policy_stats = &ctx->save.stats;
#77 lore raw

Re: [PATCH v20210701 33/40] tools: change struct precopy_stats to precopy_stats_t

Quoted 16 lines · Olaf Hering

This will help libxl_save_msgs_gen.pl to copy the struct as a region of memory.

No change in behavior intented.

---
 tools/include/xensaverestore.h  | 7 +++----
 tools/libs/saverestore/common.h | 2 +-
 tools/libs/saverestore/save.c   | 6 +++---
 3 files changed, 7 insertions(+), 8 deletions(-)
tools/include/xensaverestore.h +0 -0
index 0410f0469e..dca0134605 100644

tools/include/xensaverestore.h line 26

Quoted 4 lines
tools/include/xensaverestore.h +0 -1
@@ -23,18 +23,17 @@
 #define XCFLAGS_DEBUG     (1 << 1)
 
 /* For save's precopy_policy(). */
-struct precopy_stats

I don't think changing the existing API is a good idea. It's probably ok to add a typedef. But can't libxl_save_msgs_gen.pl been able to deal with thing like 'struct precopy_stats' ? It seems to be able to deal with 'unsigned long'.

tools/include/xensaverestore.h line 31

Quoted 8 lines
tools/include/xensaverestore.h +2 -2
-{
+typedef struct {
     unsigned int iteration;
     unsigned long total_written;
     long dirty_count; /* -1 if unknown */
-};
+} precopy_stats_t;
 

Thanks,

Signature · 1 line
Anthony PERARD
#78 lore raw

Re: [PATCH v20210701 33/40] tools: change struct precopy_stats to precopy_stats_t

Am Thu, 1 Jul 2021 17:45:12 +0100
schrieb Anthony PERARD <[email protected]>:

Quoted 3 lines

But can't libxl_save_msgs_gen.pl been able to deal with thing like 'struct precopy_stats' ? It seems to be able to deal with 'unsigned long'.

Yes, this is apparently possible.
I have to check why I thought it was required to turn this into a typedef. Right now I do not see the reason in the code comments.

Olaf

#79 lore raw

PATCH v20210701 34/40 tools: add callback to libxl for precopy_policy and precopy_stats_t

In reply to Olaf Hering

This duplicates simple_precopy_policy. To recap its purpose:
- do up to 5 iterations of copying dirty domU memory to target, including the initial copying of all domU memory, excluding the final copying while the domU is suspended
- do fewer iterations in case the domU dirtied less than 50 pages

Take the opportunity to also move xen_pfn_t into qw().

---
 tools/libs/light/libxl_dom_save.c       | 19 +++++++++++++++++++
 tools/libs/light/libxl_internal.h       |  2 ++
 tools/libs/light/libxl_save_msgs_gen.pl |  3 ++-
 3 files changed, 23 insertions(+), 1 deletion(-)
tools/libs/light/libxl_dom_save.c +19 -0
index 32e3cb5a13..3f3cff0342 100644
@@ -373,6 +373,24 @@ int libxl__save_emulator_xenstore_data(libxl__domain_save_state *dss,
     return rc;
 }
 
+static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user)
+{
+    libxl__save_helper_state *shs = user;
+    libxl__domain_save_state *dss = shs->caller_state;
+    STATE_AO_GC(dss->ao);
+
+    LOGD(DEBUG, shs->domid, "iteration %u dirty_count %ld total_written %lu",
+         stats.iteration, stats.dirty_count, stats.total_written);
+    if (stats.dirty_count >= 0 && stats.dirty_count < LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT)
+        goto stop_copy;
+    if (stats.iteration >= LIBXL_XGS_POLICY_MAX_ITERATIONS)
+        goto stop_copy;
+    return XGS_POLICY_CONTINUE_PRECOPY;
+
+stop_copy:
+    return XGS_POLICY_STOP_AND_COPY;
+}
+
 /*----- main code for saving, in order of execution -----*/
 
 void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
@@ -430,6 +448,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
         callbacks->suspend = libxl__domain_suspend_callback;
 
     callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
+    callbacks->precopy_policy = libxl__domain_save_precopy_policy;
 
     dss->sws.ao  = dss->ao;
     dss->sws.dss = dss;
tools/libs/light/libxl_internal.h +2 -0
index 439c654733..57d7e4b4b8 100644
@@ -125,6 +125,8 @@
 #define DOMID_XS_PATH "domid"
 #define PVSHIM_BASENAME "xen-shim"
 #define PVSHIM_CMDLINE "pv-shim console=xen,pv"
+#define LIBXL_XGS_POLICY_MAX_ITERATIONS 5
+#define LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT 50
 
 /* Size macros. */
 #define __AC(X,Y)   (X##Y)
tools/libs/light/libxl_save_msgs_gen.pl +2 -1
index f263ee01bb..ab55c81644 100755
@@ -23,6 +23,7 @@ our @msgs = (
                                              STRING doing_what),
                                             'unsigned long', 'done',
                                             'unsigned long', 'total'] ],
+    [ 'scxW',   "precopy_policy", ['precopy_stats_t', 'stats'] ],
     [ 'srcxA',  "suspend", [] ],
     [ 'srcxA',  "postcopy", [] ],
     [ 'srcxA',  "checkpoint", [] ],
@@ -142,7 +143,7 @@ static void bytes_put(unsigned char *const buf, int *len,
 
 END
 
-foreach my $simpletype (qw(int uint16_t uint32_t unsigned), 'unsigned long', 'xen_pfn_t') {
+foreach my $simpletype (qw(int uint16_t uint32_t unsigned precopy_stats_t xen_pfn_t), 'unsigned long') {
     my $typeid = typeid($simpletype);
     $out_body{'callout'} .= <<END;
 static int ${typeid}_get(const unsigned char **msg,
#80 lore raw

PATCH v20210701 35/40 tools: add --max_iters to libxl_domain_suspend

In reply to Olaf Hering

Migrating a large, and potentially busy, domU will take more time than neccessary due to excessive number of copying iterations.

Allow to host admin to control the number of iterations which copy cumulated domU dirty pages to the target host.

The default remains 5, which means one initial iteration to copy the entire domU memory, and up to 4 additional iterations to copy dirty memory from the still running domU. After the given number of iterations the domU is suspended, remaining dirty memory is copied and the domU is finally moved to the target host.

This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility of the new .max_iters property.

---
 docs/man/xl.1.pod.in              |  4 ++++
 tools/include/libxl.h             |  1 +
 tools/libs/light/libxl_dom_save.c |  2 +-
 tools/libs/light/libxl_domain.c   |  1 +
 tools/libs/light/libxl_internal.h |  1 +
 tools/xl/xl_cmdtable.c            |  3 ++-
 tools/xl/xl_migrate.c             | 10 +++++++++-
 7 files changed, 19 insertions(+), 3 deletions(-)
docs/man/xl.1.pod.in +4 -0
index 70a6ebf438..594387bcf4 100644
@@ -494,6 +494,10 @@ such that it will be identical on the destination host, unless that
 configuration is overridden using the B<-C> option. Note that it is not
 possible to use this option for a 'localhost' migration.
 
+=item B<--max_iters> I<iterations>
+
+Number of copy iterations before final suspend+move (default: 5)
+
 =back
 
 =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/include/libxl.h +1 -0
index 9a4d7514ed..bf77da0524 100644
@@ -1714,6 +1714,7 @@ static inline int libxl_retrieve_domain_configuration_0x041200(
 
 typedef struct {
     uint32_t flags; /* LIBXL_SUSPEND_* */
+    uint32_t max_iters;
 } libxl_domain_suspend_props;
 #define LIBXL_SUSPEND_DEBUG 1
 #define LIBXL_SUSPEND_LIVE 2
tools/libs/light/libxl_dom_save.c +1 -1
index 3f3cff0342..938c0127f3 100644
@@ -383,7 +383,7 @@ static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user)
          stats.iteration, stats.dirty_count, stats.total_written);
     if (stats.dirty_count >= 0 && stats.dirty_count < LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT)
         goto stop_copy;
-    if (stats.iteration >= LIBXL_XGS_POLICY_MAX_ITERATIONS)
+    if (stats.iteration >= dss->max_iters)
         goto stop_copy;
     return XGS_POLICY_CONTINUE_PRECOPY;
 
tools/libs/light/libxl_domain.c +1 -0
index 5dbd27900f..9f98cd7f2b 100644
@@ -527,6 +527,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
     dss->domid = domid;
     dss->fd = fd;
     dss->type = type;
+    dss->max_iters = props->max_iters ?: LIBXL_XGS_POLICY_MAX_ITERATIONS;
     dss->live = props->flags & LIBXL_SUSPEND_LIVE;
     dss->debug = props->flags & LIBXL_SUSPEND_DEBUG;
     dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
tools/libs/light/libxl_internal.h +1 -0
index 57d7e4b4b8..8cbcc5282c 100644
@@ -3649,6 +3649,7 @@ struct libxl__domain_save_state {
     int live;
     int debug;
     int checkpointed_stream;
+    uint32_t max_iters;
     const libxl_domain_remus_info *remus;
     /* private */
     int rc;
tools/xl/xl_cmdtable.c +2 -1
index ca1dfa3525..9b6b3c99aa 100644
@@ -174,7 +174,8 @@ const struct cmd_spec cmd_table[] = {
       "                of the domain.\n"
       "--debug         Ignored.\n"
       "-p              Do not unpause domain after migrating it.\n"
-      "-D              Preserve the domain id"
+      "-D              Preserve the domain id\n"
+      "--max_iters N   Number of copy iterations before final stop+move"
     },
     { "restore",
       &main_restore, 0, 1,
tools/xl/xl_migrate.c +9 -1
index 144890924f..af117d4d56 100644
@@ -178,6 +178,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
 
 static void migrate_domain(uint32_t domid, int preserve_domid,
                            const char *rune, int debug,
+                           uint32_t max_iters,
                            const char *override_config_file)
 {
     pid_t child = -1;
@@ -189,6 +190,7 @@ static void migrate_domain(uint32_t domid, int preserve_domid,
     int config_len;
     libxl_domain_suspend_props props = {
         .flags = LIBXL_SUSPEND_LIVE,
+        .max_iters = max_iters,
         };
 
     save_domain_core_begin(domid, preserve_domid, override_config_file,
@@ -542,8 +544,10 @@ int main_migrate(int argc, char **argv)
     char *host;
     int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
     int preserve_domid = 0;
+    uint32_t max_iters = 0;
     static struct option opts[] = {
         {"debug", 0, 0, 0x100},
+        {"max_iters", 1, 0, 0x101},
         {"live", 0, 0, 0x200},
         COMMON_LONG_OPTS
     };
@@ -571,6 +575,9 @@ int main_migrate(int argc, char **argv)
     case 0x100: /* --debug */
         debug = 1;
         break;
+    case 0x101: /* --max_iters */
+        max_iters = atoi(optarg);
+        break;
     case 0x200: /* --live */
         /* ignored for compatibility with xm */
         break;
@@ -605,7 +612,8 @@ int main_migrate(int argc, char **argv)
                   pause_after_migration ? " -p" : "");
     }
 
-    migrate_domain(domid, preserve_domid, rune, debug, config_filename);
+    migrate_domain(domid, preserve_domid, rune, debug,
+                   max_iters, config_filename);
     return EXIT_SUCCESS;
 }
 
#81 lore raw

PATCH v20210701 36/40 tools: add --min_remaining to libxl_domain_suspend

In reply to Olaf Hering

The decision to stop+move a domU to the new host must be based on two factors:
- the available network bandwidth for the migration stream
- the maximum time a workload within a domU can be savely suspended

Both values define how many dirty pages a workload may produce prior the final stop+move.

The default value of 50 pages is much too low with todays network bandwidths. On an idle 1GiB link these 200K will be transferred within ~2ms.

Give the admin a knob to adjust the point when the final stop+move will be done, so he can base this decision on his own needs.

This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility of the new .min_remaining property.

---
 docs/man/xl.1.pod.in              |  8 ++++++++
 tools/include/libxl.h             |  1 +
 tools/libs/light/libxl_dom_save.c |  2 +-
 tools/libs/light/libxl_domain.c   |  1 +
 tools/libs/light/libxl_internal.h |  1 +
 tools/xl/xl_cmdtable.c            | 23 ++++++++++++-----------
 tools/xl/xl_migrate.c             |  9 ++++++++-
 7 files changed, 32 insertions(+), 13 deletions(-)
docs/man/xl.1.pod.in +8 -0
index 594387bcf4..09e866ad87 100644
@@ -498,6 +498,14 @@ possible to use this option for a 'localhost' migration.
 
 Number of copy iterations before final suspend+move (default: 5)
 
+=item B<--min_remaing> I<pages>
+
+Number of remaining dirty pages. If the number of dirty pages drops that
+low, the guest is suspended and the domU will finally be moved to I<host>.
+
+This allows the host admin to control for how long the domU will likely
+be suspended during transit.
+
 =back
 
 =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/include/libxl.h +1 -0
index bf77da0524..28d70b1078 100644
@@ -1715,6 +1715,7 @@ static inline int libxl_retrieve_domain_configuration_0x041200(
 typedef struct {
     uint32_t flags; /* LIBXL_SUSPEND_* */
     uint32_t max_iters;
+    uint32_t min_remaining;
 } libxl_domain_suspend_props;
 #define LIBXL_SUSPEND_DEBUG 1
 #define LIBXL_SUSPEND_LIVE 2
tools/libs/light/libxl_dom_save.c +1 -1
index 938c0127f3..ad5df89b2c 100644
@@ -381,7 +381,7 @@ static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user)
 
     LOGD(DEBUG, shs->domid, "iteration %u dirty_count %ld total_written %lu",
          stats.iteration, stats.dirty_count, stats.total_written);
-    if (stats.dirty_count >= 0 && stats.dirty_count < LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT)
+    if (stats.dirty_count >= 0 && stats.dirty_count < dss->min_remaining)
         goto stop_copy;
     if (stats.iteration >= dss->max_iters)
         goto stop_copy;
tools/libs/light/libxl_domain.c +1 -0
index 9f98cd7f2b..06ca7a7df6 100644
@@ -528,6 +528,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
     dss->fd = fd;
     dss->type = type;
     dss->max_iters = props->max_iters ?: LIBXL_XGS_POLICY_MAX_ITERATIONS;
+    dss->min_remaining = props->min_remaining ?: LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT;
     dss->live = props->flags & LIBXL_SUSPEND_LIVE;
     dss->debug = props->flags & LIBXL_SUSPEND_DEBUG;
     dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
tools/libs/light/libxl_internal.h +1 -0
index 8cbcc5282c..e4bfb34085 100644
@@ -3650,6 +3650,7 @@ struct libxl__domain_save_state {
     int debug;
     int checkpointed_stream;
     uint32_t max_iters;
+    uint32_t min_remaining;
     const libxl_domain_remus_info *remus;
     /* private */
     int rc;
tools/xl/xl_cmdtable.c +12 -11
index 9b6b3c99aa..2cb4980c80 100644
@@ -165,17 +165,18 @@ const struct cmd_spec cmd_table[] = {
       &main_migrate, 0, 1,
       "Migrate a domain to another host",
       "[options] <Domain> <host>",
-      "-h              Print this help.\n"
-      "-C <config>     Send <config> instead of config file from creation.\n"
-      "-s <sshcommand> Use <sshcommand> instead of ssh.  String will be passed\n"
-      "                to sh. If empty, run <host> instead of ssh <host> xl\n"
-      "                migrate-receive [-d -e]\n"
-      "-e              Do not wait in the background (on <host>) for the death\n"
-      "                of the domain.\n"
-      "--debug         Ignored.\n"
-      "-p              Do not unpause domain after migrating it.\n"
-      "-D              Preserve the domain id\n"
-      "--max_iters N   Number of copy iterations before final stop+move"
+      "-h                Print this help.\n"
+      "-C <config>       Send <config> instead of config file from creation.\n"
+      "-s <sshcommand>   Use <sshcommand> instead of ssh.  String will be passed\n"
+      "                  to sh. If empty, run <host> instead of ssh <host> xl\n"
+      "                  migrate-receive [-d -e]\n"
+      "-e                Do not wait in the background (on <host>) for the death\n"
+      "                  of the domain.\n"
+      "--debug           Ignored.\n"
+      "-p                Do not unpause domain after migrating it.\n"
+      "-D                Preserve the domain id\n"
+      "--max_iters N     Number of copy iterations before final stop+move\n"
+      "--min_remaining N Number of remaining dirty pages before final stop+move"
     },
     { "restore",
       &main_restore, 0, 1,
tools/xl/xl_migrate.c +8 -1
index af117d4d56..14feb2b7ec 100644
@@ -179,6 +179,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
 static void migrate_domain(uint32_t domid, int preserve_domid,
                            const char *rune, int debug,
                            uint32_t max_iters,
+                           uint32_t min_remaining,
                            const char *override_config_file)
 {
     pid_t child = -1;
@@ -191,6 +192,7 @@ static void migrate_domain(uint32_t domid, int preserve_domid,
     libxl_domain_suspend_props props = {
         .flags = LIBXL_SUSPEND_LIVE,
         .max_iters = max_iters,
+        .min_remaining = min_remaining,
         };
 
     save_domain_core_begin(domid, preserve_domid, override_config_file,
@@ -545,9 +547,11 @@ int main_migrate(int argc, char **argv)
     int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
     int preserve_domid = 0;
     uint32_t max_iters = 0;
+    uint32_t min_remaining = 0;
     static struct option opts[] = {
         {"debug", 0, 0, 0x100},
         {"max_iters", 1, 0, 0x101},
+        {"min_remaining", 1, 0, 0x102},
         {"live", 0, 0, 0x200},
         COMMON_LONG_OPTS
     };
@@ -578,6 +582,9 @@ int main_migrate(int argc, char **argv)
     case 0x101: /* --max_iters */
         max_iters = atoi(optarg);
         break;
+    case 0x102: /* --min_remaining */
+        min_remaining = atoi(optarg);
+        break;
     case 0x200: /* --live */
         /* ignored for compatibility with xm */
         break;
@@ -613,7 +620,7 @@ int main_migrate(int argc, char **argv)
     }
 
     migrate_domain(domid, preserve_domid, rune, debug,
-                   max_iters, config_filename);
+                   max_iters, min_remaining, config_filename);
     return EXIT_SUCCESS;
 }
 
#82 lore raw

PATCH v20210701 37/40 tools: add --abort_if_busy to libxl_domain_suspend

In reply to Olaf Hering

Provide a knob to the host admin to abort the live migration of a running domU if the downtime during final transit will be too long for the workload within domU.

Adjust error reporting. Add ERROR_MIGRATION_ABORTED to allow callers of libxl_domain_suspend to distinguish between errors and the requested constraint.

Adjust precopy_policy to simplify reporting of remaining dirty pages. The loop in send_memory_live populates ->dirty_count in a different place than ->iteration. Let it proceeed one more time to provide the desired information before leaving the loop.

This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility of the new .abort_if_busy property.

---
 docs/man/xl.1.pod.in                  |  8 +++++++
 tools/include/libxl.h                 |  1 +
 tools/libs/light/libxl_dom_save.c     |  7 ++++++-
 tools/libs/light/libxl_domain.c       |  1 +
 tools/libs/light/libxl_internal.h     |  2 ++
 tools/libs/light/libxl_stream_write.c |  9 +++++++-
 tools/libs/light/libxl_types.idl      |  1 +
 tools/xl/xl_cmdtable.c                |  6 +++++-
 tools/xl/xl_migrate.c                 | 30 ++++++++++++++++++++-------
 9 files changed, 55 insertions(+), 10 deletions(-)
docs/man/xl.1.pod.in +8 -0
index 09e866ad87..37267c9171 100644
@@ -506,6 +506,14 @@ low, the guest is suspended and the domU will finally be moved to I<host>.
 This allows the host admin to control for how long the domU will likely
 be suspended during transit.
 
+=item B<--abort_if_busy>
+
+Abort migration instead of doing final suspend/move/resume if the
+guest produced more than I<min_remaining> dirty pages during th number
+of I<max_iters> iterations.
+This avoids long periods of time where the guest is suspended, which
+may confuse the workload within domU.
+
 =back
 
 =item B<remus> [I<OPTIONS>] I<domain-id> I<host>
tools/include/libxl.h +1 -0
index 28d70b1078..cc056ed627 100644
@@ -1719,6 +1719,7 @@ typedef struct {
 } libxl_domain_suspend_props;
 #define LIBXL_SUSPEND_DEBUG 1
 #define LIBXL_SUSPEND_LIVE 2
+#define LIBXL_SUSPEND_ABORT_IF_BUSY 4
 
 int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
                          libxl_domain_suspend_props *props,
tools/libs/light/libxl_dom_save.c +6 -1
index ad5df89b2c..1999a8997f 100644
@@ -383,11 +383,16 @@ static int libxl__domain_save_precopy_policy(precopy_stats_t stats, void *user)
          stats.iteration, stats.dirty_count, stats.total_written);
     if (stats.dirty_count >= 0 && stats.dirty_count < dss->min_remaining)
         goto stop_copy;
-    if (stats.iteration >= dss->max_iters)
+    if (stats.dirty_count >= 0 && stats.iteration >= dss->max_iters)
         goto stop_copy;
     return XGS_POLICY_CONTINUE_PRECOPY;
 
 stop_copy:
+    if (dss->abort_if_busy)
+    {
+        dss->remaining_dirty_pages = stats.dirty_count;
+        return XGS_POLICY_ABORT;
+    }
     return XGS_POLICY_STOP_AND_COPY;
 }
 
tools/libs/light/libxl_domain.c +1 -0
index 06ca7a7df6..e4740b063e 100644
@@ -529,6 +529,7 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
     dss->type = type;
     dss->max_iters = props->max_iters ?: LIBXL_XGS_POLICY_MAX_ITERATIONS;
     dss->min_remaining = props->min_remaining ?: LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT;
+    dss->abort_if_busy = props->flags & LIBXL_SUSPEND_ABORT_IF_BUSY;
     dss->live = props->flags & LIBXL_SUSPEND_LIVE;
     dss->debug = props->flags & LIBXL_SUSPEND_DEBUG;
     dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
tools/libs/light/libxl_internal.h +2 -0
index e4bfb34085..905d5179ba 100644
@@ -3648,9 +3648,11 @@ struct libxl__domain_save_state {
     libxl_domain_type type;
     int live;
     int debug;
+    int abort_if_busy;
     int checkpointed_stream;
     uint32_t max_iters;
     uint32_t min_remaining;
+    long remaining_dirty_pages;
     const libxl_domain_remus_info *remus;
     /* private */
     int rc;
tools/libs/light/libxl_stream_write.c +8 -1
index 634f3240d1..1ab3943f3e 100644
@@ -344,11 +344,18 @@ void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
         goto err;
 
     if (retval) {
+        if (dss->remaining_dirty_pages) {
+            LOGD(NOTICE, dss->domid, "saving domain: aborted,"
+                 " %ld remaining dirty pages.", dss->remaining_dirty_pages);
+        } else {
         LOGEVD(ERROR, errnoval, dss->domid, "saving domain: %s",
               dss->dsps.guest_responded ?
               "domain responded to suspend request" :
               "domain did not respond to suspend request");
-        if (!dss->dsps.guest_responded)
+        }
+        if (dss->remaining_dirty_pages)
+           rc = ERROR_MIGRATION_ABORTED;
+        else if(!dss->dsps.guest_responded)
             rc = ERROR_GUEST_TIMEDOUT;
         else if (dss->rc)
             rc = dss->rc;
tools/libs/light/libxl_types.idl +1 -0
index f45adddab0..b91769ee10 100644
@@ -76,6 +76,7 @@ libxl_error = Enumeration("error", [
     (-30, "QMP_DEVICE_NOT_ACTIVE"), # a device has failed to be become active
     (-31, "QMP_DEVICE_NOT_FOUND"), # the requested device has not been found
     (-32, "QEMU_API"), # QEMU's replies don't contains expected members
+    (-33, "MIGRATION_ABORTED"),
     ], value_namespace = "")
 
 libxl_domain_type = Enumeration("domain_type", [
tools/xl/xl_cmdtable.c +5 -1
index 2cb4980c80..322a47c2bc 100644
@@ -176,7 +176,11 @@ const struct cmd_spec cmd_table[] = {
       "-p                Do not unpause domain after migrating it.\n"
       "-D                Preserve the domain id\n"
       "--max_iters N     Number of copy iterations before final stop+move\n"
-      "--min_remaining N Number of remaining dirty pages before final stop+move"
+      "--min_remaining N Number of remaining dirty pages before final stop+move\n"
+      "--abort_if_busy   Abort migration instead of doing final stop+move,\n"
+      "                  if the number of dirty pages is higher than <min_remaining>\n"
+      "                  after <max_iters> iterations. Otherwise the amount of memory\n"
+      "                  to be transfered would exceed maximum allowed domU downtime."
     },
     { "restore",
       &main_restore, 0, 1,
tools/xl/xl_migrate.c +23 -7
index 14feb2b7ec..f523746e5b 100644
@@ -177,7 +177,7 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
 }
 
 static void migrate_domain(uint32_t domid, int preserve_domid,
-                           const char *rune, int debug,
+                           const char *rune, int debug, int abort_if_busy,
                            uint32_t max_iters,
                            uint32_t min_remaining,
                            const char *override_config_file)
@@ -213,14 +213,20 @@ static void migrate_domain(uint32_t domid, int preserve_domid,
 
     if (debug)
         props.flags |= LIBXL_SUSPEND_DEBUG;
+    if (abort_if_busy)
+        props.flags |= LIBXL_SUSPEND_ABORT_IF_BUSY;
     rc = libxl_domain_suspend(ctx, domid, send_fd, &props, NULL);
     if (rc) {
         fprintf(stderr, "migration sender: libxl_domain_suspend failed"
                 " (rc=%d)\n", rc);
-        if (rc == ERROR_GUEST_TIMEDOUT)
-            goto failed_suspend;
-        else
-            goto failed_resume;
+        switch (rc) {
+            case ERROR_GUEST_TIMEDOUT:
+                goto failed_suspend;
+            case ERROR_MIGRATION_ABORTED:
+                goto failed_busy;
+            default:
+                goto failed_resume;
+        }
     }
 
     //fprintf(stderr, "migration sender: Transfer complete.\n");
@@ -302,6 +308,12 @@ static void migrate_domain(uint32_t domid, int preserve_domid,
     fprintf(stderr, "Migration failed, failed to suspend at sender.\n");
     exit(EXIT_FAILURE);
 
+ failed_busy:
+    close(send_fd);
+    migration_child_report(recv_fd);
+    fprintf(stderr, "Migration aborted as requested, domain is too busy.\n");
+    exit(EXIT_FAILURE);
+
  failed_resume:
     close(send_fd);
     migration_child_report(recv_fd);
@@ -545,13 +557,14 @@ int main_migrate(int argc, char **argv)
     char *rune = NULL;
     char *host;
     int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
-    int preserve_domid = 0;
+    int preserve_domid = 0, abort_if_busy = 0;
     uint32_t max_iters = 0;
     uint32_t min_remaining = 0;
     static struct option opts[] = {
         {"debug", 0, 0, 0x100},
         {"max_iters", 1, 0, 0x101},
         {"min_remaining", 1, 0, 0x102},
+        {"abort_if_busy", 0, 0, 0x103},
         {"live", 0, 0, 0x200},
         COMMON_LONG_OPTS
     };
@@ -585,6 +598,9 @@ int main_migrate(int argc, char **argv)
     case 0x102: /* --min_remaining */
         min_remaining = atoi(optarg);
         break;
+    case 0x103: /* --abort_if_busy */
+        abort_if_busy = 1;
+        break;
     case 0x200: /* --live */
         /* ignored for compatibility with xm */
         break;
@@ -619,7 +635,7 @@ int main_migrate(int argc, char **argv)
                   pause_after_migration ? " -p" : "");
     }
 
-    migrate_domain(domid, preserve_domid, rune, debug,
+    migrate_domain(domid, preserve_domid, rune, debug, abort_if_busy,
                    max_iters, min_remaining, config_filename);
     return EXIT_SUCCESS;
 }
#83 lore raw

PATCH v20210701 38/40 tools: add API for expandable bitmaps

In reply to Olaf Hering

Since the incoming migration stream lacks info about what the highest pfn will be, some data structures can not be allocated upfront.

Add an API for expandable bitmaps, loosely based on pfn_set_populated.

---
 tools/libs/saverestore/common.c | 39 +++++++++++++++++++
 tools/libs/saverestore/common.h | 67 +++++++++++++++++++++++++++++++++
 2 files changed, 106 insertions(+)
tools/libs/saverestore/common.c +39 -0
index 7da7fa4e2c..e96173eea2 100644
@@ -163,6 +163,45 @@ static void __attribute__((unused)) build_assertions(void)
     BUILD_BUG_ON(sizeof(struct xc_sr_rec_hvm_params)        != 8);
 }
 
+/*
+ * Expand the tracking structures as needed.
+ * To avoid realloc()ing too excessively, the size increased to the nearest
+ * power of two large enough to contain the required number of bits.
+ */
+bool _sr_bitmap_expand(struct sr_bitmap *bm, unsigned long bits)
+{
+    size_t new_max;
+    size_t old_sz, new_sz;
+    void *p;
+
+    if (bits <= bm->bits)
+        return true;
+
+    /* Round up to the nearest power of two larger than bit, less 1. */
+    new_max = bits;
+    new_max |= new_max >> 1;
+    new_max |= new_max >> 2;
+    new_max |= new_max >> 4;
+    new_max |= new_max >> 8;
+    new_max |= new_max >> 16;
+    new_max |= sizeof(unsigned long) > 4 ? new_max >> 32 : 0;
+
+    /* Allocate units of unsigned long */
+    new_max = (new_max + BITS_PER_LONG - 1) & ~(BITS_PER_LONG - 1);
+
+    old_sz = bitmap_size(bm->bits);
+    new_sz = bitmap_size(new_max);
+    p = realloc(bm->p, new_sz);
+    if (!p)
+        return false;
+
+    memset(p + old_sz, 0, new_sz - old_sz);
+    bm->p = p;
+    bm->bits = new_max;
+
+    return true;
+}
+
 /*
  * Local variables:
  * mode: C
tools/libs/saverestore/common.h +67 -0

73 lines not on this page · raw message

#84 lore raw

PATCH v20210701 39/40 tools: use xg_sr_bitmap for populated_pfns

In reply to Olaf Hering

v02:
- remove xg_ prefix from called functions

---
 tools/libs/saverestore/common.h          | 21 +++++++-
 tools/libs/saverestore/restore.c         | 69 ------------------------
 tools/libs/saverestore/restore_x86_hvm.c |  9 ++++
 tools/libs/saverestore/restore_x86_pv.c  |  7 +++
 4 files changed, 35 insertions(+), 71 deletions(-)
tools/libs/saverestore/common.h +19 -2
index e6a269c482..a610483fe7 100644
@@ -403,8 +403,7 @@ struct xc_sr_context
             uint32_t     xenstore_domid,  console_domid;
 
             /* Bitmap of currently populated PFNs during restore. */
-            unsigned long *populated_pfns;
-            xen_pfn_t max_populated_pfn;
+            struct sr_bitmap populated_pfns;
 
             /* Sender has invoked verify mode on the stream. */
             bool verify;
@@ -647,6 +646,24 @@ static inline bool page_type_has_stream_data(uint32_t type)
     }
     return ret;
 }
+
+static inline bool pfn_is_populated(struct xc_sr_context *ctx, xen_pfn_t pfn)
+{
+    return sr_test_bit(pfn, &ctx->restore.populated_pfns);
+}
+
+static inline int pfn_set_populated(struct xc_sr_context *ctx, xen_pfn_t pfn)
+{
+    xc_interface *xch = ctx->xch;
+
+    if ( sr_set_bit(pfn, &ctx->restore.populated_pfns) == false )
+    {
+        PERROR("Failed to realloc populated_pfns bitmap");
+        errno = ENOMEM;
+        return -1;
+    }
+    return 0;
+}
 #endif
 /*
  * Local variables:
tools/libs/saverestore/restore.c +0 -69

87 lines not on this page · raw message

tools/libs/saverestore/restore_x86_hvm.c +9 -0
index bd63bd2818..97e7e0f48c 100644
@@ -136,6 +136,7 @@ static int x86_hvm_localise_page(struct xc_sr_context *ctx,
 static int x86_hvm_setup(struct xc_sr_context *ctx)
 {
     xc_interface *xch = ctx->xch;
+    unsigned long max_pfn;
 
     if ( ctx->restore.guest_type != DHDR_TYPE_X86_HVM )
     {
@@ -161,6 +162,13 @@ static int x86_hvm_setup(struct xc_sr_context *ctx)
     }
 #endif
 
+    max_pfn = max(ctx->restore.p2m_size, ctx->dominfo.max_memkb >> (PAGE_SHIFT-10));
+    if ( !sr_bitmap_expand(&ctx->restore.populated_pfns, max_pfn) )
+    {
+        PERROR("Unable to allocate memory for populated_pfns bitmap");
+        return -1;
+    }
+
     return 0;
 }
 
@@ -241,6 +249,7 @@ static int x86_hvm_stream_complete(struct xc_sr_context *ctx)
 
 static int x86_hvm_cleanup(struct xc_sr_context *ctx)
 {
+    sr_bitmap_free(&ctx->restore.populated_pfns);
     free(ctx->x86.hvm.restore.context.ptr);
 
     free(ctx->x86.restore.cpuid.ptr);
tools/libs/saverestore/restore_x86_pv.c +7 -0
index 96608e5231..c73a3cd99f 100644
@@ -1060,6 +1060,12 @@ static int x86_pv_setup(struct xc_sr_context *ctx)
     if ( rc )
         return rc;
 
+    if ( !sr_bitmap_expand(&ctx->restore.populated_pfns, 32 * 1024 / 4) )
+    {
+        PERROR("Unable to allocate memory for populated_pfns bitmap");
+        return -1;
+    }
+
     ctx->x86.pv.restore.nr_vcpus = ctx->dominfo.max_vcpu_id + 1;
     ctx->x86.pv.restore.vcpus = calloc(sizeof(struct xc_sr_x86_pv_restore_vcpu),
                                        ctx->x86.pv.restore.nr_vcpus);
@@ -1153,6 +1159,7 @@ static int x86_pv_stream_complete(struct xc_sr_context *ctx)
  */
 static int x86_pv_cleanup(struct xc_sr_context *ctx)
 {
+    sr_bitmap_free(&ctx->restore.populated_pfns);
     free(ctx->x86.pv.p2m);
     free(ctx->x86.pv.p2m_pfns);
 
#85 lore raw

PATCH v20210701 40/40 tools/libxc: use superpages during restore of HVM guest

In reply to Olaf Hering

During creating of a HVM domU meminit_hvm() tries to map superpages. After save/restore or migration this mapping is lost, everything is allocated in single pages. This causes a performance degradation after migration.

Add neccessary code to preallocate a superpage for an incoming chunk of pfns. In case a pfn was not populated on the sending side, it must be freed on the receiving side to avoid over-allocation.

The existing code for x86_pv is moved unmodified into its own file.

v02:
- remove xg_ prefix from called functions

---
 tools/libs/guest/xg_dom_x86.c            |   5 -
 tools/libs/guest/xg_private.h            |   5 +
 tools/libs/saverestore/common.c          |   1 -
 tools/libs/saverestore/common.h          |  28 +-
 tools/libs/saverestore/restore.c         |  62 +---
 tools/libs/saverestore/restore_x86_hvm.c | 370 ++++++++++++++++++++++-
 tools/libs/saverestore/restore_x86_pv.c  |  61 +++-
 7 files changed, 455 insertions(+), 77 deletions(-)
tools/libs/guest/xg_dom_x86.c +0 -5
index d2eb89ce01..ec0d18fd60 100644
@@ -44,11 +44,6 @@
 
 #define SUPERPAGE_BATCH_SIZE 512
 
-#define SUPERPAGE_2MB_SHIFT   9
-#define SUPERPAGE_2MB_NR_PFNS (1UL << SUPERPAGE_2MB_SHIFT)
-#define SUPERPAGE_1GB_SHIFT   18
-#define SUPERPAGE_1GB_NR_PFNS (1UL << SUPERPAGE_1GB_SHIFT)
-
 #define X86_CR0_PE 0x01
 #define X86_CR0_ET 0x10
 
tools/libs/guest/xg_private.h +5 -0
index 28441ee13f..b7372e6bd5 100644
@@ -179,4 +179,9 @@ struct xc_cpu_policy {
 };
 #endif /* x86 */
 
+#define SUPERPAGE_2MB_SHIFT   9
+#define SUPERPAGE_2MB_NR_PFNS (1UL << SUPERPAGE_2MB_SHIFT)
+#define SUPERPAGE_1GB_SHIFT   18
+#define SUPERPAGE_1GB_NR_PFNS (1UL << SUPERPAGE_1GB_SHIFT)
+
 #endif /* XG_PRIVATE_H */
tools/libs/saverestore/common.c +0 -1
index e96173eea2..8dbd516b1b 100644
@@ -1,5 +1,4 @@
 #include <assert.h>
-
 #include "common.h"
 
 #include <xen-tools/libs.h>
tools/libs/saverestore/common.h +20 -8
index a610483fe7..3d392f1ac9 100644
@@ -219,6 +219,16 @@ struct xc_sr_restore_ops
      */
     int (*setup)(struct xc_sr_context *ctx);
 
+    /**
+     * Populate PFNs
+     *
+     * Given a set of pfns, obtain memory from Xen to fill the physmap for the
+     * unpopulated subset.
+     */
+    int (*populate_pfns)(struct xc_sr_context *ctx, unsigned count,
+                         const xen_pfn_t *original_pfns, const uint32_t *types);
+
+
     /**
      * Process an individual record from the stream.  The caller shall take
      * care of processing common records (e.g. END, PAGE_DATA).
@@ -366,6 +376,8 @@ struct xc_sr_context
 
             int send_back_fd;
             unsigned long p2m_size;
+            unsigned long max_pages;
+            unsigned long tot_pages;
             xc_hypercall_buffer_t dirty_bitmap_hbuf;
 
             /* From Image Header. */
@@ -503,6 +515,14 @@ struct xc_sr_context
                     {
                         /* HVM context blob. */
                         struct xc_sr_blob context;
+
+                        /* Bitmap of currently allocated PFNs during restore. */
+                        struct sr_bitmap attempted_1g;
+                        struct sr_bitmap attempted_2m;
+                        struct sr_bitmap allocated_pfns;
+                        xen_pfn_t prev_populated_pfn;
+                        xen_pfn_t iteration_tracker_pfn;
+                        unsigned long iteration;
                     } restore;
                 };
             } hvm;
@@ -567,14 +587,6 @@ int read_record_header(struct xc_sr_context *ctx, int fd, struct xc_sr_rhdr *rhd
 int read_record_data(struct xc_sr_context *ctx, int fd, struct xc_sr_rhdr *rhdr,
                      struct xc_sr_record *rec);
 
-/*
- * This would ideally be private in restore.c, but is needed by
- * x86_pv_localise_page() if we receive pagetables frames ahead of the
- * contents of the frames they point at.
- */
-int populate_pfns(struct xc_sr_context *ctx, unsigned int count,
-                  const xen_pfn_t *original_pfns, const uint32_t *types);
-
 /* Handle a STATIC_DATA_END record. */
 int handle_static_data_end(struct xc_sr_context *ctx);
 
tools/libs/saverestore/restore.c +4 -58

80 lines not on this page · raw message

tools/libs/saverestore/restore_x86_hvm.c +366 -4

402 lines not on this page · raw message

tools/libs/saverestore/restore_x86_pv.c +60 -1

79 lines not on this page · raw message