Kemari: VM synchronization mechanism for fault tolerance
Subsystems (2)
cover Kemari: VM synchronization mechanism for fault tolerance
Hi,
This patch series implements Kemari, an open-source virtual machine synchronization mechanism for fault tolerance. It offers a feasible approach to fault tolerance that does not require the use of specific hardware or modification of applications/OS. Kemari aims to keep VMs transparently running in times of hardware failures. It transfers the state of the primary VM to the secondary VM when the primary VM is about to send an event to devices such as storage and networks. Kemari is applicable to 32bit HVM with PV Drivers. For more information, please visit http://www.osrg.net/kemari/
Overview:
Kemari in VMM taps event channel, pauses the guest (not suspend), prepares for transfer, and Kemari in userland transfers the guest. On failover, Kemari on the secondary restores the guest, and the backend drivers in dom0 set up the backend rings from the state of the shared rings in the guest.
[01] xen: add ECS_TAP state to event channel
[02] xen: core Kemari code
[03] xend: change parameter type of xc_{set,get}_hvm_param
[04] xend: Kemari controller interface in libxc
[05] xend: Kemari sender in libxc
[06] xend: Kemari receiver libxc
[07] xend: add Kemari support to python
[08] xend: add dev state "Attached" to python
[09] linux: add XenbusStateAttached to xenbus
[10] linux: XenbusStateAttached handler for blkback
[11] linux: XenbusStateAttached handler for netback
[12] ioemu: use signal to save qemu state for Kemari
[13] ioemu: use shared region with to flip logdirty_bitmap
The patches can be applied to
xen, xend: xen-3.3-testing.hg c/s 18430:19201eebab16 linux: linux-2.6.18-xen.hg c/s 707:e410857fd83c ioemu: qemu-xen-3.3-testing.git c/s d489fcbef71a527076a0f9ab56a71d7a7846d5b3:
We'll also keep working to update the patches to the latest trees. Could you please review the code? Any comments are appreciated.
Thanks,
Yoshi
RFC PATCH 1/13 Kemari: add ECS_TAP state to event channel
This patch implements event channel tapping. If an inter-domain channel is set to ECS_TAP, it calls the function registered when an event is sent to the channel.
--- xen/common/event_channel.c | 150 ++++++++++++++++++++++++++++++++++++++++++++- xen/include/xen/event.h | 14 ++++ xen/include/xen/sched.h | 10 +++ 3 files changed, 173 insertions(+), 1 deletion(-)
xen/include/xen/sched.h +10 -0
@@ -19,6 +19,7 @@#include <xen/xenoprof.h> #include <xen/rcupdate.h> #include <xen/irq.h> +#include <xen/kemari.h> #ifdef CONFIG_COMPAT #include <compat/vcpu.h>
@@ -47,6 +48,7 @@#define ECS_PIRQ 4/* Channel is bound to a physical IRQ line. */ #define ECS_VIRQ 5/* Channel is bound to a virtual IRQ line. */ #define ECS_IPI 6/* Channel is bound to a virtual IPI line. */ +#define ECS_TAP 7/* Channel is bound and tapped. */ u8 state;/* ECS_* */ u8 consumer_is_xen;/* Consumed by Xen or by guest? */ u16 notify_vcpu_id;/* VCPU for local delivery notification */
@@ -61,6 +63,11 @@u16 pirq;/* state == ECS_PIRQ */ u16 virq;/* state == ECS_VIRQ */ } u; +struct { +u8 mode;/* Tap IN, OUT or both. */ +/* Fucntion to call when an event is detected. */ +long (*redirect) (struct evtchn *lchn,struct evtchn *rchn); + } tap;#ifdef FLASK_ENABLE void *ssid;#endif
@@ -249,6 +256,9 @@/* OProfile support. */ struct xenoprof *xenoprof;int32_t time_offset_seconds; + +/* Kemari support. */ +struct kemari *kemari;struct rcu_head rcu;
xen/include/xen/event.h +14 -0
@@ -79,4 +79,18 @@mb ();/* set blocked status /then/ caller does his work */ \ } while ( 0 ) +struct evtchn_bind_tap { +/* IN parameters. */ +domid_t tap_dom; +uint32_t tap_port; +uint8_t mode; +long (*redirect) (struct evtchn *lchn,struct evtchn *rchn); +}; + +void notify_via_xen_evtchn_tap (struct domain *ld,int lport); + +long evtchn_bind_tap (struct evtchn_bind_tap *bind_tap); + +long evtchn_unbind_tap (struct evtchn_bind_tap *bind_tap); +#endif /* __XEN_EVENT_H__ */
xen/common/event_channel.c +149 -1
186 lines not on this page · raw message
RFC PATCH 2/13 Kemari: core kemari code
In reply to Yoshiaki TamuraThis patch implements the core of Kemari. If Kemari is turned on, VM synchronization starts when an event is tapped. First, Kemari pauses the guest VM and locks the grant tables. Then, it extracts dirtied pfn from the bitmap, copies pfns to the shared buffer, and notifies tools in userland via xen event channel in userland to transfer the guest.
--- xen/arch/x86/Makefile | 1 xen/arch/x86/domain.c | 4 xen/arch/x86/domctl.c | 16 xen/arch/x86/kemari/Makefile | 1 xen/arch/x86/kemari/kemari.c | 666 +++++++++++++++++++++++++++++++++++++++++ xen/include/public/domctl.h | 33 ++ xen/include/public/io/xenbus.h | 4 xen/include/public/kemari.h | 97 +++++ xen/include/xen/kemari.h | 75 ++++ 9 files changed, 896 insertions(+), 1 deletion(-)
xen/arch/x86/Makefile +1 -0
@@ -4,6 +4,7 @@ subdir-y += hvm subdir-y += mm subdir-y += oprofile +subdir-y += kemari subdir-$(x86_32) += x86_32 subdir-$(x86_64) += x86_64
xen/arch/x86/kemari/Makefile new file +1 -0
@@ -0,0 +1,1 @@ +obj-y += kemari.o
xen/include/xen/kemari.h new file +75 -0
75 lines not on this page · raw message
xen/arch/x86/kemari/kemari.c new file +666 -0
666 lines not on this page · raw message
xen/include/public/kemari.h new file +97 -0
97 lines not on this page · raw message
xen/arch/x86/domain.c +4 -0
@@ -1821,6 +1821,10 @@/* Free page used by xen oprofile buffer. */ free_xenoprof_pages (d); +/* Turn off Kemari. */ +if ( d->kemari ) +kemari_off (d); +if (is_hvm_domain (d) )hvm_domain_relinquish_resources (d);
xen/arch/x86/domctl.c +16 -0
@@ -20,6 +20,7 @@#include <xen/trace.h> #include <xen/console.h> #include <xen/iocap.h> +#include <xen/kemari.h> #include <xen/paging.h> #include <asm/irq.h> #include <asm/hvm/hvm.h>
@@ -997,6 +998,21 @@ }break ; +case XEN_DOMCTL_kemari_op: + { +struct domain *d =rcu_lock_domain_by_id (domctl->domain); + + ret = -ESRCH ; +if (unlikely (d ==NULL ) ) +break ; + + ret =do_kemari_op (d, &domctl->u.kemari_op); + +copy_to_guest (u_domctl, domctl,1 ); +rcu_unlock_domain (d); + } +break ; +default : ret = -ENOSYS ;break ;
xen/include/public/domctl.h +33 -0
@@ -614,6 +614,38 @@#define XEN_DOMCTL_set_machine_address_size 51#define XEN_DOMCTL_get_machine_address_size 52 +/* Kemari interface */ +#define XEN_DOMCTL_kemari_op 53 + +#define _XEN_KEMARI_OP_enable 0 +#define XEN_KEMARI_OP_enable (1UL<<_XEN_KEMARI_OP_enable) +#define _XEN_KEMARI_OP_off 1 +#define XEN_KEMARI_OP_off (1UL<<_XEN_KEMARI_OP_off) +#define _XEN_KEMARI_OP_attach 2 +#define XEN_KEMARI_OP_attach (1UL<<_XEN_KEMARI_OP_attach) +#define _XEN_KEMARI_OP_detach 3 +#define XEN_KEMARI_OP_detach (1UL<<_XEN_KEMARI_OP_detach) + +struct xen_domctl_kemari_op { +uint32_t cmd; + +union { +struct { +uint32_t port; +uint32_t num_pages; +uint64_t mfn; + } enable;/* XEN_KEMARI_OP_enable */ +struct { +uint32_t port; +uint16_t evtchn_tap_mode; + } attach;/* XEN_KEMARI_OP_attach */ +struct { +uint32_t port; + } detach;/* XEN_KEMARI_OP_detach */ + } u; +}; +typedef struct xen_domctl_kemari_op xen_domctl_kemari_op_t ; +DEFINE_XEN_GUEST_HANDLE (xen_domctl_kemari_op_t);struct xen_domctl {uint32_t cmd;
@@ -654,6 +686,7 @@struct xen_domctl_set_opt_feature set_opt_feature;struct xen_domctl_set_target set_target;struct xen_domctl_subscribe subscribe; +struct xen_domctl_kemari_op kemari_op;#if defined(__i386__) || defined(__x86_64__)struct xen_domctl_cpuid cpuid;#endif
xen/include/public/io/xenbus.h +3 -1
@@ -63,7 +63,9 @@*/ XenbusStateReconfiguring =7 , - XenbusStateReconfigured = 8 + XenbusStateReconfigured =8 , + + XenbusStateAttached = 9 };typedef enum xenbus_state XenbusState ;
RFC PATCH 3/13 Kemari: change parameter type of xc_{set, get}_hvm_param
In reply to Yoshiaki TamuraThis patch changes arguments "value" of xc_set_hvm_param and xc_get_hvm_param from 32bit to 64bit. This is because Xen sets and gets a 64bit variable for those "value"s in xc_set_hvm_param and xc_get_hvm_param, respectively. We need this change because callback irqs got from xc_get_hvm_param and set by xc_set_hvm_param are 64bit.
--- tools/libxc/xc_cpuid_x86.c | 2 +- tools/libxc/xc_domain.c | 4 ++-- tools/libxc/xc_domain_save.c | 8 ++++---- tools/libxc/xc_resume.c | 2 +- tools/python/xen/lowlevel/xc/xc.c | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-)
tools/libxc/xc_cpuid_x86.c +1 -1
@@ -148,7 +148,7 @@ int xc, domid_t domid, const unsigned int *input, unsigned int *regs) { char brand[13]; -unsigned long pae; +unsigned long long pae;int is_pae;xc_get_hvm_param (xc, domid,HVM_PARAM_PAE_ENABLED , &pae);
tools/libxc/xc_domain.c +2 -2
@@ -690,7 +690,7 @@return do_domctl (xc_handle, &domctl); } -int xc_set_hvm_param (int handle,domid_t dom,int param,unsigned long value) +int xc_set_hvm_param (int handle,domid_t dom,int param,unsigned long long value) {DECLARE_HYPERCALL ;xen_hvm_param_t arg;
@@ -709,7 +709,7 @@return rc; } -int xc_get_hvm_param (int handle,domid_t dom,int param,unsigned long *value) +int xc_get_hvm_param (int handle,domid_t dom,int param,unsigned long long *value) {DECLARE_HYPERCALL ;xen_hvm_param_t arg;
tools/libxc/xc_domain_save.c +4 -4
@@ -1400,7 +1400,7 @@ } chunk = { -3, 0 };xc_get_hvm_param (xc_handle, dom,HVM_PARAM_IDENT_PT , - (unsigned long *)&chunk.ident_pt); + &chunk.ident_pt);if ( (chunk.ident_pt !=0 ) &&write_exact (io_fd, &chunk,sizeof (chunk)) )
@@ -1425,11 +1425,11 @@/* Save magic-page locations. */ memset (magic_pfns,0 ,sizeof (magic_pfns));xc_get_hvm_param (xc_handle, dom,HVM_PARAM_IOREQ_PFN , - (unsigned long *)&magic_pfns[0 ]); + &magic_pfns[0 ]);xc_get_hvm_param (xc_handle, dom,HVM_PARAM_BUFIOREQ_PFN , - (unsigned long *)&magic_pfns[1 ]); + &magic_pfns[1 ]);xc_get_hvm_param (xc_handle, dom,HVM_PARAM_STORE_PFN , - (unsigned long *)&magic_pfns[2 ]); + &magic_pfns[2 ]);if (write_exact (io_fd, magic_pfns,sizeof (magic_pfns)) ) {PERROR ("Error when writing to state file (7)" );
tools/libxc/xc_resume.c +1 -1
@@ -27,7 +27,7 @@/* HVM guests without PV drivers do not have a return code to modify. */ if ( info.hvm ) { -unsigned long irq =0 ; +unsigned long long irq =0 ;xc_get_hvm_param (xc_handle, domid,HVM_PARAM_CALLBACK_IRQ , &irq);if ( !irq )return 0 ;
tools/python/xen/lowlevel/xc/xc.c +2 -2
@@ -490,7 +490,7 @@ { uint32_t dom; int param; - unsigned long value; + unsigned long long value; static char *kwd_list[] = { "domid", "param", NULL }; if ( !PyArg_ParseTupleAndKeywords(args, kwds, "ii", kwd_list,
@@ -500,7 +500,7 @@if (xc_get_hvm_param (self->xc_handle, dom, param, &value) !=0 )return pyxc_error_to_exception (); -return PyLong_FromUnsignedLong (value); +return PyLong_FromUnsignedLongLong (value); }
RFC PATCH 4/13 Kemari: Kemari controller interface in libxc
In reply to Yoshiaki TamuraThis patch implements a userland interface against Kemari in the VMM. Userland programs control the VMM part of Kemari through this interface. This patch also modifies header files of libxc.
--- tools/libxc/Makefile | 2 + tools/libxc/xc_dom_kemari.c | 79 ++++++++++++++++++++++++++++++++++++++++++ tools/libxc/xenctrl.h | 13 +++++- tools/libxc/xenguest.h | 45 +++++++++++++++++++++++ tools/libxc/xg_private.h | 1 tools/libxc/xg_save_restore.h | 2 - tools/xcutils/Makefile | 1 7 files changed, 140 insertions(+), 3 deletions(-)
tools/libxc/Makefile +2 -0
@@ -30,6 +30,8 @@ GUEST_SRCS-y := GUEST_SRCS-y += xg_private.c GUEST_SRCS-$(CONFIG_MIGRATE) += xc_domain_restore.c xc_domain_save.c +GUEST_SRCS-$(CONFIG_MIGRATE) += xc_dom_kemari.c xc_dom_kemari_save.c +GUEST_SRCS-$(CONFIG_MIGRATE) += xc_dom_kemari_restore.c GUEST_SRCS-$(CONFIG_HVM) += xc_hvm_build.c vpath %.c ../../xen/common/libelf
tools/libxc/xc_dom_kemari.c new file +79 -0
79 lines not on this page · raw message
tools/libxc/xenctrl.h +2 -2
@@ -1009,8 +1009,8 @@*/ xc_error_handler xc_set_error_handler (xc_error_handler handler); -int xc_set_hvm_param (int handle,domid_t dom,int param,unsigned long value); -int xc_get_hvm_param (int handle,domid_t dom,int param,unsigned long *value); +int xc_set_hvm_param (int handle,domid_t dom,int param,unsigned long long value); +int xc_get_hvm_param(int handle, domid_t dom, int param, unsigned long long
*value);
/* IA64 specific, nvram save */
int xc_ia64_save_to_nvram(int xc_handle, uint32_t dom);
@@ -1150,4 +1150,13 @@
int xc_pm_get_cxstat(int xc_handle, int cpuid, struct xc_cx_stat *cxpt); int xc_pm_reset_cxstat(int xc_handle, int cpuid);
+/* kemari control interface */
+int xc_kemari_control(int xc_handle,
+ uint32_t domid, + uint32_t cmd, + evtchn_port_t *port, + uint32_t *num_pages, + uint64_t *mfn, + uint16_t tap_mode);
+
#endif /* XENCTRL_H */
tools/libxc/xenguest.h +45 -0
51 lines not on this page · raw message
tools/libxc/xg_private.h +1 -0
@@ -17,6 +17,7 @@#include <xen/memory.h> #include <xen/elfnote.h> +#include <xen/kemari.h> #ifndef ELFSIZE #include <limits.h>
tools/libxc/xg_save_restore.h +1 -1
@@ -13,7 +13,7 @@** We process save/restore/migrate in batches of pages; the below ** determines how many pages we (at maximum) deal with in each batch. */ -#define MAX_BATCH_SIZE 1024/* up to 1024 pages (4MB) at a time */ +#define MAX_BATCH_SIZE 262144/* up to 262144 pages (1GB) at a time */ /* When pinning page tables at the end of restore, we also use batching. */ #define MAX_PIN_BATCH 1024
tools/xcutils/Makefile +1 -0
@@ -19,6 +19,7 @@ PROG_DEP = .*.d PROGRAMS = xc_restore xc_save readnotes lsevtchn +PROGRAMS += xc_kemari_save xc_kemari_restore LDLIBS = $(LDFLAGS_libxenctrl) $(LDFLAGS_libxenguest) $(LDFLAGS_libxenstore)
RFC PATCH 5/13 Kemari: Kemari sender
In reply to Yoshiaki TamuraThis patch implements a program to send an HVM guest image communicating with the VMM part of Kemari. After sending the whole guest memory first, it keeps sending the delta of the guest status when specified events are detected.
--- tools/libxc/xc_dom_kemari_save.c | 1124 +++++++++++++++++++++++++++++++++++++++ tools/xcutils/xc_kemari_save.c | 518 +++++++++++++++++ 2 files changed, 1642 insertions(+)
tools/libxc/xc_dom_kemari_save.c new file +1124 -0
@@ -0,0 +1,1124 @@ +/****************************************************************************** +* xc_dom_kemari_save.c +* +* Save the state of a running Linux session. +* +* Copyright (c) 2008 Nippon Telegraph and Telephone Corporation. +* +* This program is free software; you can redistribute it and/or modify it +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. +* +* This program is distributed in the hope it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +* +* You should have received a copy of the GNU General Public License along with +* this program; if not, write to the Free Software Foundation, Inc., 59 Temple +* Place - Suite 330, Boston, MA 02111-1307 USA. +* +* This source code is based on xc_domain_save.c. +* Copied BITS_PER_LONG, BITS_TO_LONGS, BITMAP_SIZE, BITMAP_SHIFT, +* RATE_IS_MAX, test_bit, clear_bit, set_bit, tv_delta, noncached_write, +* initialize_mbit_rate, and ratewrite from xc_domain_save.c +* +* Copyright (c) 2003, K A Fraser. +*/ + +#include <inttypes.h> +#include <time.h> +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> +#include <limits.h> +#include <sys/types.h> +#include <sys/time.h> + +#include "xc_private.h" +#include "xc_dom.h" +#include "xg_private.h" +#include "xg_save_restore.h" + +#include <xen/hvm/params.h> +#include "xc_e820.h" + +#ifdef __MINIOS__ +/* +* Caution: atomicity of following alternative libc functions are broken. +*/ +static ssize_t sendfile (int out_fd,int in_fd,off_t *offset,size_t count) +{ +char buf[1024 ]; +int len, wrote_len =0 ; + +if (offset !=NULL ) { +ERROR ("Sorry sendfile for stubdomain should not have offset" ); + errno =EIO ; +return -1 ; + } + +while (count >0 ) { + len = (count <sizeof (buf))?count:sizeof (buf); + len =read (in_fd, buf, len); +if (len <0 ) +return -1 ; +if (write_exact (out_fd, buf, len)) +return -1 ; + wrote_len += len; + count -= len; + } +return wrote_len; +} + +#define IOV_MAX 1024 +struct iovec { +void *iov_base;/* Base address. */ +size_t iov_len;/* Length. */ +}; +static ssize_t writev (int d,const struct iovec *iov,int iovcnt) +{ +int i; +int len, wrote_len; + +if (iovcnt <0 || iovcnt >IOV_MAX ) { + errno =EINVAL ; +return -1 ; + } + +for (i =0 , wrote_len =0 ; i < iovcnt; i++) { + len =write (d, iov[i].iov_base, iov[i].iov_len); +if (len <0 ) +return -1 ; + + wrote_len += len; +if (wrote_len <0 ) {/* integer overflow */ + errno =EINVAL ; +return -1 ; + } + +if (len != iov[i].iov_len) +return wrote_len; + } + +return wrote_len; +} +#else /* !__MINIOS__ */ +#include <sys/sendfile.h> +#include <sys/uio.h> +#endif /* __MINIOS__ */ + +/* HVM: shared-memory bitmaps for getting log-dirty bits from qemu-dm */ +static unsigned long *qemu_bitmaps[2 ]; +static int qemu_active; +static int qemu_non_active; + +/* number of pfns this guest has (i.e. number of entries in the P2M) */ +static unsigned long p2m_size; + +/* page frame numbers */ +static unsigned long *pfn_type =NULL ; + +/* The new domain's shared-info frame number. */ +static unsigned long shared_info_frame; + +/* +* guest memory +*/ +#define GUEST_MEM_ENTRY_SIZE 1024/* up to 4MB at a time. */ +static unsigned char ** guest_memory =NULL ; +static unsigned long ** guest_memory_status =NULL ; +static unsigned long guest_memory_size =0 ; + +static inline int map_guest_mem (int xc_handle,uint32_t domid, +unsigned long base) +{ +int j; +unsigned char * region_base; +unsigned long * pfn_base; + + pfn_base = guest_memory_status[base]; + +memset (pfn_base,0 ,GUEST_MEM_ENTRY_SIZE ); +for (j =0 ; j <GUEST_MEM_ENTRY_SIZE ; j++) { + pfn_base[j] = base *GUEST_MEM_ENTRY_SIZE + j; + } + region_base =xc_map_foreign_batch ( + xc_handle, domid,PROT_READ , pfn_base,GUEST_MEM_ENTRY_SIZE ); +if ( region_base ==NULL ) + { +PERROR ("map failed at guest memory frame 0x%lx - 0x%lx (%lu)" , + base *GUEST_MEM_ENTRY_SIZE , (base +1 )*GUEST_MEM_ENTRY_SIZE -1 , + base); +return -1 ; + } + +/* Look for and skip completely empty batches. */ +for ( j =0 ; j <GUEST_MEM_ENTRY_SIZE ; j++ ) + pfn_base[j] &=XEN_DOMCTL_PFINFO_LTAB_MASK ; +for ( j =0 ; j <GUEST_MEM_ENTRY_SIZE ; j++ ) +if ( pfn_base[j] !=XEN_DOMCTL_PFINFO_XTAB ) +break ; +if ( j ==GUEST_MEM_ENTRY_SIZE ) + { +munmap (region_base,GUEST_MEM_ENTRY_SIZE *PAGE_SIZE ); + guest_memory[base] =NULL ; +return 1 ; + } + + guest_memory[base] = region_base; + +return 0 ; +} + +static inline unsigned char *search_guest_mem (int xc_handle,uint32_t domid, +unsigned long mfn) +{ +unsigned long base = mfn /GUEST_MEM_ENTRY_SIZE ; +unsigned long offset = mfn %GUEST_MEM_ENTRY_SIZE ; + +if (base >= guest_memory_size) { +ERROR ("Error base(%lu) is greater than guest_memory_size(%lu)\n" , + base, guest_memory_size); +return NULL ; + } + +if ( guest_memory_status[base][offset] ==XEN_DOMCTL_PFINFO_XTAB ) { +/* reload XTAB place */ +munmap (guest_memory[base],GUEST_MEM_ENTRY_SIZE *PAGE_SIZE ); + guest_memory[base] =NULL ; +DPRINTF ("guest_memory[%lu] (frame 0x%lx - 0x%lx) will be remapped\n" , + base, base *GUEST_MEM_ENTRY_SIZE , + (base +1 ) *GUEST_MEM_ENTRY_SIZE -1 ); + } + +if (guest_memory[base] ==NULL ) +if (map_guest_mem (xc_handle, domid, offset)) +return NULL ; + +return guest_memory[base] + offset *PAGE_SIZE ; +/* Since I don't care of XEN_DOMCTL_PFINFO_LTAB_MASK, +this program may cause some accidents. */ +} + +static inline int init_guest_mem (int xc_handle,uint32_t dom) +{ +int i; + + guest_memory_size = p2m_size /GUEST_MEM_ENTRY_SIZE +1 ; +DPRINTF ("guest_memory_size: %lu\n" , guest_memory_size); + +/* mapped memory */ + guest_memory =xg_memalign (PAGE_SIZE , + guest_memory_size *sizeof (guest_memory[0 ])); +if (guest_memory ==NULL ) + { +PERROR ("failed to allocate guest_memory" ); +return -1 ; + } +if (lock_pages (guest_memory, guest_memory_size *sizeof (guest_memory[0 ]))) + { +ERROR ("Unable to lock guest_memory array" ); +return -1 ; + } + +/* memory status */ + guest_memory_status =xg_memalign (PAGE_SIZE , + guest_memory_size *sizeof (guest_memory_status[0 ])); +if ( guest_memory_status ==NULL ) + { +ERROR ("failed to alloc memory for guest_memory_status" ); + errno =ENOMEM ; +return -1 ; + } +if (lock_pages (guest_memory_status, + guest_memory_size *sizeof (guest_memory_status[0 ]))) + { +ERROR ("Unable to lock guest_memory_status array" ); +return -1 ; + } + +for (i =0 ; i < guest_memory_size; i++) { + guest_memory_status[i] =xg_memalign (PAGE_SIZE , +GUEST_MEM_ENTRY_SIZE *sizeof (guest_memory_status[0 ][0 ])); +if (guest_memory_status[i] ==NULL ) { +ERROR ("failed to alloc memory for guest_memory_status[%d]" , i); + errno =ENOMEM ; +return -1 ; + } +if (lock_pages (guest_memory_status, + guest_memory_size *sizeof (guest_memory_status[0 ][0 ]))) + { +ERROR ("Unable to lock guest_memory_status[%d]" , i); +return -1 ; + } + } + +for (i =0 ; i < guest_memory_size; i++) +if (map_guest_mem (xc_handle, dom, i) <0 ) +return -1 ; + +return 0 ; +} + +static int writev_exact (int fd,const struct iovec *iov,size_t count) +{ +int i; +size_t sum; +for (i =0 , sum =0 ; i < count; i++) + sum += iov[i].iov_len; + +if (writev (fd, iov, count) != sum) +return -1 ; +else +return 0 ; +} + +/* grep fodder: machine_to_phys */ + + +/* +** During (live) save/migrate, we maintain a number of bitmaps to track +** which pages we have to send, to fixup, and to skip. +*/ + +#define BITS_PER_LONG (sizeof(unsigned long) * 8) +#define BITS_TO_LONGS (bits) (((bits)+BITS_PER_LONG-1)/BITS_PER_LONG) +#define BITMAP_SIZE (BITS_TO_LONGS(p2m_size) * sizeof(unsigned long)) + +#define BITMAP_ENTRY (_nr,_bmap) \ + ((volatile unsigned long *)(_bmap))[(_nr)/BITS_PER_LONG] + +#define BITMAP_SHIFT (_nr) ((_nr) % BITS_PER_LONG) + +static inline int test_bit (int nr,volatile void * addr) +{ +return (BITMAP_ENTRY (nr, addr) >>BITMAP_SHIFT (nr)) &1 ; +} + +static inline void clear_bit (int nr,volatile void * addr) +{ +BITMAP_ENTRY (nr, addr) &= ~(1UL <<BITMAP_SHIFT (nr)); +} + +static inline void set_bit (int nr,volatile void * addr) +{ +BITMAP_ENTRY (nr, addr) |= (1UL <<BITMAP_SHIFT (nr)); +} + +static uint64_t tv_delta (struct timeval *new,struct timeval *old) +{ +return (((new->tv_sec - old->tv_sec)*1000000 ) + + (new->tv_usec - old->tv_usec)); +} + +static int noncached_write (int fd,void *buffer,int len) +{ +static int write_count =0 ; +int rc = (write_exact (fd, buffer, len) ==0 ) ? len :-1 ; + + write_count += len; +if ( write_count >= (MAX_PAGECACHE_USAGE *PAGE_SIZE ) ) + { +/* Time to discard cache - dont care if this fails */ +discard_file_cache (fd,0 /* no flush */ ); + write_count =0 ; + } + +return rc; +} + +#ifdef ADAPTIVE_SAVE + +/* +** We control the rate at which we transmit (or save) to minimize impact +** on running domains (including the target if we're doing live migrate). +*/ + +#define MAX_MBIT_RATE 500/* maximum transmit rate for migrate */ +#define START_MBIT_RATE 100/* initial transmit rate for migrate */ + +/* Scaling factor to convert between a rate (in Mb/s) and time (in usecs) */ +#define RATE_TO_BTU 781250 + +/* Amount in bytes we allow ourselves to send in a burst */ +#define BURST_BUDGET (100*1024) + +/* We keep track of the current and previous transmission rate */ +static int mbit_rate, ombit_rate =0 ; + +/* Have we reached the maximum transmission rate? */ +#define RATE_IS_MAX () (mbit_rate == MAX_MBIT_RATE) + +static inline void initialize_mbit_rate () +{ + mbit_rate =START_MBIT_RATE ; +} + +static int ratewrite (int io_fd,void *buf,int n) +{ +static int budget =0 ; +static int burst_time_us =-1 ; +static struct timeval last_put = {0 }; +struct timeval now; +struct timespec delay; +long long delta; + +if (START_MBIT_RATE ==0 ) +return noncached_write (io_fd, buf, n); + + budget -= n; +if ( budget <0 ) + { +if ( mbit_rate != ombit_rate ) + { + burst_time_us =RATE_TO_BTU / mbit_rate; + ombit_rate = mbit_rate; +DPRINTF ("rate limit: %d mbit/s burst budget %d slot time %d\n" , + mbit_rate,BURST_BUDGET , burst_time_us); + } +if ( last_put.tv_sec ==0 ) + { + budget +=BURST_BUDGET ; +gettimeofday (&last_put,NULL ); + } +else + { +while ( budget <0 ) + { +gettimeofday (&now,NULL ); + delta =tv_delta (&now, &last_put); +while ( delta > burst_time_us ) + { + budget +=BURST_BUDGET ; + last_put.tv_usec += burst_time_us; +if ( last_put.tv_usec >1000000 ) + { + last_put.tv_usec -=1000000 ; + last_put.tv_sec++; + } + delta -= burst_time_us; + } +if ( budget >0 ) +break ; + delay.tv_sec =0 ; + delay.tv_nsec =1000 * (burst_time_us - delta); +while ( delay.tv_nsec >0 ) +if (nanosleep (&delay, &delay) ==0 ) +break ; + } + } + } +return noncached_write (io_fd, buf, n); +} + +#else /* ! ADAPTIVE SAVE */ + +#define RATE_IS_MAX () (0) +#define ratewrite (_io_fd, _buf, _n) noncached_write((_io_fd), (_buf), (_n)) +#define initialize_mbit_rate () + +#endif + +static int print_stats (int xc_handle,uint32_t domid,int pages_sent, +xc_shadow_op_stats_t *stats,int print) +{ +static struct timeval wall_last; +static long long d0_cpu_last; +static long long d1_cpu_last; + +struct timeval wall_now; +long long wall_delta; +long long d0_cpu_now, d0_cpu_delta; +long long d1_cpu_now, d1_cpu_delta; + +gettimeofday (&wall_now,NULL ); + + d0_cpu_now =xc_domain_get_cpu_usage (xc_handle,0 ,/* FIXME */ 0 )/1000 ; + d1_cpu_now =xc_domain_get_cpu_usage (xc_handle, domid,/* FIXME */ 0 )/1000 ; + +if ( (d0_cpu_now ==-1 ) || (d1_cpu_now ==-1 ) ) +DPRINTF ("ARRHHH!!\n" ); + + wall_delta =tv_delta (&wall_now,&wall_last)/1000 ; +if ( wall_delta ==0 ) + wall_delta =1 ; + + d0_cpu_delta = (d0_cpu_now - d0_cpu_last)/1000 ; + d1_cpu_delta = (d1_cpu_now - d1_cpu_last)/1000 ; + +if ( print ) +DPRINTF ("delta %lldms, dom0 %d%%, target %d%%, sent %dMb/s, " +"dirtied %dMb/s %" PRId32" pages\n" , + wall_delta, + (int )((d0_cpu_delta*100 )/wall_delta), + (int )((d1_cpu_delta*100 )/wall_delta), + (int )((pages_sent*PAGE_SIZE )/(wall_delta*(1000 /8 ))), + (int )((stats->dirty_count*PAGE_SIZE )/(wall_delta*(1000 /8 ))), + stats->dirty_count); + +#ifdef ADAPTIVE_SAVE +if ( ((stats->dirty_count*PAGE_SIZE )/(wall_delta*(1000 /8 ))) > mbit_rate ) + { + mbit_rate = (int )((stats->dirty_count*PAGE_SIZE )/(wall_delta*(1000 /8 ))) + +50 ; +if ( mbit_rate >MAX_MBIT_RATE ) + mbit_rate =MAX_MBIT_RATE ; + } +#endif + + d0_cpu_last = d0_cpu_now; + d1_cpu_last = d1_cpu_now; + wall_last = wall_now; + +return 0 ; +} + + +static int send_ident_pt (int xc_handle,int io_fd,uint32_t dom) +{ +struct { +int minusthree; +uint32_t pad; +uint64_t ident_pt; + } chunk = {-3 ,0 }; + +xc_get_hvm_param (xc_handle, dom,HVM_PARAM_IDENT_PT , + &chunk.ident_pt); + +if ( (chunk.ident_pt !=0 ) && +write_exact (io_fd, &chunk,sizeof (chunk)) ) + { +PERROR ("Error when writing the ident_pt for EPT guest" ); +return -1 ; + } + +return 0 ; +} + +static int send_qemu_image (int xc_handle,int io_fd,uint32_t dom) +{ +char path[128 ]; +struct stat st; +struct { +int minusfour; +uint32_t image_size; + } chunk = {-4 ,0 }; +int qemu_fd; +int rc =-1 ; + +snprintf (path,sizeof (path),"/dev/shm/qemu-save.%d" , dom); +if ((qemu_fd =open (path,O_RDONLY )) ==-1 ) + { +PERROR ("Error when opening qemu image %s" , path); + goto out; + } + +if (fstat (qemu_fd, &st) ==-1 ) + { +PERROR ("Error fstat qemu file %s" , path); + goto out; + } + chunk.image_size = st.st_size; + +if (write_exact (io_fd, &chunk,sizeof (chunk)) ) + { +PERROR ("Error when writing header for qemu image" ); + goto out; + } + +if (sendfile (io_fd, qemu_fd,NULL , chunk.image_size) != + chunk.image_size) + { +PERROR ("Error when writing qemu image" ); + goto out; + } +close (qemu_fd); + + rc =0 ; +out: +return rc; +} + +static int send_hvm_context (int xc_handle,int io_fd, +struct kemari_ring *ring,uint32_t dom) +{ +uint32_t buf_size = ring->hvm_ctxt.buf_size; +uint32_t rec_size = ring->hvm_ctxt.rec_size; +uint8_t *hvm_buf = (uint8_t *)ring + ring->hvm_ctxt.buf_offset; +int rc =-1 ; + +/* Get HVM context from Xen and save it too */ +if ( (rec_size =xc_domain_hvm_getcontext (xc_handle, dom, hvm_buf, + buf_size)) ==-1 ) + { +ERROR ("HVM:Could not get hvm buffer" ); + goto out; + } + +if (write_exact (io_fd, &rec_size,sizeof (uint32_t )) ) + { +PERROR ("error write hvm buffer size" ); + goto out; + } + +if (write_exact (io_fd, hvm_buf, rec_size) ) + { +PERROR ("write HVM info failed!\n" ); + goto out; + } + rc =0 ; + +out: +return rc; +} + +int xc_kemari_save (int xc_handle,int io_fd,uint32_t dom, +void *kemari_ring,uint32_t flags, +int hvm,void *(*init_qemu_maps)(int ,unsigned )) +{ +int rc =1 , i, j, iter =0 ; +int debug = (flags &XCFLAGS_DEBUG ); +int sent_last_iter, skip_this_iter; +xc_dominfo_t info; +struct kemari_ring *ring = (struct kemari_ring *)kemari_ring; + +/* base of the region in which domain memory is mapped */ +unsigned char *region_base =NULL ; + +/* bitmap of pages: +- that should be sent this iteration (unless later marked as skip); +- to skip this iteration because already dirty; +- to fixup by sending at the end if not already resent; */ +unsigned long *to_send =NULL , *to_fix =NULL ; + +xc_shadow_op_stats_t stats; + +unsigned long needed_to_fix =0 ; +unsigned long total_sent =0 ; + +/* HVM: magic frames for ioreqs and xenstore comms. */ +uint64_t magic_pfns[3 ];/* ioreq_pfn, bufioreq_pfn, store_pfn */ + +/* callback irq */ +uint64_t callback_irq =0 ; + +if ( !hvm ) + { +ERROR ("HVM domain is required for the kemari migration." ); +return 1 ; + } + +initialize_mbit_rate (); + +if (xc_domain_getinfo (xc_handle, dom,1 , &info) !=1 ) + { +ERROR ("Could not get domain info" ); +return 1 ; + } + + shared_info_frame = info.shared_info_frame; +DPRINTF ("xc_kemari_save: shared_info_frame: %lu\n" , shared_info_frame); + +/* Get the size of the P2M table */ + p2m_size =xc_memory_op (xc_handle, XENMEM_maximum_gpfn, &dom) +1 ; +DPRINTF ("xc_kemari_save: p2m_size: %lu\n" , p2m_size); + +/* Domain is still running at this point */ + { +/* Get qemu-dm logging dirty pages too */ +void *seg =init_qemu_maps (dom,BITMAP_SIZE ); + qemu_bitmaps[0 ] = seg; + qemu_bitmaps[1 ] = seg +BITMAP_SIZE ; + qemu_active =0 ; + qemu_non_active =1 ; + } + +/* pretend we sent all the pages last iteration */ + sent_last_iter = p2m_size; + +/* Setup to_send / to_fix bitmaps */ + to_send =xg_memalign (PAGE_SIZE ,ROUNDUP (BITMAP_SIZE ,PAGE_SHIFT )); + to_fix =calloc (1 ,BITMAP_SIZE ); + +if ( !to_send || !to_fix ) + { +ERROR ("Couldn't allocate to_send array" ); + goto out; + } + +memset (to_send,0xff ,BITMAP_SIZE ); + +if (lock_pages (to_send,BITMAP_SIZE ) ) + { +ERROR ("Unable to lock to_send" ); +return 1 ; + } + + pfn_type =xg_memalign (PAGE_SIZE ,ROUNDUP ( +MAX_BATCH_SIZE *sizeof (*pfn_type),PAGE_SHIFT )); +if ( pfn_type ==NULL ) + { +ERROR ("failed to alloc memory for pfn_type arrays" ); + errno =ENOMEM ; + goto out; + } +memset (pfn_type,0 , +ROUNDUP (MAX_BATCH_SIZE *sizeof (*pfn_type),PAGE_SHIFT )); + +if (lock_pages (pfn_type,MAX_BATCH_SIZE *sizeof (*pfn_type)) ) + { +ERROR ("Unable to lock pfn_type array" ); + goto out; + } + +/* Start writing out the saved-domain record. */ +if (write_exact (io_fd, &p2m_size,sizeof (unsigned long )) ) + { +PERROR ("write: p2m_size" ); + goto out; + } + +/* send shared_info_frame */ +if (write_exact (io_fd, &shared_info_frame,sizeof (unsigned long )) ) + { +PERROR ("write: shared_info_frame" ); + goto out; + } + +/* Save magic-page locations. */ +memset (magic_pfns,0 ,sizeof (magic_pfns)); +xc_get_hvm_param (xc_handle, dom,HVM_PARAM_IOREQ_PFN , + &magic_pfns[0 ]); +xc_get_hvm_param (xc_handle, dom,HVM_PARAM_BUFIOREQ_PFN , + &magic_pfns[1 ]); +xc_get_hvm_param (xc_handle, dom,HVM_PARAM_STORE_PFN , + &magic_pfns[2 ]); +DPRINTF ("kemari_restore: magic_pfns 0: %lld, 1: %lld, 2: %lld\n" , + magic_pfns[0 ], magic_pfns[1 ], magic_pfns[2 ]); +if (write_exact (io_fd, magic_pfns,sizeof (magic_pfns)) ) + { +PERROR ("Error when writing to state file (7)" ); + goto out; + } + +xc_get_hvm_param (xc_handle, dom,HVM_PARAM_CALLBACK_IRQ , + &callback_irq); +DPRINTF ("kemari_restore: callback irq %llx" , callback_irq); +if (write_exact (io_fd, &callback_irq,sizeof (callback_irq)) ) + { +PERROR ("Error when writing to state file (8)" ); + goto out; + } + +print_stats (xc_handle, dom,0 , &stats,0 ); + +/* Now write out each data page, canonicalising page tables as we go... */ + { +unsigned int prev_pc, sent_this_iter,N , batch, run; + + iter++; + sent_this_iter =0 ; + skip_this_iter =0 ; + prev_pc =0 ; +N =0 ; + +DPRINTF ("Saving memory pages: iter %d 0%%" , iter); + +while (N < p2m_size ) + { +unsigned int this_pc = (N *100 ) / p2m_size; + +if ( (this_pc - prev_pc) >=5 ) + { +DPRINTF ("\b\b\b\b%3d%%" , this_pc); + prev_pc = this_pc; + } + +/* load pfn_type[] with the mfn of all the pages we're doing in +this batch. */ +for ( batch =0 ; + (batch <MAX_BATCH_SIZE ) && (N < p2m_size); +N ++ ) + { +int n =N ; + +if ( debug ) + { +DPRINTF ("%d pfn= %08lx mfn= %08lx %d" , + iter, (unsigned long )n, + (long unsigned int )0 , +test_bit (n, to_send)); +DPRINTF ("\n" ); + } + +if ( !( (test_bit (n, to_send)) || (test_bit (n, to_fix))) ) +continue ; + +/* Skip PFNs that aren't really there */ +if (((n >=0xa0 && n <0xc0 )/* VGA hole */ + || (n >= (HVM_BELOW_4G_MMIO_START >>PAGE_SHIFT ) + && n < (1ULL <<32 ) >>PAGE_SHIFT ))/* MMIO */ ) { +if (n >= shared_info_frame && n <= shared_info_frame +32 ) { +/* DPRINTF("shared_info_frame or grant: %d\n", n); */ + }else { +continue ; + } + } + +/* +** we get here if: +** 1. page is marked to_send & hasn't already been re-dirtied +** 2. add in pages that still need fixup (net bufs) +*/ + +/* Hypercall interfaces operate in PFNs for HVM guests +* and MFNs for PV guests */ + pfn_type[batch] = n; + +if ( !is_mapped (pfn_type[batch]) ) + { +/* +** not currently in psuedo-physical map -- set bit +** in to_fix since we must send this page in last_iter +** unless its sent sooner anyhow, or it never enters +** pseudo-physical map (e.g. for ballooned down doms) +*/ +set_bit (n, to_fix); +continue ; + } + +if (test_bit (n, to_fix) && + !test_bit (n, to_send) ) + { + needed_to_fix++; +DPRINTF ("Fix! iter %d, pfn %x. mfn %lx\n" , + iter, n, pfn_type[batch]); + } + +clear_bit (n, to_fix); + + batch++; + } + +if ( batch ==0 ) + goto skip;/* vanishingly unlikely... */ + + region_base =xc_map_foreign_batch ( + xc_handle, dom,PROT_READ , pfn_type, batch); +if ( region_base ==NULL ) + { +ERROR ("map batch failed" ); + goto out; + } + + { +/* Look for and skip completely empty batches. */ +for ( j =0 ; j < batch; j++ ) +if ( (pfn_type[j] &XEN_DOMCTL_PFINFO_LTAB_MASK ) != +XEN_DOMCTL_PFINFO_XTAB ) +break ; +if ( j == batch ) + { +munmap (region_base, batch*PAGE_SIZE ); +continue ;/* bail on this batch: no valid pages */ + } + } + +if (write_exact (io_fd, &batch,sizeof (unsigned int )) ) + { +PERROR ("Error when writing to state file (2)" ); + goto out; + } + +if (write_exact (io_fd, pfn_type,sizeof (unsigned long )*batch) ) + { +PERROR ("Error when writing to state file (3)" ); + goto out; + } + +/* entering this loop, pfn_type is now in pfns (Not mfns) */ + run =0 ; +for ( j =0 ; j < batch; j++ ) + { +unsigned long pfn, pagetype; + + pfn = pfn_type[j] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = pfn_type[j] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( pagetype !=0 ) + { +/* If the page is not a normal data page, write out any +run of pages we may have previously acumulated */ +if ( run ) + { +if (ratewrite (io_fd, + (char *)region_base+(PAGE_SIZE *(j-run)), +PAGE_SIZE *run) !=PAGE_SIZE *run ) + { +ERROR ("Error when writing to state file (4a)" +" (errno %d)" , errno); + goto out; + } + run =0 ; + } + } + +/* skip pages that aren't present */ +if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) +continue ; + + pagetype &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ; + +if ( (pagetype >=XEN_DOMCTL_PFINFO_L1TAB ) && + (pagetype <=XEN_DOMCTL_PFINFO_L4TAB ) ) + { +DPRINTF ("canonicalize_pagetable pagetype = %lx pfn = %lu\n" , pagetype, pfn); + } +else + { +/* We have a normal page: accumulate it for writing. */ + run++; + } + }/* end of the write out for this batch */ + +if ( run ) + { +/* write out the last accumulated run of pages */ +if (ratewrite (io_fd, + (char *)region_base+(PAGE_SIZE *(j-run)), +PAGE_SIZE *run) !=PAGE_SIZE *run ) + { +ERROR ("Error when writing to state file (4c)" +" (errno %d)" , errno); + goto out; + } + } + + sent_this_iter += batch; + +munmap (region_base, batch*PAGE_SIZE ); + + }/* end of this while loop for this iteration */ + + skip: + + total_sent += sent_this_iter; + +DPRINTF ("\r %d: sent %d, skipped %d, " , + iter, sent_this_iter, skip_this_iter ); + + { +print_stats ( xc_handle, dom, sent_this_iter, &stats,1 ); + +DPRINTF ("Total pages sent= %ld (%.2fx)\n" , + total_sent, ((float )total_sent)/p2m_size ); +DPRINTF ("(of which %ld were fixups)\n" , needed_to_fix ); + } + }/* end of infinite for loop */ + +DPRINTF ("All memory is saved\n" ); + +if (send_ident_pt (xc_handle, io_fd, dom) <0 ) + goto out; + +/* Zero terminate */ + i =0 ; +if (write_exact (io_fd, &i,sizeof (int )) ) + { +PERROR ("Error when writing to state file (6')" ); + goto out; + } + +if (send_hvm_context (xc_handle, io_fd, ring, dom) <0 ) + goto out; + +if (!debug) + { +int rcv_status; +if (read_exact (io_fd, &rcv_status,sizeof (int ))) { +ERROR ("Error when reading receiver status" ); + goto out; + } +DPRINTF ("status received: %d\n" , rcv_status); + } + +if (init_guest_mem (xc_handle, dom) <0 ) + goto out; + +/* HVM guests are done now */ + rc =0 ; + + out: + +/* Flush last write and discard cache for file. */ +discard_file_cache (io_fd,1 /* flush */ ); + +free (to_send); +free (to_fix); + +DPRINTF ("Save exit rc=%d\n" ,rc); + +return !!rc; +} + + +int xc_kemari_update (int xc_handle,int io_fd,uint32_t dom, +void *kemari_ring,uint32_t flags, +void (*qemu_save_image)(int ), +void (*qemu_end_flip)(void ), +void (*qemu_end_save)(void ), +void (*qemu_image_sent)(void )) +{ +int rc =1 , k; +int debug = (flags &XCFLAGS_DEBUG ); +uint32_t i, j, index =0 ; +unsigned int batch =0 ; +struct kemari_ring *ring = (struct kemari_ring *)kemari_ring; +struct kemari_ent *buf; +struct iovec iov[MAX_BATCH_SIZE +2 ];/* 2 for batch and pfn_type */ +int iovcnt =2 ; + +#define ADD_IOV (base, len) do { \ + iov[iovcnt].iov_base = base; \ + iov[iovcnt].iov_len = len; \ + iovcnt++; \ +} while (0) + + + +/* flip active qemu */ + qemu_active = qemu_non_active; + qemu_non_active = qemu_active ?0 :1 ; +qemu_save_image (qemu_active); + +/* +* main iteration starts from here +*/ +while (ring->cons < ring->prod) { + +kemari_ring_read (ring, &buf); + +for (i = buf->u.index.start, j = buf->u.index.end; i < j; i++) { + +int next, offset =0 ; + + index = i *BITS_PER_LONG ; + +kemari_ring_read (ring, &buf); + +while (buf->u.dirty_bitmap && offset <BITS_PER_LONG ) { +int n; + next =ffs (buf->u.dirty_bitmap); + buf->u.dirty_bitmap >>= next; + offset += next; + n = offset + index -1 ; +if (((n >=0xa0 && n <0xc0 )/* VGA hole */ + || (n >= (HVM_BELOW_4G_MMIO_START >>PAGE_SHIFT ) + && n < (1ULL <<32 ) >>PAGE_SHIFT ))/* MMIO */ ) { +if (n >= shared_info_frame && n <= shared_info_frame +32 ) { + ; + }else { +continue ; + } + } +ADD_IOV (search_guest_mem (xc_handle, dom, n),PAGE_SIZE ); + pfn_type[batch] = n; + batch++; + } + +if ((batch +BITS_PER_LONG -1 <MAX_BATCH_SIZE ) && + !(ring->cons == ring->prod)) +continue ; + +/* Pull in the dirty bits from qemu-dm too */ +qemu_end_flip (); +for ( k =0 ; k <BITMAP_SIZE /BITS_PER_LONG ; k++) { +if (qemu_bitmaps[qemu_non_active][k] !=0 ) { +unsigned int bmp = qemu_bitmaps[qemu_non_active][k]; + + index = k *BITS_PER_LONG ; +while (bmp && offset <BITS_PER_LONG ) { +int n, next, offset =0 ; + next =ffs (bmp); + bmp >>= next; + offset += next; + n = offset + index -1 ; + +ADD_IOV (search_guest_mem (xc_handle, dom, n),PAGE_SIZE ); + pfn_type[batch] = n; + batch++; + } + qemu_bitmaps[qemu_non_active][k] =0 ; + } +if (batch >=MAX_BATCH_SIZE ) { +ERROR ("Sorry, reached MAX_BATCH_SIZE. " +"We will fix this lator." ); + goto out; + } + } + +PPRINTF ("batch %d\n" , batch); + +/* send pages */ + iov[0 ].iov_base = &batch; + iov[0 ].iov_len =sizeof (batch); + + iov[1 ].iov_base = pfn_type; + iov[1 ].iov_len =sizeof (pfn_type[0 ]) * batch; + +for (k =0 ; k < iovcnt /IOV_MAX +1 ; k++) { +int count = (iovcnt<IOV_MAX *(k+1 ))?(iovcnt-IOV_MAX *k):IOV_MAX ; +if (writev_exact (io_fd, &iov[IOV_MAX * k], count)) { +ERROR ("Error when writing pages state file (2--4)" +" (errno %d)" , errno); + goto out; + } + } + + batch =0 ; + } + } + +if (send_ident_pt (xc_handle, io_fd, dom) <0 ) + goto out; +qemu_end_save (); +if (!debug &&send_qemu_image (xc_handle, io_fd, dom) <0 ) + goto out; +qemu_image_sent (); + +/* Zero terminate */ + i =0 ; +if (write_exact (io_fd, &i,sizeof (int )) ) + { +PERROR ("Error when writing to state file (6')" ); + goto out; + } + +if (send_hvm_context (xc_handle, io_fd, ring, dom) <0 ) + goto out; + +if (!debug) + { +int rcv_status; +if (read_exact (io_fd, &rcv_status,sizeof (int ))) { +ERROR ("Error when reading receiver status" ); + goto out; + } + } + + rc =0 ; +out: + +return rc; +} + +/* +* Local variables: +* mode: C +* c-set-style: "BSD" +* c-basic-offset: 4 +* tab-width: 4 +* indent-tabs-mode: nil +* End: +*/
tools/xcutils/xc_kemari_save.c new file +518 -0
518 lines not on this page · raw message
RFC PATCH 6/13 Kemari: Kemari receiver
In reply to Yoshiaki TamuraThis patch implements a program to receive an HVM guest image in Kemari specific way. After receiving the whole guest memory first, it keeps receiving delta of guest status. Then, after the failover, it sets up the guest image including QEMU status as xc_restore does.
--- tools/libxc/xc_dom_kemari_restore.c | 709 ++++++++++++++++++++++++++++++++++++ tools/xcutils/xc_kemari_restore.c | 88 ++++ 2 files changed, 797 insertions(+)
tools/libxc/xc_dom_kemari_restore.c new file +437 -0
@@ -0,0 +1,709 @@ +/****************************************************************************** +* xc_dom_kemari_restore.c +* +* Restore the state of a guest session for kemari. +* +* Copyright (c) 2008 Nippon Telegraph and Telephone Corporation. +* +* This program is free software; you can redistribute it and/or modify it +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. +* +* This program is distributed in the hope it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +* +* You should have received a copy of the GNU General Public License along with +* this program; if not, write to the Free Software Foundation, Inc., 59 Temple +* Place - Suite 330, Boston, MA 02111-1307 USA. +* +* This source code is based on xc_domain_restore.c. +* +* Copyright (c) 2003, K A Fraser. +* Copyright (c) 2006, Intel Corporation +* Copyright (c) 2007, XenSource Inc. +*/ + +#include <stdlib.h> +#include <unistd.h> + +#include "xg_private.h" +#include "xg_save_restore.h" +#include "xc_dom.h" + +#include <xen/hvm/ioreq.h> +#include <xen/hvm/params.h> + +/* number of pfns this guest has (i.e. number of entries in the P2M) */ +static unsigned long p2m_size; + +/* number of 'in use' pfns in the guest (i.e. #P2M entries with a valid mfn) */ +static unsigned long nr_pfns; + +/* A table mapping each PFN to its new MFN. */ +static xen_pfn_t *p2m =NULL ; + +/* A table of P2M mappings in the current region */ +static xen_pfn_t *p2m_batch =NULL ; + +int xc_kemari_restore (int xc_handle,int io_fd,uint32_t dom, +unsigned int store_evtchn,unsigned long *store_mfn, +unsigned int console_evtchn,unsigned long *console_mfn, +unsigned int hvm,unsigned int pae) +{ +int rc =1 , frc, i, n, m; +unsigned long mfn, pfn; +unsigned int prev_pc, this_pc; + +/* The new domain's shared-info frame number. */ +unsigned long shared_info_frame; + +/* A table containing the type of each PFN (/not/ MFN!). */ +unsigned long *pfn_type =NULL ; + +/* A table of MFNs to map in the current region */ +xen_pfn_t *region_mfn =NULL ; + +/* Types of the pfns in the current region */ +unsigned long region_pfn_type[MAX_BATCH_SIZE ]; + +/* Our mapping of the current region (batch) */ +char *region_base; + +/* Magic frames in HVM guests: ioreqs and xenstore comms. */ +uint64_t magic_pfns[3 ];/* ioreq_pfn, bufioreq_pfn, store_pfn */ + +/* Temporary buffered memory space until all pages are read. */ +char *tmp_region =NULL ; + +/* if true, go into transaction mode */ +int kemari_transaction_mode =0 ; + +/* index for grant table */ +int grant_idx =0 ; + +/* Callback IRQ */ +uint64_t callback_irq =0 ; + +/* active and non-active id of flip buffer */ +int info_active =0 , info_non_active =1 ; + +/* Buffer for holding HVM context */ +uint8_t *hvm_buf[2 ] = {NULL ,NULL }; +uint32_t hvm_buf_size =0 ; + +/* Buffer for qemu image */ +uint8_t *qemu_image[2 ] = {NULL ,NULL }; +uint32_t qemu_image_size[2 ] = {0 ,0 }; +uint32_t qemu_buff_size =0 ; + +/* Buffer for the EPT identity PT location. */ +uint64_t ident_pt[2 ] = {0 ,0 }; + + +if ( !hvm ) { +ERROR ("Kemari only works on HVM domain." ); + goto out; + } + +/* For info only */ + nr_pfns =0 ; + +if (read_exact (io_fd, &p2m_size,sizeof (unsigned long )) ) + { +ERROR ("read: p2m_size" ); + goto out; + } +DPRINTF ("xc_kemari_restore start: p2m_size = %lx\n" , p2m_size); + +/* We want zeroed memory so use calloc rather than malloc. */ + p2m =calloc (p2m_size,sizeof (xen_pfn_t)); + pfn_type =calloc (p2m_size,sizeof (unsigned long )); + + region_mfn =xg_memalign (PAGE_SIZE ,ROUNDUP ( +MAX_BATCH_SIZE *sizeof (xen_pfn_t),PAGE_SHIFT )); + p2m_batch =xg_memalign (PAGE_SIZE ,ROUNDUP ( +MAX_BATCH_SIZE *sizeof (xen_pfn_t),PAGE_SHIFT )); + +/* use aligned page for speed up memmove(3) */ + tmp_region =xg_memalign (PAGE_SIZE ,PAGE_SIZE *MAX_BATCH_SIZE ); + +if ( (p2m ==NULL ) || (pfn_type ==NULL ) || + (region_mfn ==NULL ) || (p2m_batch ==NULL ) || + (tmp_region ==NULL ) ) + { +ERROR ("memory alloc failed" ); + errno =ENOMEM ; + goto out; + } + +memset (region_mfn,0 , +ROUNDUP (MAX_BATCH_SIZE *sizeof (xen_pfn_t),PAGE_SHIFT )); +memset (p2m_batch,0 , +ROUNDUP (MAX_BATCH_SIZE *sizeof (xen_pfn_t),PAGE_SHIFT )); +memset (tmp_region,0 ,PAGE_SIZE *MAX_BATCH_SIZE ); + +if (lock_pages (region_mfn,sizeof (xen_pfn_t) *MAX_BATCH_SIZE ) ) + { +ERROR ("Could not lock region_mfn" ); + goto out; + } + +if (lock_pages (p2m_batch,sizeof (xen_pfn_t) *MAX_BATCH_SIZE ) ) + { +ERROR ("Could not lock p2m_batch" ); + goto out; + } + +if (lock_pages (tmp_region,sizeof (xen_pfn_t) *MAX_BATCH_SIZE ) ) + { +ERROR ("Could not lock region_mfn" ); + goto out; + } + +/* Get the domain's shared-info frame. */ +if (read_exact (io_fd, &shared_info_frame,sizeof (unsigned long ))) + { +ERROR ("Error when reading shared_info_frame" ); + goto out; + } +DPRINTF ("xc_kemari_restore: shared_info_frame: %lx\n" , shared_info_frame); + +/* read HVM-specific parameters */ +if (read_exact (io_fd, magic_pfns,sizeof (magic_pfns)) ) + { +ERROR ("error reading magic page addresses" ); + goto out; + } + +if (read_exact (io_fd, &callback_irq,sizeof (callback_irq))) + { +ERROR ("error reading magic page addresses" ); + goto out; + } + +/* Mark all PFNs as invalid; we allocate on demand */ +for ( pfn =0 ; pfn < p2m_size; pfn++ ) + p2m[pfn] =INVALID_P2M_ENTRY ; + +/* +* Now simply read each saved frame into its new machine frame. +* We uncanonicalise page tables as we go. +*/ + prev_pc =0 ; + + n = m =0 ; +for ( ; ; ) + { +int num_pages; +int nr_mfns; + + num_pages =0 ; +for ( ; ; ) { +int j; + + this_pc = (n *100 ) / p2m_size; +if ( (this_pc - prev_pc) >=5 ) + { +PPRINTF ("\b\b\b\b%3d%%" , this_pc); + prev_pc = this_pc; + } + +if (read_exact (io_fd, &j,sizeof (int )) ) + { +ERROR ("Error when reading batch size" ); + goto build; + } + +PPRINTF ("batch %d\n" ,j); + +if ( j ==-3 ) + { +/* Skip padding 4 bytes then read the EPT identity PT location. */ +if (read_exact (io_fd, &ident_pt[info_non_active], +sizeof (uint32_t )) || +read_exact (io_fd, &ident_pt[info_non_active], +sizeof (uint64_t )) ) + { +ERROR ("error read the address of the EPT identity map" ); + goto build; + } + +continue ; + } + +if (j ==-4 ) + { +uint32_t rec_size; +if (read_exact (io_fd, &rec_size,sizeof (uint32_t )) ) + { +ERROR ("error read the qemu file size" ); + goto build; + } + +if (qemu_buff_size < rec_size) + { + qemu_buff_size = rec_size; + qemu_image[0 ] =realloc (qemu_image[0 ], qemu_buff_size); + qemu_image[1 ] =realloc (qemu_image[1 ], qemu_buff_size); +if ((qemu_image[0 ] ==NULL ) || (qemu_image[1 ] ==NULL )) + { +ERROR ("error allocate memory" ); + goto out; + } + } + + qemu_image_size[info_non_active] = rec_size; +if (read_exact (io_fd, qemu_image[info_non_active], + qemu_image_size[info_non_active]) ) + { +ERROR ("error read the qemu image file" ); + goto build; + } + +continue ; + } + +if ( j ==0 ) +break ;/* our work here is done */ + +/* j > 0: Read pages here */ +if ( (j >MAX_BATCH_SIZE ) || (j <0 ) ) + { +ERROR ("Max batch size exceeded. Giving up. %d" , j); + goto out; + } + +if (read_exact (io_fd, region_pfn_type, j*sizeof (unsigned long )) ) + { +ERROR ("Error when reading region pfn types" ); + goto build; + } + +if (kemari_transaction_mode) { +if (num_pages !=0 ) + { +ERROR ("Sorry! You cannot execute page-send-phase " +"twice. We will fix this bug in the future." ); +DPRINTF ("Sorry\n" ); + goto out; + } + num_pages = j; + +/* Since there are not invalid pages, we don't need to skip */ +if (read_exact (io_fd, tmp_region,PAGE_SIZE * num_pages) ) + { +ERROR ("Error when reading page at kemari transaction mode" ); + goto build; + } + +continue ; + } + +/* Normal mode */ +/* First pass for this batch: work out how much memory to alloc */ + nr_mfns =0 ; +for ( i =0 ; i < j; i++ ) + { +unsigned long pfn, pagetype; + pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( (pagetype !=XEN_DOMCTL_PFINFO_XTAB ) && + (p2m[pfn] ==INVALID_P2M_ENTRY ) ) + { +/* Have a live PFN which hasn't had an MFN allocated */ + p2m_batch[nr_mfns++] = pfn; + p2m[pfn]--; + } + } + +/* Now allocate a bunch of mfns for this batch */ +if ( nr_mfns && + (xc_domain_memory_populate_physmap (xc_handle, dom, nr_mfns,0 , +0 , p2m_batch) !=0 ) ) + { +ERROR ("Failed to allocate memory for batch.! %d\n" , nr_mfns); +for (i =0 ; i < nr_mfns; i++) +DPRINTF ("p2m_batch[%d] = %lx\n" , i, p2m_batch[i]); + errno =ENOMEM ; + goto out; + } + +/* set special pages */ + { +struct xen_add_to_physmap xatp; +for (i =0 ; i < nr_mfns; i++) +if (p2m_batch[i] == shared_info_frame) { + xatp.domid = dom; + xatp.space = XENMAPSPACE_shared_info; + xatp.idx =0 ; + xatp.gpfn = shared_info_frame; +DPRINTF ("setting up shared_info_frame: %lu\n" , + shared_info_frame); +if (xc_memory_op (xc_handle, XENMEM_add_to_physmap, &xatp) + !=0 ) + { +ERROR ("Error setting shared_info_frame" ); + goto out; + } + }else if ((p2m_batch[i] > shared_info_frame) + && (p2m_batch[i] <= shared_info_frame +32 )) { + xatp.domid = dom; + xatp.space = XENMAPSPACE_grant_table; + xatp.idx = grant_idx; + xatp.gpfn = p2m_batch[i]; +DPRINTF ("grant[%d]: %lu\n" , grant_idx, xatp.gpfn); +if (xc_memory_op (xc_handle, XENMEM_add_to_physmap, + &xatp) !=0 ) + { +PERROR ("Cannot map grant table pfn: %lu" , xatp.gpfn); + goto out; + } + grant_idx++; + } + } + +/* Second pass for this batch: update p2m[] and region_mfn[] */ + nr_mfns =0 ; +for ( i =0 ; i < j; i++ ) + { +unsigned long pfn, pagetype; + pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) + region_mfn[i] = ~0UL ;/* map will fail but we don't care */ +else + { +if ( p2m[pfn] == (INVALID_P2M_ENTRY -1 ) ) + { +/* We just allocated a new mfn above; update p2m */ + p2m[pfn] = p2m_batch[nr_mfns++]; + nr_pfns++; + } + +/* setup region_mfn[] for batch map. +* For HVM guests, this interface takes PFNs, not MFNs */ + region_mfn[i] = pfn; + } + } + +/* Map relevant mfns */ + region_base =xc_map_foreign_batch ( + xc_handle, dom,PROT_WRITE , region_mfn, j); + +if ( region_base ==NULL ) + { +ERROR ("map batch failed" ); + goto out; + } + +for ( i =0 ; i < j; i++ ) + { +void *page; +unsigned long pagetype; + pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) +/* a bogus/unmapped page: skip it */ +continue ; + +if ( pfn > p2m_size ) + { +ERROR ("pfn out of range" ); + goto out; + } + + pfn_type[pfn] = pagetype; + + mfn = p2m[pfn]; + + page = region_base + i*PAGE_SIZE ; + +if (read_exact (io_fd, page,PAGE_SIZE ) ) + { +ERROR ("Error when reading page (type was %lx)" , pagetype); + goto out; + } + + pagetype &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ; + +if ( (pagetype >=XEN_DOMCTL_PFINFO_L1TAB ) && + (pagetype <=XEN_DOMCTL_PFINFO_L4TAB ) ) + { + DPRINTF("uncanonicalize_pagetable pagetype = %lx pfn =
%lu\n", pagetype, pfn);
+ }
+ else if ( pagetype != XEN_DOMCTL_PFINFO_NOTAB )
+ {+ ERROR("Bogus page type %lx page table is out of range: "
+ "i=%d p2m_size=%lu", pagetype, i, p2m_size);
+ goto out;
+
+ }
+ } /* end of 'batch' for loop */
+
+ munmap(region_base, j*PAGE_SIZE);
+ n+= j; /* crude stats */
+ }
+
+ /* HVM specific */
+ {
+ uint32_t rec_len;
+
+ /* Read HVM context */
+ if ( read_exact(io_fd, &rec_len, sizeof(uint32_t)) )
+ {+ ERROR("error read hvm context size!\n");
+ goto build;
+ }
+
+ if (rec_len != hvm_buf_size)
+ {
+ if (hvm_buf[info_non_active] == NULL)
+ { /* hvm_buf will be reused. */
+ hvm_buf_size = rec_len;
+ hvm_buf[0] = malloc(hvm_buf_size);
+ hvm_buf[1] = malloc(hvm_buf_size);
+ if ( hvm_buf[0] == NULL || hvm_buf[1] == NULL)
+ {
+ ERROR("memory alloc for hvm context buffer failed");
+ errno = ENOMEM;
+ goto out;+ }
+ } else {
+ ERROR("Sorry, we did not thought about HVM image size "
+ "change.");
+ goto out;
+ }
+ }
+
+ if ( read_exact(io_fd, hvm_buf[info_non_active], hvm_buf_size) )
+ {
+ ERROR("error loading the HVM context");
+ goto build;
+ }
+ }
+
+ /*
+ * Commit!
+ */
+ {
+ int zero = 0;
+
+ if ( write_exact(io_fd, &zero, sizeof(int))) {
+ ERROR("Error when replying to sender (errno %d)", errno);
+ goto out;
+ }
+ }
+
+ /* commit pages */
+ if (kemari_transaction_mode && num_pages > 0)
+ {
+ int nr_mfns;
+ /* First pass for this batch: work out how much memory to alloc */
+ nr_mfns = 0;
+ for ( i = 0; i < num_pages; i++ )
+ {
+ unsigned long pfn, pagetype;
+ pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK;
+ pagetype = region_pfn_type[i] & XEN_DOMCTL_PFINFO_LTAB_MASK;+
+ if ( (pagetype != XEN_DOMCTL_PFINFO_XTAB) &&
+ (p2m[pfn] == INVALID_P2M_ENTRY) )
+ {
+ /* Have a live PFN which hasn't had an MFN allocated */
+ p2m_batch[nr_mfns++] = pfn;
+ p2m[pfn]--;
+ DPRINTF("Cannot be occur!!! no map for pfn: %lu\n", pfn);+ }
+ }
+
+ /* Now allocate a bunch of mfns for this batch */
+ if ( nr_mfns &&
+ (xc_domain_memory_populate_physmap(xc_handle, dom, nr_mfns, 0,
+ 0, p2m_batch) != 0) )
+ {
+ ERROR("Failed to allocate memory for batch.!\n");
+ errno = ENOMEM;
+ goto out;
+ }
+
+ /* Second pass for this batch: update p2m[] and region_mfn[] */
+ nr_mfns = 0;
+ for ( i = 0; i < num_pages; i++ )
+ {
+ unsigned long pfn, pagetype;
+ pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK;
+ pagetype = region_pfn_type[i] & XEN_DOMCTL_PFINFO_LTAB_MASK;+
+ if ( pagetype == XEN_DOMCTL_PFINFO_XTAB ) {
+ DPRINTF("pfn %lu = XEN_DOMCTL_PFINFO_XTAB\n", pfn);
+ region_mfn[i] = ~0UL; /* map will fail but we don't care */
+ }
+ else
+ {
+ if ( p2m[pfn] == (INVALID_P2M_ENTRY-1) )
+ {
+ /* We just allocated a new mfn above; update p2m */
+ p2m[pfn] = p2m_batch[nr_mfns++];
+ nr_pfns++;+ }
+
+ /* setup region_mfn[] for batch map.
+ * For HVM guests, this interface takes PFNs, not MFNs */
+ region_mfn[i] = pfn;
+ }
+ }
+
+ /* Map relevant mfns */
+ region_base = xc_map_foreign_batch(
+ xc_handle, dom, PROT_WRITE, region_mfn, num_pages);
+
+ if ( region_base == NULL )
+ {
+ ERROR("map batch failed");
+ goto out;
+ }
+
+ for ( i = 0; i < num_pages; i++ )
+ {
+ void *page, *spage;
+ unsigned long pagetype;
+
+ pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK;
+ pagetype = region_pfn_type[i] & XEN_DOMCTL_PFINFO_LTAB_MASK;
+
+ if ( pfn > p2m_size )
+ {
+ ERROR("pfn out of range");
+ goto out;
+ }
+
+ pfn_type[pfn] = pagetype;
+
+ mfn = p2m[pfn];
+
+ page = region_base + i*PAGE_SIZE;
+ spage = tmp_region + i*PAGE_SIZE;
+
+ if ( !memmove(page, spage, PAGE_SIZE) )
+ {
+ ERROR("Error when reading page (type was %lx)", pagetype);
+ goto out;
+ }
+
+ } /* end of 'batch' for loop */
+
+ munmap(region_base, num_pages*PAGE_SIZE);
+ num_pages = 0; /* clear num_pages for refill */
+ }
+
+ /* commit HVM specific status */
+ info_active = info_non_active;
+ info_non_active = info_active ? 0 : 1;
+
+ /* HVM success! */
+ rc = 0;
+ kemari_transaction_mode = 1;
+ }
+
+ build: /* building HVM context */
+ DPRINTF("building status %d\n", rc);
+ if (rc == 0)
+ {+ FILE *qemu_fp;
+ char path[128];
+
+ /* set the EPT identity PT location */
+ xc_set_hvm_param(xc_handle, dom, HVM_PARAM_IDENT_PT,
+ ident_pt[info_active]);
+
+ if ( (frc = xc_set_hvm_param(xc_handle, dom,
+ HVM_PARAM_IOREQ_PFN, magic_pfns[0]))
+ || (frc = xc_set_hvm_param(xc_handle, dom,
+ HVM_PARAM_BUFIOREQ_PFN, magic_pfns[1]))
+ || (frc = xc_set_hvm_param(xc_handle, dom,
+ HVM_PARAM_STORE_PFN, magic_pfns[2]))
+ || (frc = xc_set_hvm_param(xc_handle, dom,
+ HVM_PARAM_PAE_ENABLED, pae))
+ || (frc = xc_set_hvm_param(xc_handle, dom,
+ HVM_PARAM_STORE_EVTCHN,
+ store_evtchn))
+ || (frc = xc_set_hvm_param(xc_handle, dom,
+ HVM_PARAM_CALLBACK_IRQ,
+ callback_irq)) )
+ {
+ ERROR("error setting HVM params: %i", frc);
+ rc = 3;
+ goto out;
+ }
+ *store_mfn = magic_pfns[2];
+ DPRINTF("kemari_restore: magic_pfns 0: %lld, 1: %lld, 2: %lld\n",+ magic_pfns[0], magic_pfns[1], magic_pfns[2]);
+
+ frc = xc_domain_hvm_setcontext(xc_handle, dom, hvm_buf[info_active],
+ hvm_buf_size);
+ if ( frc )
+ {
+ ERROR("error setting the HVM context");
+ rc = 4;
+ goto out;+ }
+
+ if (qemu_image_size[info_active] == 0)
+ {
+ ERROR("Did not received QEMU image");
+ rc = 5;
+ goto out;
+ }
+ snprintf(path, sizeof(path), "/var/lib/xen/qemu-save.%d", dom);
+ if ((qemu_fp = fopen(path, "w")) == NULL)
+ {
+ ERROR("error opening QEMU image");
+ rc = 5;
+ goto out;+ }
+ if (fwrite(qemu_image[info_active], qemu_image_size[info_active],
+ 1, qemu_fp) != 1)
+ {
+ ERROR("error writing QEMU image");
+ rc = 5;
+ goto out;+ }
+ fclose(qemu_fp);
+ }
+
+ out:
+ if ( (rc != 0) && (dom != 0) )
+ xc_domain_destroy(xc_handle, dom);
+ free(p2m); + free(pfn_type); + free(region_mfn); + free(p2m_batch); + free(tmp_region); + free(hvm_buf[0]); + free(hvm_buf[1]); + free(qemu_image[0]); + free(qemu_image[1]);
+
+ /* discard cache for save file */
+ discard_file_cache(io_fd, 1 /*flush*/);
+
+ DPRINTF("Restore exit with rc=%d\n", rc);
+
+ return rc;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
+
tools/xcutils/xc_kemari_restore.c new file +88 -0
88 lines not on this page · raw message
RFC PATCH 7/13 Kemari: add Kemari support to python
In reply to Yoshiaki TamuraThis patch implements python code for supporting Kemari. It provides --kemari option and notifies it to the destination. If --kemari is specified, xc_kemari_save is called instead of xc_save on the sender, and xc_kemari_restore is called instead of xc_restore on the receiver.
--- tools/python/xen/xend/XendAPI.py | 3 - tools/python/xen/xend/XendCheckpoint.py | 81 ++++++++++++++++++++++++++++---- tools/python/xen/xend/XendDomain.py | 6 +- tools/python/xen/xm/migrate.py | 10 +++ 4 files changed, 85 insertions(+), 15 deletions(-)
tools/python/xen/xend/XendAPI.py +2 -1
@@ -1780,9 +1780,10 @@ port = other_config.get("port", 0) node = other_config.get("node", -1) ssl = other_config.get("ssl", None) + kemari = other_config.get("kemari", None) xendom.domain_migrate(xeninfo.getDomid(), destination_url, - bool(live), port, node, ssl) + bool(live), port, node, ssl, kemari) return xen_api_success_void() def VM_save(self, _, vm_ref, dest, checkpoint):
tools/python/xen/xend/XendCheckpoint.py +41 -3
70 lines not on this page · raw message
dominfo.getDomid()):
write_exact(fd, QEMU_SIGNATURE, "could not write qemu signature") qemu_fd = os.open("/var/lib/xen/qemu-save.%d" % dominfo.getDomid(),
os.O_RDONLY)
@@ -138,7 +176,7 @@
os.close(qemu_fd)
os.remove("/var/lib/xen/qemu-save.%d" % dominfo.getDomid())- if checkpoint:
+ if checkpoint or kemari:
dominfo.resumeDomain()
else:
dominfo.destroy()
@@ -184,6 +222,16 @@
raise XendError("not a valid guest state file: config parse")
vmconfig = p.get_val()
+
+ # Checks kemari is enabled or not. + # Since Xen do not know kemari option, this option will not be migrated. + is_kemari = False + kemari_device_info = [] + for v in vmconfig:
+ if v[0] == 'kemari' and v[1]:
+ is_kemari = True
+ if v[0] == 'kemari-device-info' and v[1]:
+ kemari_device_info = v[1]
if not relocating:
domconfig = XendConfig(sxp_obj = vmconfig)
@@ -258,7 +306,15 @@
shadow_cur = xc.shadow_mem_control(dominfo.getDomid(), shadow / 1024) dominfo.info['shadow_memory'] = shadow_cur
- cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE), + # Use Kemari restore. Switching mechanism between normal migration + # and Kemari migration will be implemented lator. + # + if is_kemari:
+ cmd = map(str, [xen.util.auxbin.pathTo(XC_KEMARI_RESTORE),
+ fd, dominfo.getDomid(),
+ store_port, console_port, int(is_hvm), pae, apic])
+ else:
+ cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE),
fd, dominfo.getDomid(),
store_port, console_port, int(is_hvm), pae, apic])
log.debug("[xc_restore]: %s", string.join(cmd))
@@ -266,6 +322,8 @@
handler = RestoreInputHandler()
forkHelper(cmd, fd, handler.handler, True)
+ if is_kemari:
+ os.close(fd)
# We don't want to pass this fd to any other children -- we # might need to recover the disk space that backs it.
@@ -285,7 +343,7 @@
# get qemu state and create a tmp file for dm restore
# Even PV guests may have QEMU stat, but its not currently
# used so only bother with HVM currently.
- if is_hvm:
+ if is_hvm and not is_kemari:
qemu_signature = read_exact(fd, len(QEMU_SIGNATURE),
"invalid device model signature read")
if qemu_signature != QEMU_SIGNATURE:
@@ -303,8 +361,10 @@
os.close(qemu_fd)
restore_image.setCpuid()-
- os.read(fd, 1) # Wait for source to close connection
+ if is_kemari:
+ restore_image.setCpuid()
+ else:
+ os.read(fd, 1) # Wait for source to close connection
dominfo.completeRestore(handler.store_mfn, handler.console_mfn)
@@ -322,7 +382,10 @@
lock = False;
try:- dominfo.waitForDevices() # Wait for backends to set up
+ if is_kemari:
+ dominfo.waitForAttachedDevices(kemari_device_info)
+ else:
+ dominfo.waitForDevices() # Wait for backends to set up
except Exception, exn:
log.exception(exn)
tools/python/xen/xend/XendDomain.py +1 -1
@@ -1267,7 +1267,7 @@ return val - def domain_migrate(self, domid, dst, live=False, port=0, node=-1, ssl=None): + def domain_migrate(self, domid, dst, live=False, port=0, node=-1, ssl=None,
kemari=None):
"""Start domain migration.
@param domid: Domain ID or Name
@@ -1332,7 +1332,7 @@
try:
XendCheckpoint.save(p2cwrite, dominfo, True, live, dst,
- node=node)
+ node=node, kemari=kemari)
finally:
sock.shutdown()
sock.close()
@@ -1358,7 +1358,7 @@
try:
XendCheckpoint.save(sock.fileno(), dominfo, True, live,
- dst, node=node)
+ dst, node=node, kemari=kemari)
finally:
sock.close()
tools/python/xen/xm/migrate.py +8 -2
@@ -51,6 +51,10 @@ fn=set_true, default=None, use="Use ssl connection for migration.") +gopts.opt('kemari', short='k', + fn=set_true, default=None, + use="Use kemari migration.") + def help(): return str(gopts)
@@ -70,7 +74,8 @@ other_config = { "port": opts.vals.port, "node": opts.vals.node, - "ssl": opts.vals.ssl + "ssl": opts.vals.ssl, + "kemari": opts.vals.kemari } server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live), other_config)
@@ -78,4 +83,5 @@ server.xend.domain.migrate(dom, dst, opts.vals.live, opts.vals.port, opts.vals.node, - opts.vals.ssl) + opts.vals.ssl, + opts.vals.kemari)
RFC PATCH 8/13 Kemari: add dev state "Attached" to python
In reply to Yoshiaki TamuraThis patch implements python code that sends information necessary to attach devices. It also restores the information to the XenStore.
--- tools/python/xen/xend/XendDomainInfo.py | 8 +++ tools/python/xen/xend/server/DevController.py | 61 ++++++++++++++++++++++++++ tools/python/xen/xend/server/vfbif.py | 4 + 3 files changed, 73 insertions(+)
tools/python/xen/xend/XendDomainInfo.py +8 -0
@@ -868,6 +868,14 @@ """ for devclass in XendDevices.valid_devices(): self.getDeviceController(devclass).waitForDevices() + + def waitForAttachedDevices(self, devinfo): + """Wait for this domain's configured devices to connect. + + @raise VmError: if any device fails to initialise. + """ + for devclass in XendDevices.valid_devices(): + self.getDeviceController(devclass).waitForAttachedDevices(devinfo) def hvm_destroyPCIDevice(self, vslot): log.debug("hvm_destroyPCIDevice called %s", vslot)
tools/python/xen/xend/server/DevController.py +61 -0
79 lines not on this page · raw message
tools/python/xen/xend/server/vfbif.py +4 -0
@@ -39,6 +39,10 @@ if devinfo[i] is not None]) def waitForDevice(self, devid): + # is a qemu-dm managed device, don't wait for hotplug for these. + return + + def waitForAttachedDevice(self, devid, devinfo): # is a qemu-dm managed device, don't wait for hotplug for these. return
RFC PATCH 9/13 Kemari: add XenbusStateAttached to xenbus
In reply to Yoshiaki TamuraThis patch adds a new xenbus state XenbusStateAttached, which is used when Kemari restores the guest on the secondary after the failover.
--- drivers/xen/xenbus/xenbus_client.c | 3 ++- include/xen/interface/io/xenbus.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-)
drivers/xen/xenbus/xenbus_client.c +2 -1
@@ -52,7 +52,8 @@ [ XenbusStateInitialised ] = "Initialised", [ XenbusStateConnected ] = "Connected", [ XenbusStateClosing ] = "Closing", - [ XenbusStateClosed ] = "Closed", + [ XenbusStateClosed ] = "Closed", + [ XenbusStateAttached ] = "Attached", };return (state <ARRAY_SIZE (name)) ? name[state] :"INVALID" ; }
include/xen/interface/io/xenbus.h +3 -1
@@ -63,7 +63,9 @@*/ XenbusStateReconfiguring =7 , - XenbusStateReconfigured = 8 + XenbusStateReconfigured =8 , + + XenbusStateAttached = 9 };typedef enum xenbus_state XenbusState ;
RFC PATCH 10/13 Kemari: XenbusStateAttached handler for blkback
In reply to Yoshiaki TamuraThis patch implements XenbusStateAttached handler for blkback, which is called when Kemari restores the guest. First, the handler attaches the back-end ring to the device channel using BACK_RING_ATTACH macro. Then, it adjusts producer and consumer indexes of the back-end, so that the front-end driver in the guest can continue transparently.
--- drivers/xen/blkback/common.h | 2 + drivers/xen/blkback/interface.c | 41 +++++++++++++++++++++++++++ drivers/xen/blkback/xenbus.c | 59 +++
drivers/xen/blkback/common.h +2 -0
@@ -100,6 +100,8 @@void blkif_disconnect (blkif_t *blkif);void blkif_free (blkif_t *blkif);int blkif_map (blkif_t *blkif,unsigned long shared_page,unsigned int evtchn); +int blkif_attach (blkif_t *blkif,unsigned long shared_page, +unsigned int evtchn);#define blkif_get (_b) (atomic_inc(&(_b)->refcnt))#define blkif_put (_b) \
drivers/xen/blkback/interface.c +41 -0
47 lines not on this page · raw message
drivers/xen/blkback/xenbus.c +59 -0
76 lines not on this page · raw message
RFC PATCH 11/13 Kemari: XenbusStateAttached handler for netback
In reply to Yoshiaki TamuraThis patch implements XenbusStateAttached handler for netback. The procedure is almost the same as blkback except that it handles both tx and rx back-end rings.
--- drivers/xen/netback/common.h | 2 drivers/xen/netback/interface.c | 58 +++++++++++++++++++++ drivers/xen/netback/netback.c | 3 + drivers/xen/netback/xenbus.c | 109 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+)
drivers/xen/netback/common.h +2 -0
@@ -182,6 +182,8 @@netif_t *netif_alloc (domid_t domid,unsigned int handle);int netif_map (netif_t *netif,unsigned long tx_ring_ref,unsigned long rx_ring_ref,unsigned int evtchn); +int netif_attach (netif_t *netif,unsigned long tx_ring_ref, +unsigned long rx_ring_ref,unsigned int evtchn);#define netif_get (_b) (atomic_inc(&(_b)->refcnt))#define netif_put (_b) \
drivers/xen/netback/interface.c +58 -0
64 lines not on this page · raw message
drivers/xen/netback/netback.c +3 -0
@@ -396,6 +396,9 @@ copy_gop->dest.offset =0 ; copy_gop->dest.u.ref = req->gref; copy_gop->len = size; +if (req->gref ==0 ) +printk ("did%d,gref%d,size%d,rc%lu,i%d\n" , + netif->domid, req->gref, size, netif->rx.req_cons, i); }else { meta->copy =0 ;if (!xen_feature (XENFEAT_auto_translated_physmap)) {
drivers/xen/netback/xenbus.c +109 -0
133 lines not on this page · raw message
RFC PATCH 12/13 Kemari: use signal to save qemu state for Kemari
In reply to Yoshiaki TamuraThis patch implements signal handler for saving QEMU status quickly. It also includes a feature borrowed from KVM-17, which saves QEMU status quickly without repositioning file offset by fseek.
--- vl.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-)
commit 3050ae2f8fffc2238a9e763b2534f3778b1bb372
Author: Yoshisato YANAGISAWA <[email protected]> Date: Thu Feb 26 14:25:40 2009 +0900
Add features required for Kemari.
vl.c +93 -1
@@ -258,6 +258,7 @@ struct drive_opt {static CPUState *cur_cpu;static CPUState *next_cpu;static int event_pending =1 ; +volatile int kemari_enabled =0 ;#define TFR (expr) do { if ((expr) != -1) break; } while (errno == EINTR)
@@ -6152,6 +6153,36 @@ int register_savevm(const char *idstr,#define QEMU_VM_FILE_MAGIC 0x5145564d#define QEMU_VM_FILE_VERSION 0x00000002 +#define QEMU_VM_FILE_KEMARI 0x80000000 + +static int qemu_savevm_state_kemari (QEMUFile *f) +{ +SaveStateEntry *se; +int len, ret; + +qemu_put_be32 (f,QEMU_VM_FILE_MAGIC ); +qemu_put_be32 (f, (QEMU_VM_FILE_VERSION |QEMU_VM_FILE_KEMARI )); + +for (se = first_se; se !=NULL ; se = se->next) { +/* ignore vga because it is too large. */ +if (strstr (se->idstr,"vga" )) +continue ; + + len =strlen (se->idstr); + +qemu_put_byte (f, len); +qemu_put_buffer (f, (uint8_t *)se->idstr, len); +qemu_put_be32 (f, se->instance_id); +qemu_put_be32 (f, se->version_id); + + se->save_state (f, se->opaque); + } + +qemu_put_byte (f,0 ); + + ret =0 ; +return ret; +}static int qemu_savevm_state (QEMUFile *f) {
@@ -6210,6 +6241,40 @@ static SaveStateEntry *find_se(const char *idstr, int instance_id)return NULL ; } +static int qemu_loadvm_state_kemari (QEMUFile *f) +{ +SaveStateEntry *se; +int len, ret, instance_id, version_id; +unsigned int v; +char idstr[256 ]; + +fprintf (stderr,"qemu: loading VM in Kemari mode.\n" ); +for (;;) { + len =qemu_get_byte (f); +if (len ==0 ) +break ; +qemu_get_buffer (f, (uint8_t *)idstr, len); + idstr[len] ='\0' ; + instance_id =qemu_get_be32 (f); + version_id =qemu_get_be32 (f); + se =find_se (idstr, instance_id); +if (!se) { +fprintf (stderr,"qemu: warning: instance 0x%x of device '%s' not present in current VM\n" , + instance_id, idstr); + }else { + ret = se->load_state (f, se->opaque, version_id); +if (ret <0 ) { +fprintf (stderr,"qemu: warning: error while loading state for instance 0x%x of device '%s'\n" , + instance_id, idstr); + } + } + } + ret =0 ; + + the_end: +return ret; +} +static int qemu_loadvm_state (QEMUFile *f) {SaveStateEntry *se;
@@ -6223,6 +6288,10 @@ static int qemu_loadvm_state(QEMUFile *f) goto fail; v =qemu_get_be32 (f);if (v !=QEMU_VM_FILE_VERSION ) { +if (v == (QEMU_VM_FILE_VERSION |QEMU_VM_FILE_KEMARI )) { + ret =qemu_loadvm_state_kemari (f); + goto the_end; + } fail: ret =-1 ; goto the_end;
@@ -6386,7 +6455,10 @@ void do_savevm(const char *name)term_printf ("Could not open VM state file\n" ); goto the_end; } - ret =qemu_savevm_state (f); +if (kemari_enabled) + ret =qemu_savevm_state_kemari (f); +else + ret =qemu_savevm_state (f); sn->vm_state_size =qemu_ftell (f);qemu_fclose (f);if (ret <0 ) {
@@ -7893,6 +7965,15 @@ static BOOL WINAPI qemu_ctrl_handler(DWORD type) }#endif +#ifndef _WIN32 +void xenstore_process_logdirty_event (void ); +static void kemari_handler (int dummy) +{ + kemari_enabled =1 ;/* QEMU will run in kemari mode */ +xenstore_process_logdirty_event (); +} +#endif /* !_WIN32 */ +#define MAX_NET_CLIENTS 32int main (int argc,char **argv)
@@ -7969,6 +8050,17 @@ int main(int argc, char **argv) act.sa_handler =SIG_IGN ; sigaction(SIGPIPE, &act, NULL); } + { +struct sigaction act; +sigfillset (&act.sa_mask); + act.sa_handler = kemari_handler; +#ifdef CONFIG_DM + act.sa_flags =SA_RESTART ; +#else /* !CONFIG_DM */ + act.sa_flags =0 ; +#endif /* CONFIG_DM */ +sigaction (SIGUSR1 , &act,NULL ); + }#else SetConsoleCtrlHandler (qemu_ctrl_handler, TRUE);/* Note: cpu_interrupt() is currently not SMP safe, so we force
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 3 lines · Yoshiaki Tamura
This patch implements signal handler for saving QEMU status quickly. It also includes a feature borrowed from KVM-17, which saves QEMU status quickly without repositioning file offset by fseek.
We got rid of the signal interface not so long ago because it doesn't work for stubdoms.
The new way of asking qemu to save the VM state is writing a command on xenstore (look at xenstore.c:xenstore_process_dm_command_event). Would that work for you?
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 6 lines · Stefano Stabellini
Quoted 3 lines · Yoshiaki Tamura
This patch implements signal handler for saving QEMU status quickly. It also includes a feature borrowed from KVM-17, which saves QEMU status quickly without repositioning file offset by fseek.
Quoted 9 lines
We got rid of the signal interface not so long ago because it doesn't work for stubdoms.
The new way of asking qemu to save the VM state is writing a command on xenstore (look at xenstore.c:xenstore_process_dm_command_event). Would that work for you?
xenstore is attractive because it can send more information between components than signals.
However, we used the signal handler to avoid the latency of xenstore. Although we haven't measured the latency, we need a simple and quick interface to save the QEMU state because Kemari transfers the saved state on every synchronization which happens quite often.
Have you measured, or any thoughts on the latency of xenstore compared to signals?
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 1 line · Yoshiaki Tamura
Have you measured, or any thoughts on the latency of xenstore compared to signals?
I did a couple of quick tests and it seems that signals are faster then xenstore but still xenstore offers good performances.
I measured the time using rdtsc at the sender size and at the receiver size, the difference between the two values is the following:
signals: 368836
xenstore: 1984632
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Stefano Stabellini writes ("Re: [Xen-devel] [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari"):
Quoted 8 lines
I did a couple of quick tests and it seems that signals are faster then xenstore but still xenstore offers good performances.
I measured the time using rdtsc at the sender size and at the receiver size, the difference between the two values is the following:
signals: 368836
xenstore: 1984632
This is with an unloaded xenstored, I take it. What are these units ?
Ian.
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 11 lines · Ian Jackson
Stefano Stabellini writes ("Re: [Xen-devel] [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari"):
Quoted 8 lines
I did a couple of quick tests and it seems that signals are faster then xenstore but still xenstore offers good performances.
I measured the time using rdtsc at the sender size and at the receiver size, the difference between the two values is the following:
signals: 368836
xenstore: 1984632This is with an unloaded xenstored, I take it.
Yes
Quoted 2 lines
What are these units ?
clock cycles
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 9 lines · Stefano Stabellini
Quoted 7 lines · Ian Jackson
Stefano Stabellini writes ("Re: [Xen-devel] [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari"):
Quoted 6 lines
I did a couple of quick tests and it seems that signals are faster then xenstore but still xenstore offers good performances.
I measured the time using rdtsc at the sender size and at the receiver size, the difference between the two values is the following:
Quoted 6 lines
Quoted 3 lines
Quoted 2 lines
signals: 368836
xenstore: 1984632This is with an unloaded xenstored, I take it.
Yes
Thanks for measuring the numbers. It made things clear. I think we need a signal-based interface as Ian said previously.
On our test environment, AMD Barcelona 2.3GHz, the total cpu cycles of Kemari in userland when running I/O intensive applications is around 3000000.
Although Xen transferring code and QEMU saving code is processed concurrently, using xenstored for staring QEMU portion would lower the performance. Especially if the xenstored gets slower when items are loaded.
Yoshi
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 9 lines · Yoshiaki Tamura
Thanks for measuring the numbers. It made things clear. I think we need a signal-based interface as Ian said previously.
On our test environment, AMD Barcelona 2.3GHz, the total cpu cycles of Kemari in
userland when running I/O intensive applications is around 3000000. Although Xen transferring code and QEMU saving code is processed concurrently, using xenstored for staring QEMU portion would lower the performance. Especially if the xenstored gets slower when items are loaded.
Using signals won't work with qemu-dm in a stub domain.
-- Keir
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 6 lines · Keir Fraser
Quoted 3 lines · Yoshiaki Tamura
Thanks for measuring the numbers. It made things clear. I think we need a signal-based interface as Ian said previously.
Quoted 7 lines
Quoted 5 lines
On our test environment, AMD Barcelona 2.3GHz, the total cpu cycles of Kemari in userland when running I/O intensive applications is around 3000000. Although Xen transferring code and QEMU saving code is processed concurrently, using xenstored for staring QEMU portion would lower the performance. Especially if the xenstored gets slower when items are loaded.
Using signals won't work with qemu-dm in a stub domain.
Yes. I have the following options in my mind to solve it.
1. Switch using signals for qemu-dm in Dom0, and xenstored for stub domains.
2. Prepare a dedicated event channel for saving QEMU state.
I guess using a dedicated event channel would be quicker than xenstored but slower than signals. If it is, using a dedicated event channel for option 1 may be better.
Yoshi
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
In reply to Yoshiaki TamuraQuoted 16 lines · Yoshiaki Tamura
Quoted 6 lines · Stefano Stabellini
Quoted 4 lines · Ian Jackson
Stefano Stabellini writes ("Re: [Xen-devel] [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari"):
Quoted 3 lines
I did a couple of quick tests and it seems that signals are faster then xenstore but still xenstore offers good performances.
Quoted 8 lines
Quoted 6 lines
Quoted 5 lines
I measured the time using rdtsc at the sender size and at the receiver size, the difference between the two values is the following:
signals: 368836
xenstore: 1984632This is with an unloaded xenstored, I take it.
Yes
Quoted 9 lines
Thanks for measuring the numbers. It made things clear. I think we need a signal-based interface as Ian said previously.
On our test environment, AMD Barcelona 2.3GHz, the total cpu cycles of Kemari in userland when running I/O intensive applications is around 3000000. Although Xen transferring code and QEMU saving code is processed concurrently, using xenstored for staring QEMU portion would lower the performance. Especially if the xenstored gets slower when items are loaded.
Using an event channel to notify qemu in a stubdom that a particular event occurred improves things a lot: it is even faster than signals!
event channel: 20880
performances using an event channel with qemu as a process in dom0 are about the same.
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 8 lines · Stefano Stabellini
Using an event channel to notify qemu in a stubdom that a particular event occurred improves things a lot: it is even faster than signals!
event channel: 20880
performances using an event channel with qemu as a process in dom0 are about the same.
An event channel is the way to go I think.
-- Keir
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 5 lines · Keir Fraser
Quoted 3 lines · Stefano Stabellini
Using an event channel to notify qemu in a stubdom that a particular event occurred improves things a lot: it is even faster than signals!
Quoted 7 lines
Quoted 4 lines
event channel: 20880
performances using an event channel with qemu as a process in dom0 are about the same.
An event channel is the way to go I think.
I agree.
We'll update the patches.
Yoshi
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
In reply to Yoshiaki Tamura-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Quoted 11 lines · Yoshiaki Tamura
Thanks for measuring the numbers. It made things clear. I think we need a signal-based interface as Ian said previously.
On our test environment, AMD Barcelona 2.3GHz, the total cpu cycles of Kemari in userland when running I/O intensive applications is around 3000000.
Although Xen transferring code and QEMU saving code is processed concurrently, using xenstored for staring QEMU portion would lower the performance. Especially if the xenstored gets slower when items are loaded.
Quoted 1 line
Yoshi
For Remus, we ended up using a Unix-domain socket to handshake with QEMU for suspend/resume, for the same reason (to improve performance compared to Xenstore). Just another approach :) and also another approach that won't work with stubdom.
Chris
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.9 (GNU/Linux)
Comment: GnuPT 2.7.2
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org
iEYEARECAAYFAknBMV8ACgkQXUF6hOTGP7eGCQCfZrO7mauFehuKtGdyyF2hOOIw QQgAn01JCwlMf0yYRSpwgG4z+KC4qYmo
=6sjl
-----END PGP SIGNATURE-----
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
Quoted 15 lines · Christopher Head
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1Quoted 11 lines · Yoshiaki Tamura
Thanks for measuring the numbers. It made things clear. I think we need a signal-based interface as Ian said previously.
On our test environment, AMD Barcelona 2.3GHz, the total cpu cycles of Kemari in userland when running I/O intensive applications is around 3000000.
Although Xen transferring code and QEMU saving code is processed concurrently, using xenstored for staring QEMU portion would lower the performance. Especially if the xenstored gets slower when items are loaded.
Quoted 6 lines
Quoted 1 line
Yoshi
For Remus, we ended up using a Unix-domain socket to handshake with QEMU for suspend/resume, for the same reason (to improve performance compared to Xenstore). Just another approach :) and also another approach that won't work with stubdom.
Thanks for your info:)
It's good to know that we weren't the only who wanted this kind of interface.
Yoshi
Re: [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari
In reply to Stefano StabelliniStefano Stabellini writes ("Re: [Xen-devel] [RFC][PATCH 12/13] Kemari: use signal to save qemu state for Kemari"):
Quoted 6 lines
Quoted 4 lines · Yoshiaki Tamura
This patch implements signal handler for saving QEMU status quickly. It also includes a feature borrowed from KVM-17, which saves QEMU status quickly without repositioning file offset by fseek.
Quoted 6 lines
We got rid of the signal interface not so long ago because it doesn't work for stubdoms.
The new way of asking qemu to save the VM state is writing a command on xenstore (look at xenstore.c:xenstore_process_dm_command_event). Would that work for you?
I think that in this case a signal-based interface is probably fine, because it does indeed need very good performance which is difficult to do with xenstore.
Ian.
RFC PATCH 13/13 Kemari: use shared region with to flip logdirty_bitmap
In reply to Yoshiaki TamuraThis patch provides a shared memory to communicate between xc_kemari_save and QEMU directly. The number of active logdirty_bitmap and condition are sent through the shared memory.
--- xenstore.c | 62 ++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 17 deletions(-)
commit 3050ae2f8fffc2238a9e763b2534f3778b1bb372
Author: Yoshisato YANAGISAWA <[email protected]> Date: Thu Feb 26 14:25:40 2009 +0900
Add features required for Kemari.
xenstore.c +45 -17
@@ -593,6 +593,7 @@ int xenstore_fd(void)unsigned long *logdirty_bitmap =NULL ;unsigned long logdirty_bitmap_size;extern int vga_ram_size, bios_size; +extern volatile int kemari_enabled;void xenstore_process_logdirty_event (void ) {
@@ -600,6 +601,8 @@ void xenstore_process_logdirty_event(void)static char *active_path =NULL ;static char *next_active_path =NULL ;static char *seg =NULL ; +static char *kemari_qemu_info =NULL ; +static char *qemu_file =NULL ;unsigned int len;int i;
@@ -666,6 +669,8 @@ void xenstore_process_logdirty_event(void) seg =NULL ; return; } + kemari_qemu_info = seg + logdirty_bitmap_size *2 ; +asprintf (&qemu_file,"/dev/shm/qemu-save.%d" , domid);/* use tmpfs */ #endif /* Remember the paths for the next-active and active entries */
@@ -683,26 +688,49 @@ void xenstore_process_logdirty_event(void) } } -fprintf (logfile,"Triggered log-dirty buffer switch\n" ); +if (kemari_enabled) { +while (kemari_qemu_info[1 ]) +xen_rmb (); + +/* Switch buffers */ + i = kemari_qemu_info[0 ]; +if (i !=0 && i !=1 ) { +fprintf (logfile,"Log-dirty: bad next-active entry: %s\n" , act); +exit (1 ); + } + logdirty_bitmap = (unsigned long *)(seg + i * logdirty_bitmap_size); + kemari_qemu_info[1 ] =1 ; +xen_wmb (); + +/* Save QEMU status */ +while (kemari_qemu_info[2 ]) +xen_rmb (); +do_savevm (qemu_file); + kemari_qemu_info[2 ] =1 ; +xen_wmb (); + }else { + +fprintf (logfile,"Triggered log-dirty buffer switch\n" ); -/* Read the required active buffer from the store */ - act =xs_read (xsh,XBT_NULL , next_active_path, &len); -if (!act) { -fprintf (logfile,"Log-dirty: can't read next-active\n" ); -exit (1 ); - } +/* Read the required active buffer from the store */ + act =xs_read (xsh,XBT_NULL , next_active_path, &len); +if (!act) { +fprintf (logfile,"Log-dirty: can't read next-active\n" ); +exit (1 ); + } -/* Switch buffers */ - i = act[0 ] -'0' ; -if (i !=0 && i !=1 ) { -fprintf (logfile,"Log-dirty: bad next-active entry: %s\n" , act); -exit (1 ); - } - logdirty_bitmap = (unsigned long *)(seg + i * logdirty_bitmap_size); +/* Switch buffers */ + i = act[0 ] -'0' ; +if (i !=0 && i !=1 ) { +fprintf (logfile,"Log-dirty: bad next-active entry: %s\n" , act); +exit (1 ); + } + logdirty_bitmap = (unsigned long *)(seg + i * logdirty_bitmap_size); -/* Ack that we've switched */ -xs_write (xsh,XBT_NULL , active_path, act, len); -free (act); +/* Ack that we've switched */ +xs_write (xsh,XBT_NULL , active_path, act, len); +free (act); + } }