guest checkpointing patches take 3

diff -r fb46005e0756 -r fa960afcb0ac tools/libxc/xc_domain.c
@@ -86,6 +86,16 @@ int xc_domain_shutdown(int xc_handle,
 
  out1:
     return ret;
+}
+
+
+int xc_domain_resume(int xc_handle,
+                      uint32_t domid)
+{
+    DECLARE_DOMCTL;
+    domctl.cmd = XEN_DOMCTL_resumedomain;
+    domctl.domain = (domid_t)domid;
+    return do_domctl(xc_handle, &domctl);
 }
 
 

diff -r fb46005e0756 -r fa960afcb0ac tools/libxc/xenctrl.h
@@ -236,6 +236,18 @@ int xc_domain_destroy(int xc_handle,
 int xc_domain_destroy(int xc_handle,
                       uint32_t domid);
 
+
+/**
+ * This function resumes a suspended domain. The domain should have
+ * been previously suspended.
+ *
+ * @parm xc_handle a handle to an open hypervisor interface
+ * @parm domid the domain id to resume
+ * return 0 on success, -1 on failure
+ */
+int xc_domain_resume(int xc_handle,
+                      uint32_t domid);
+
 /**
  * This function will shutdown a domain. This is intended for use in
  * fully-virtualized domains where this operation is analogous to the

diff -r fb46005e0756 -r fa960afcb0ac tools/python/xen/lowlevel/xc/xc.c
@@ -160,6 +160,10 @@ static PyObject *pyxc_domain_destroy(XcO
     return dom_op(self, args, xc_domain_destroy);
 }
 
+static PyObject *pyxc_domain_resume(XcObject *self, PyObject *args)
+{
+    return dom_op(self, args, xc_domain_resume);
+}
 
 static PyObject *pyxc_vcpu_setaffinity(XcObject *self,
                                        PyObject *args,

@@ -1027,6 +1031,13 @@ static PyMethodDef pyxc_methods[] = {
       METH_VARARGS, "\n"
       "Destroy a domain.\n"
       " dom [int]:    Identifier of domain to be destroyed.\n\n"
+      "Returns: [int] 0 on success; -1 on error.\n" },
+    
+    { "domain_resume", 
+      (PyCFunction)pyxc_domain_resume, 
+      METH_VARARGS, "\n"
+      "Resume execution of a suspended domain.\n"
+      " dom [int]: Identifier of domain to be resumed.\n\n"
       "Returns: [int] 0 on success; -1 on error.\n" },
 
     { "vcpu_setaffinity", 

diff -r fb46005e0756 -r fa960afcb0ac xen/common/domctl.c
@@ -250,6 +250,31 @@ ret_t do_domctl(XEN_GUEST_HANDLE(xen_dom
     }
     break;
 
+    case XEN_DOMCTL_resumedomain:
+    {
+        struct domain *d = find_domain_by_id(op->domain);
+        struct vcpu *v;
+
+        ret = -ESRCH;
+        if ( d != NULL )
+        {
+            ret = -EINVAL;
+            printk("Resuming domain %d\n", op->domain);
+            if ( (d != current->domain) && (d->vcpu[0] != NULL) &&
+                 test_bit(_DOMF_shutdown, &d->domain_flags) )
+            {
+                clear_bit(_DOMF_shutdown, &d->domain_flags);
+
+                for_each_vcpu (d, v)
+                    vcpu_wake (v);
+
+                ret = 0;
+            }
+            put_domain(d);
+        }
+    }
+    break;
+
     case XEN_DOMCTL_createdomain:
     {
         struct domain *d;

diff -r fb46005e0756 -r fa960afcb0ac xen/include/public/domctl.h
@@ -63,6 +63,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_domctl_creat
 #define XEN_DOMCTL_destroydomain      2
 #define XEN_DOMCTL_pausedomain        3
 #define XEN_DOMCTL_unpausedomain      4
+#define XEN_DOMCTL_resumedomain      27
 
 #define XEN_DOMCTL_getdomaininfo      5
 struct xen_domctl_getdomaininfo {

diff -r fa960afcb0ac -r 204aa85417bd tools/python/xen/lowlevel/xs/xs.c
@@ -618,6 +618,33 @@ static PyObject *xspy_introduce_domain(X
     return none(result);
 }
 
+#define xspy_resume_domain_doc "\n"                                \
+	"Tell xenstore to clear its shutdown flag for a domain.\n" \
+	"This ensures that a subsequent shutdown will fire the\n"  \
+	"appropriate watches.\n"                                   \
+	" dom [int]: domain id\n"			           \
+        "\n"						           \
+        "Returns None on success.\n"				   \
+        "Raises xen.lowlevel.xs.Error on error.\n"
+
+static PyObject *xspy_resume_domain(XsHandle *self, PyObject *args)
+{
+    uint32_t dom;
+
+    struct xs_handle *xh = xshandle(self);
+    bool result = 0;
+
+    if (!xh)
+        return NULL;
+    if (!PyArg_ParseTuple(args, "i", &dom))
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    result = xs_resume_domain(xh, dom);
+    Py_END_ALLOW_THREADS
+
+    return none(result);
+}
 
 #define xspy_release_domain_doc "\n"					\
 	"Tell xenstore to release its channel to a domain.\n"		\

@@ -789,6 +816,7 @@ static PyMethodDef xshandle_methods[] = 
     XSPY_METH(transaction_start, METH_NOARGS),
     XSPY_METH(transaction_end,   METH_VARARGS | METH_KEYWORDS),
     XSPY_METH(introduce_domain,  METH_VARARGS),
+    XSPY_METH(resume_domain,     METH_VARARGS),
     XSPY_METH(release_domain,    METH_VARARGS),
     XSPY_METH(close,             METH_NOARGS),
     XSPY_METH(get_domain_path,   METH_VARARGS),

diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xenstored_core.c
@@ -164,6 +164,7 @@ static char *sockmsg_string(enum xsd_soc
 	case XS_WATCH_EVENT: return "WATCH_EVENT";
 	case XS_ERROR: return "ERROR";
 	case XS_IS_DOMAIN_INTRODUCED: return "XS_IS_DOMAIN_INTRODUCED";
+	case XS_RESUME: return "RESUME";
 	default:
 		return "**UNKNOWN**";
 	}

@@ -1265,6 +1266,10 @@ static void process_message(struct conne
 
 	case XS_GET_DOMAIN_PATH:
 		do_get_domain_path(conn, onearg(in));
+		break;
+
+	case XS_RESUME:
+		do_resume(conn, onearg(in));
 		break;
 
 	default:

diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xenstored_domain.c
@@ -395,6 +395,43 @@ void do_release(struct connection *conn,
 	send_ack(conn, XS_RELEASE);
 }
 
+void do_resume(struct connection *conn, const char *domid_str)
+{
+	struct domain *domain;
+	unsigned int domid;
+
+	if (!domid_str) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	domid = atoi(domid_str);
+	if (!domid) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	if (conn->id != 0) {
+		send_error(conn, EACCES);
+		return;
+	}
+
+	domain = find_domain_by_domid(domid);
+	if (!domain) {
+		send_error(conn, ENOENT);
+		return;
+	}
+
+	if (!domain->conn) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	domain->shutdown = 0;
+	
+	send_ack(conn, XS_RESUME);
+}
+
 void do_get_domain_path(struct connection *conn, const char *domid_str)
 {
 	char *path;

diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xenstored_domain.h
@@ -32,6 +32,9 @@ void do_release(struct connection *conn,
 void do_release(struct connection *conn, const char *domid_str);
 
 /* domid */
+void do_resume(struct connection *conn, const char *domid_str);
+
+/* domid */
 void do_get_domain_path(struct connection *conn, const char *domid_str);
 
 /* Returns the event channel handle */

diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xs.c
@@ -719,6 +719,12 @@ bool xs_release_domain(struct xs_handle 
 	return xs_bool(single_with_domid(h, XS_RELEASE, domid));
 }
 
+/* clear the shutdown bit for the given domain */
+bool xs_resume_domain(struct xs_handle *h, unsigned int domid)
+{
+	return xs_bool(single_with_domid(h, XS_RESUME, domid));
+}
+
 char *xs_get_domain_path(struct xs_handle *h, unsigned int domid)
 {
 	char domid_str[MAX_STRLEN(domid)];

diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xs.h
@@ -133,6 +133,11 @@ bool xs_introduce_domain(struct xs_handl
 			 unsigned int domid,
 			 unsigned long mfn,
                          unsigned int eventchn); 
+/* Resume a domain.
+ * Clear the shutdown flag for this domain in the store.
+ */
+bool xs_resume_domain(struct xs_handle *h, unsigned int domid);
+
 /* Release a domain.
  * Tells the store domain to release the memory page to the domain.
  */

diff -r fa960afcb0ac -r 204aa85417bd xen/include/public/io/xs_wire.h
@@ -45,7 +45,8 @@ enum xsd_sockmsg_type
     XS_SET_PERMS,
     XS_WATCH_EVENT,
     XS_ERROR,
-    XS_IS_DOMAIN_INTRODUCED
+    XS_IS_DOMAIN_INTRODUCED,
+    XS_RESUME
 };
 
 #define XS_WRITE_NONE "NONE"

diff -r 204aa85417bd -r 477813c50e5d tools/libxc/xc_linux_save.c
@@ -569,7 +569,18 @@ static xen_pfn_t *xc_map_m2p(int xc_hand
     return m2p;
 }
 
-
+static int xc_set_vcpucontext(int xc_handle, uint32_t dom,
+                              vcpu_guest_context_t *ctxt)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_setvcpucontext;
+    domctl.domain = (domid_t)dom;
+    domctl.u.vcpucontext.vcpu = 0;
+    set_xen_guest_handle(domctl.u.vcpucontext.ctxt, ctxt);
+
+    return xc_domctl(xc_handle, &domctl);
+}
 
 int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
                   uint32_t max_factor, uint32_t flags, int (*suspend)(int))

@@ -1188,6 +1199,12 @@ int xc_linux_save(int xc_handle, int io_
         goto out;
     }
 
+    /* set hypercall return code to indicate domain is resuming */
+    ctxt.user_regs.eax = 1;
+    rc = xc_set_vcpucontext(xc_handle, dom, &ctxt);
+    if (rc)
+        ERROR("Error setting hypercall return code");
+
     /* Success! */
     rc = 0;

diff -r f89126a59073 -r fbe5ab1ecd21 tools/libxc/xc_linux_save.c
@@ -1181,6 +1181,12 @@ int xc_linux_save(int xc_handle, int io_
         goto out;
     }
 
+    /* set hypercall return code to indicate domain is resuming */
+    ctxt.user_regs.eax = 1;
+    rc = xc_vcpu_setcontext(xc_handle, dom, 0, &ctxt);
+    if (rc)
+        ERROR("Error setting hypercall return code");
+
     /* Success! */
     rc = 0;
 

diff -r f89126a59073 -r 2048d2698d46 tools/libxc/xc_hvm_build.c
@@ -270,7 +270,6 @@ static int xc_hvm_build_internal(int xc_
                                  char *image,
                                  unsigned long image_size)
 {
-    struct xen_domctl launch_domctl;
     vcpu_guest_context_t ctxt;
     int rc;
 

@@ -287,20 +286,7 @@ static int xc_hvm_build_internal(int xc_
         goto error_out;
     }
 
-    if ( lock_pages(&ctxt, sizeof(ctxt) ) )
-    {
-        PERROR("%s: ctxt mlock failed", __func__);
-        goto error_out;
-    }
-
-    memset(&launch_domctl, 0, sizeof(launch_domctl));
-    launch_domctl.domain = (domid_t)domid;
-    launch_domctl.u.vcpucontext.vcpu   = 0;
-    set_xen_guest_handle(launch_domctl.u.vcpucontext.ctxt, &ctxt);
-    launch_domctl.cmd = XEN_DOMCTL_setvcpucontext;
-    rc = xc_domctl(xc_handle, &launch_domctl);
-
-    unlock_pages(&ctxt, sizeof(ctxt));
+    rc = xc_vcpu_setcontext(xc_handle, domid, 0, &ctxt);
 
     return rc;
 

diff -r f89126a59073 -r 2048d2698d46 tools/libxc/xc_linux_build.c
@@ -1092,7 +1092,6 @@ static int xc_linux_build_internal(int x
                                    unsigned int console_evtchn,
                                    unsigned long *console_mfn)
 {
-    struct xen_domctl launch_domctl;
     DECLARE_DOMCTL;
     int rc;
     struct vcpu_guest_context st_ctxt, *ctxt = &st_ctxt;

@@ -1109,12 +1108,6 @@ static int xc_linux_build_internal(int x
     }
 
     memset(ctxt, 0, sizeof(*ctxt));
-
-    if ( lock_pages(ctxt, sizeof(*ctxt) ) )
-    {
-        PERROR("%s: ctxt lock failed", __func__);
-        return 1;
-    }
 
     domctl.cmd = XEN_DOMCTL_getdomaininfo;
     domctl.domain = (domid_t)domid;

@@ -1170,14 +1163,7 @@ static int xc_linux_build_internal(int x
     ctxt->kernel_sp = ctxt->user_regs.esp;
 #endif /* x86 */
 
-    memset(&launch_domctl, 0, sizeof(launch_domctl));
-
-    launch_domctl.domain = (domid_t)domid;
-    launch_domctl.u.vcpucontext.vcpu   = 0;
-    set_xen_guest_handle(launch_domctl.u.vcpucontext.ctxt, ctxt);
-
-    launch_domctl.cmd = XEN_DOMCTL_setvcpucontext;
-    rc = xc_domctl(xc_handle, &launch_domctl);
+    rc = xc_vcpu_setcontext(xc_handle, domid, 0, ctxt);
 
     return rc;
 

diff -r f89126a59073 -r 2048d2698d46 tools/libxc/xc_linux_restore.c
@@ -195,12 +195,6 @@ int xc_linux_restore(int xc_handle, int 
     if(!get_platform_info(xc_handle, dom,
                           &max_mfn, &hvirt_start, &pt_levels)) {
         ERROR("Unable to get platform info.");
-        return 1;
-    }
-
-    if (lock_pages(&ctxt, sizeof(ctxt))) {
-        /* needed for build domctl, but might as well do early */
-        ERROR("Unable to lock ctxt");
         return 1;
     }
 

@@ -845,12 +839,7 @@ int xc_linux_restore(int xc_handle, int 
 
     DPRINTF("Domain ready to be built.\n");
 
-    domctl.cmd = XEN_DOMCTL_setvcpucontext;
-    domctl.domain = (domid_t)dom;
-    domctl.u.vcpucontext.vcpu   = 0;
-    set_xen_guest_handle(domctl.u.vcpucontext.ctxt, &ctxt);
-    rc = xc_domctl(xc_handle, &domctl);
-
+    rc = xc_vcpu_setcontext(xc_handle, dom, 0, &ctxt);
     if (rc != 0) {
         ERROR("Couldn't build the domain");
         goto out;

diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c
@@ -85,13 +85,20 @@ static void pre_suspend(void)
 		mfn_to_pfn(xen_start_info->console.domU.mfn);
 }
 
-static void post_suspend(void)
+static void post_suspend(int reconnect)
 {
 	int i, j, k, fpp;
 	extern unsigned long max_pfn;
 	extern unsigned long *pfn_to_mfn_frame_list_list;
 	extern unsigned long *pfn_to_mfn_frame_list[];
 
+	if (!reconnect) {
+		xen_start_info->store_mfn =
+			pfn_to_mfn(xen_start_info->store_mfn);
+		xen_start_info->console.domU.mfn =
+			pfn_to_mfn(xen_start_info->console.domU.mfn);
+	}
+	
 	set_fixmap(FIX_SHARED_INFO, xen_start_info->shared_info);
 
 	HYPERVISOR_shared_info = (shared_info_t *)fix_to_virt(FIX_SHARED_INFO);

@@ -120,7 +127,7 @@ static void post_suspend(void)
 #define switch_idle_mm()	((void)0)
 #define mm_pin_all()		((void)0)
 #define pre_suspend()		((void)0)
-#define post_suspend()		((void)0)
+#define post_suspend(x)		((void)0)
 
 #endif
 

@@ -158,16 +165,18 @@ int __xen_suspend(void)
 	pre_suspend();
 
 	/*
-	 * We'll stop somewhere inside this hypercall. When it returns,
-	 * we'll start resuming after the restore.
+	 * This hypercall returns 1 if suspend was cancelled or
+	 * the domain was merely checkpointed, and 0 if it is
+	 * resuming in a new domain.
 	 */
-	HYPERVISOR_suspend(virt_to_mfn(xen_start_info));
+	err = !HYPERVISOR_suspend(virt_to_mfn(xen_start_info));
 
-	post_suspend();
+	post_suspend(err);
 
 	gnttab_resume();
 
-	irq_resume();
+	if (err)
+		irq_resume();
 
 	time_resume();
 

@@ -175,9 +184,10 @@ int __xen_suspend(void)
 
 	local_irq_enable();
 
-	xencons_resume();
+	if (err)
+		xencons_resume();
 
-	xenbus_resume();
+	xenbus_resume(err);
 
 	smp_resume();
 

diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
@@ -727,11 +727,15 @@ void xenbus_suspend(void)
 }
 EXPORT_SYMBOL_GPL(xenbus_suspend);
 
-void xenbus_resume(void)
-{
-	xb_init_comms();
-	xs_resume();
-	bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+void xenbus_resume(int reconnect)
+{
+	if (reconnect)
+		xb_init_comms();
+	xs_resume(reconnect);
+
+	if (reconnect) {
+		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+	}
 	xenbus_backend_resume(resume_dev);
 }
 EXPORT_SYMBOL_GPL(xenbus_resume);

diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c
@@ -668,31 +668,23 @@ EXPORT_SYMBOL_GPL(unregister_xenbus_watc
 
 void xs_suspend(void)
 {
+	down_write(&xs_state.suspend_mutex);
+	mutex_lock(&xs_state.request_mutex);
+}
+
+void xs_resume(int reconnect)
+{
 	struct xenbus_watch *watch;
 	char token[sizeof(watch) * 2 + 1];
 
-	down_write(&xs_state.suspend_mutex);
-
-	/* No need for watches_lock: the suspend_mutex is sufficient. */
-	list_for_each_entry(watch, &watches, list) {
-		sprintf(token, "%lX", (long)watch);
-		xs_unwatch(watch->node, token);
-	}
-
-	mutex_lock(&xs_state.request_mutex);
-}
-
-void xs_resume(void)
-{
-	struct xenbus_watch *watch;
-	char token[sizeof(watch) * 2 + 1];
-
 	mutex_unlock(&xs_state.request_mutex);
 
-	/* No need for watches_lock: the suspend_mutex is sufficient. */
-	list_for_each_entry(watch, &watches, list) {
-		sprintf(token, "%lX", (long)watch);
-		xs_watch(watch->node, token);
+	if (reconnect) {
+		/* No need for watches_lock: the suspend_mutex is sufficient. */
+		list_for_each_entry(watch, &watches, list) {
+			sprintf(token, "%lX", (long)watch);
+			xs_watch(watch->node, token);
+		}
 	}
 
 	up_write(&xs_state.suspend_mutex);

diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/include/xen/xenbus.h
@@ -159,14 +159,14 @@ int register_xenbus_watch(struct xenbus_
 int register_xenbus_watch(struct xenbus_watch *watch);
 void unregister_xenbus_watch(struct xenbus_watch *watch);
 void xs_suspend(void);
-void xs_resume(void);
+void xs_resume(int reconnect);
 
 /* Used by xenbus_dev to borrow kernel's store connection. */
 void *xenbus_dev_request_and_reply(struct xsd_sockmsg *msg);
 
 /* Called from xen core code. */
 void xenbus_suspend(void);
-void xenbus_resume(void);
+void xenbus_resume(int reconnect);
 
 #define XENBUS_IS_ERR_READ(str) ({			\
 	if (!IS_ERR(str) && strlen(str) == 0) {		\

diff -r dafb7b444c2c -r 1a3b78919a43 linux-2.6-xen-sparse/drivers/xen/core/reboot.c
@@ -204,6 +204,8 @@ static int setup_shutdown_watcher(struct
 	else
 		xenbus_write(XBT_NIL, "control", "feature-sysrq", "1");
 
+	xenbus_write(XBT_NIL, "control", "feature-resumable", "1");
+
 	return NOTIFY_DONE;
 }
 

diff -r 1a3b78919a43 -r 70508fff24c5 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
@@ -672,6 +672,26 @@ static int suspend_dev(struct device *de
 	return 0;
 }
 
+static int unsuspend_dev(struct device *dev, void *data)
+{
+	int err = 0;
+	struct xenbus_driver *drv;
+	struct xenbus_device *xdev;
+
+	DPRINTK("");
+
+	if (dev->driver == NULL)
+		return 0;
+	drv = to_xenbus_driver(dev->driver);
+	xdev = container_of(dev, struct xenbus_device, dev);
+	if (drv->unsuspend)
+		err = drv->unsuspend(xdev);
+	if (err)
+		printk(KERN_WARNING
+		       "xenbus: unsuspend %s failed: %i\n", dev->bus_id, err);
+	return 0;
+}
+
 static int resume_dev(struct device *dev, void *data)
 {
 	int err;

@@ -735,6 +755,9 @@ void xenbus_resume(int reconnect)
 
 	if (reconnect) {
 		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+	} else {
+		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL,
+				 unsuspend_dev);
 	}
 	xenbus_backend_resume(resume_dev);
 }

diff -r 1a3b78919a43 -r 70508fff24c5 linux-2.6-xen-sparse/include/xen/xenbus.h
@@ -101,6 +101,7 @@ struct xenbus_driver {
 				 enum xenbus_state backend_state);
 	int (*remove)(struct xenbus_device *dev);
 	int (*suspend)(struct xenbus_device *dev);
+	int (*unsuspend)(struct xenbus_device *dev);
 	int (*resume)(struct xenbus_device *dev);
 	int (*uevent)(struct xenbus_device *, char **, int, char *, int);
 	struct device_driver driver;