Teach xm save to checkpoint a running domain

diff -r a84fc0de350d -r 4514d8e0843c xen/common/domctl.c
@@ -250,6 +250,31 @@ ret_t do_domctl(XEN_GUEST_HANDLE(xen_dom
     }
     break;
 
+    case XEN_DOMCTL_resumedomain:
+    {
+        struct domain *d = find_domain_by_id(op->domain);
+        struct vcpu *v;
+
+        ret = -ESRCH;
+        if ( d != NULL )
+        {
+            ret = -EINVAL;
+            printk("Resuming domain %d\n", op->domain);
+            if ( (d != current->domain) && (d->vcpu[0] != NULL) &&
+                 test_bit(_DOMF_shutdown, &d->domain_flags) )
+            {
+                clear_bit(_DOMF_shutdown, &d->domain_flags);
+
+                for_each_vcpu (d, v)
+                    vcpu_wake (v);
+
+                ret = 0;
+            }
+            put_domain(d);
+        }
+    }
+    break;
+
     case XEN_DOMCTL_createdomain:
     {
         struct domain *d;

diff -r a84fc0de350d -r 4514d8e0843c xen/include/public/domctl.h
@@ -63,6 +63,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_domctl_creat
 #define XEN_DOMCTL_destroydomain      2
 #define XEN_DOMCTL_pausedomain        3
 #define XEN_DOMCTL_unpausedomain      4
+#define XEN_DOMCTL_resumedomain      26
 
 #define XEN_DOMCTL_getdomaininfo      5
 struct xen_domctl_getdomaininfo {

diff -r 8fdd24d1dfbf -r 74431b023b3c xen/common/domctl.c
@@ -250,6 +250,31 @@ ret_t do_domctl(XEN_GUEST_HANDLE(xen_dom
     }
     break;
 
+    case XEN_DOMCTL_resumedomain:
+    {
+        struct domain *d = find_domain_by_id(op->domain);
+        struct vcpu *v;
+
+        ret = -ESRCH;
+        if ( d != NULL )
+        {
+            ret = -EINVAL;
+            printk("Resuming domain %d\n", op->domain);
+            if ( (d != current->domain) && (d->vcpu[0] != NULL) &&
+                 test_bit(_DOMF_shutdown, &d->domain_flags) )
+            {
+                clear_bit(_DOMF_shutdown, &d->domain_flags);
+
+                for_each_vcpu (d, v)
+                    vcpu_wake (v);
+
+                ret = 0;
+            }
+            put_domain(d);
+        }
+    }
+    break;
+
     case XEN_DOMCTL_createdomain:
     {
         struct domain *d;

diff -r 8fdd24d1dfbf -r 74431b023b3c xen/include/public/domctl.h
@@ -63,6 +63,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_domctl_creat
 #define XEN_DOMCTL_destroydomain      2
 #define XEN_DOMCTL_pausedomain        3
 #define XEN_DOMCTL_unpausedomain      4
+#define XEN_DOMCTL_resumedomain      27
 
 #define XEN_DOMCTL_getdomaininfo      5
 struct xen_domctl_getdomaininfo {

diff -r 4514d8e0843c -r 30e07c0e462b tools/libxc/xc_domain.c
@@ -86,6 +86,16 @@ int xc_domain_shutdown(int xc_handle,
 
  out1:
     return ret;
+}
+
+
+int xc_domain_resume(int xc_handle,
+                      uint32_t domid)
+{
+    DECLARE_DOMCTL;
+    domctl.cmd = XEN_DOMCTL_resumedomain;
+    domctl.domain = (domid_t)domid;
+    return do_domctl(xc_handle, &domctl);
 }
 
 

diff -r 4514d8e0843c -r 30e07c0e462b tools/libxc/xenctrl.h
@@ -236,6 +236,18 @@ int xc_domain_destroy(int xc_handle,
 int xc_domain_destroy(int xc_handle,
                       uint32_t domid);
 
+
+/**
+ * This function resumes a suspended domain. The domain should have
+ * been previously suspended.
+ *
+ * @parm xc_handle a handle to an open hypervisor interface
+ * @parm domid the domain id to resume
+ * return 0 on success, -1 on failure
+ */
+int xc_domain_resume(int xc_handle,
+                      uint32_t domid);
+
 /**
  * This function will shutdown a domain. This is intended for use in
  * fully-virtualized domains where this operation is analogous to the

diff -r 30e07c0e462b -r 2cdbd7c62a0e tools/python/xen/lowlevel/xc/xc.c
@@ -160,6 +160,10 @@ static PyObject *pyxc_domain_destroy(XcO
     return dom_op(self, args, xc_domain_destroy);
 }
 
+static PyObject *pyxc_domain_resume(XcObject *self, PyObject *args)
+{
+    return dom_op(self, args, xc_domain_resume);
+}
 
 static PyObject *pyxc_vcpu_setaffinity(XcObject *self,
                                        PyObject *args,

@@ -1027,6 +1031,13 @@ static PyMethodDef pyxc_methods[] = {
       METH_VARARGS, "\n"
       "Destroy a domain.\n"
       " dom [int]:    Identifier of domain to be destroyed.\n\n"
+      "Returns: [int] 0 on success; -1 on error.\n" },
+    
+    { "domain_resume", 
+      (PyCFunction)pyxc_domain_resume, 
+      METH_VARARGS, "\n"
+      "Resume execution of a suspended domain.\n"
+      " dom [int]: Identifier of domain to be resumed.\n\n"
       "Returns: [int] 0 on success; -1 on error.\n" },
 
     { "vcpu_setaffinity", 

diff -r 2cdbd7c62a0e -r c6154e7b94ab tools/xenstore/xenstored_core.c
@@ -164,6 +164,7 @@ static char *sockmsg_string(enum xsd_soc
 	case XS_WATCH_EVENT: return "WATCH_EVENT";
 	case XS_ERROR: return "ERROR";
 	case XS_IS_DOMAIN_INTRODUCED: return "XS_IS_DOMAIN_INTRODUCED";
+	case XS_RESUME: return "RESUME";
 	default:
 		return "**UNKNOWN**";
 	}

@@ -1265,6 +1266,10 @@ static void process_message(struct conne
 
 	case XS_GET_DOMAIN_PATH:
 		do_get_domain_path(conn, onearg(in));
+		break;
+
+	case XS_RESUME:
+		do_resume(conn, onearg(in));
 		break;
 
 	default:

diff -r 2cdbd7c62a0e -r c6154e7b94ab tools/xenstore/xenstored_domain.c
@@ -395,6 +395,43 @@ void do_release(struct connection *conn,
 	send_ack(conn, XS_RELEASE);
 }
 
+void do_resume(struct connection *conn, const char *domid_str)
+{
+	struct domain *domain;
+	unsigned int domid;
+
+	if (!domid_str) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	domid = atoi(domid_str);
+	if (!domid) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	if (conn->id != 0) {
+		send_error(conn, EACCES);
+		return;
+	}
+
+	domain = find_domain_by_domid(domid);
+	if (!domain) {
+		send_error(conn, ENOENT);
+		return;
+	}
+
+	if (!domain->conn) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	domain->shutdown = 0;
+	
+	send_ack(conn, XS_RESUME);
+}
+
 void do_get_domain_path(struct connection *conn, const char *domid_str)
 {
 	char *path;

diff -r 2cdbd7c62a0e -r c6154e7b94ab tools/xenstore/xenstored_domain.h
@@ -32,6 +32,9 @@ void do_release(struct connection *conn,
 void do_release(struct connection *conn, const char *domid_str);
 
 /* domid */
+void do_resume(struct connection *conn, const char *domid_str);
+
+/* domid */
 void do_get_domain_path(struct connection *conn, const char *domid_str);
 
 /* Returns the event channel handle */

diff -r 2cdbd7c62a0e -r c6154e7b94ab tools/xenstore/xs.c
@@ -719,6 +719,12 @@ bool xs_release_domain(struct xs_handle 
 	return xs_bool(single_with_domid(h, XS_RELEASE, domid));
 }
 
+/* clear the shutdown bit for the given domain */
+bool xs_resume_domain(struct xs_handle *h, unsigned int domid)
+{
+	return xs_bool(single_with_domid(h, XS_RESUME, domid));
+}
+
 char *xs_get_domain_path(struct xs_handle *h, unsigned int domid)
 {
 	char domid_str[MAX_STRLEN(domid)];

diff -r 2cdbd7c62a0e -r c6154e7b94ab tools/xenstore/xs.h
@@ -133,6 +133,11 @@ bool xs_introduce_domain(struct xs_handl
 			 unsigned int domid,
 			 unsigned long mfn,
                          unsigned int eventchn); 
+/* Resume a domain.
+ * Clear the shutdown flag for this domain in the store.
+ */
+bool xs_resume_domain(struct xs_handle *h, unsigned int domid);
+
 /* Release a domain.
  * Tells the store domain to release the memory page to the domain.
  */

diff -r 2cdbd7c62a0e -r c6154e7b94ab xen/include/public/io/xs_wire.h
@@ -45,7 +45,8 @@ enum xsd_sockmsg_type
     XS_SET_PERMS,
     XS_WATCH_EVENT,
     XS_ERROR,
-    XS_IS_DOMAIN_INTRODUCED
+    XS_IS_DOMAIN_INTRODUCED,
+    XS_RESUME
 };
 
 #define XS_WRITE_NONE "NONE"

diff -r c6154e7b94ab -r 5ae12d620c6d tools/python/xen/lowlevel/xs/xs.c
@@ -618,6 +618,33 @@ static PyObject *xspy_introduce_domain(X
     return none(result);
 }
 
+#define xspy_resume_domain_doc "\n"                                \
+	"Tell xenstore to clear its shutdown flag for a domain.\n" \
+	"This ensures that a subsequent shutdown will fire the\n"  \
+	"appropriate watches.\n"                                   \
+	" dom [int]: domain id\n"			           \
+        "\n"						           \
+        "Returns None on success.\n"				   \
+        "Raises xen.lowlevel.xs.Error on error.\n"
+
+static PyObject *xspy_resume_domain(XsHandle *self, PyObject *args)
+{
+    uint32_t dom;
+
+    struct xs_handle *xh = xshandle(self);
+    bool result = 0;
+
+    if (!xh)
+        return NULL;
+    if (!PyArg_ParseTuple(args, "i", &dom))
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    result = xs_resume_domain(xh, dom);
+    Py_END_ALLOW_THREADS
+
+    return none(result);
+}
 
 #define xspy_release_domain_doc "\n"					\
 	"Tell xenstore to release its channel to a domain.\n"		\

@@ -789,6 +816,7 @@ static PyMethodDef xshandle_methods[] = 
     XSPY_METH(transaction_start, METH_NOARGS),
     XSPY_METH(transaction_end,   METH_VARARGS | METH_KEYWORDS),
     XSPY_METH(introduce_domain,  METH_VARARGS),
+    XSPY_METH(resume_domain,     METH_VARARGS),
     XSPY_METH(release_domain,    METH_VARARGS),
     XSPY_METH(close,             METH_NOARGS),
     XSPY_METH(get_domain_path,   METH_VARARGS),

diff -r 5ae12d620c6d -r b5414e713c97 tools/libxc/xc_linux_restore.c
@@ -690,6 +690,8 @@ int xc_linux_restore(int xc_handle, int 
         ERROR("Suspend record frame number is bad");
         goto out;
     }
+    /* HYPERVISOR_suspend returns 1 to let guest know it should reconnect */
+    ctxt.user_regs.eax = 1;
     ctxt.user_regs.edx = mfn = p2m[pfn];
     start_info = xc_map_foreign_range(
         xc_handle, dom, PAGE_SIZE, PROT_READ | PROT_WRITE, mfn);

diff -r b5414e713c97 -r b10287396924 linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c
@@ -85,13 +85,20 @@ static void pre_suspend(void)
 		mfn_to_pfn(xen_start_info->console.domU.mfn);
 }
 
-static void post_suspend(void)
+static void post_suspend(int reconnect)
 {
 	int i, j, k, fpp;
 	extern unsigned long max_pfn;
 	extern unsigned long *pfn_to_mfn_frame_list_list;
 	extern unsigned long *pfn_to_mfn_frame_list[];
 
+	if (!reconnect) {
+		xen_start_info->store_mfn =
+			pfn_to_mfn(xen_start_info->store_mfn);
+		xen_start_info->console.domU.mfn =
+			pfn_to_mfn(xen_start_info->console.domU.mfn);
+	}
+	
 	set_fixmap(FIX_SHARED_INFO, xen_start_info->shared_info);
 
 	HYPERVISOR_shared_info = (shared_info_t *)fix_to_virt(FIX_SHARED_INFO);

@@ -120,7 +127,7 @@ static void post_suspend(void)
 #define switch_idle_mm()	((void)0)
 #define mm_pin_all()		((void)0)
 #define pre_suspend()		((void)0)
-#define post_suspend()		((void)0)
+#define post_suspend(x)		((void)0)
 
 #endif
 

@@ -158,16 +165,18 @@ int __xen_suspend(void)
 	pre_suspend();
 
 	/*
-	 * We'll stop somewhere inside this hypercall. When it returns,
-	 * we'll start resuming after the restore.
+	 * This hypercall returns 0 if suspend was cancelled or
+	 * the domain was merely checkpointed, and 1 if it is
+	 * resuming in a new domain.
 	 */
-	HYPERVISOR_suspend(virt_to_mfn(xen_start_info));
+	err = HYPERVISOR_suspend(virt_to_mfn(xen_start_info));
 
-	post_suspend();
+	post_suspend(err);
 
 	gnttab_resume();
 
-	irq_resume();
+	if (err)
+		irq_resume();
 
 	time_resume();
 

@@ -175,9 +184,10 @@ int __xen_suspend(void)
 
 	local_irq_enable();
 
-	xencons_resume();
+	if (err)
+		xencons_resume();
 
-	xenbus_resume();
+	xenbus_resume(err);
 
 	smp_resume();
 

diff -r b5414e713c97 -r b10287396924 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
@@ -727,11 +727,15 @@ void xenbus_suspend(void)
 }
 EXPORT_SYMBOL_GPL(xenbus_suspend);
 
-void xenbus_resume(void)
-{
-	xb_init_comms();
-	xs_resume();
-	bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+void xenbus_resume(int reconnect)
+{
+	if (reconnect)
+		xb_init_comms();
+	xs_resume(reconnect);
+
+	if (reconnect) {
+		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+	}
 	xenbus_backend_resume(resume_dev);
 }
 EXPORT_SYMBOL_GPL(xenbus_resume);

diff -r b5414e713c97 -r b10287396924 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c
@@ -668,31 +668,23 @@ EXPORT_SYMBOL_GPL(unregister_xenbus_watc
 
 void xs_suspend(void)
 {
+	down_write(&xs_state.suspend_mutex);
+	mutex_lock(&xs_state.request_mutex);
+}
+
+void xs_resume(int reconnect)
+{
 	struct xenbus_watch *watch;
 	char token[sizeof(watch) * 2 + 1];
 
-	down_write(&xs_state.suspend_mutex);
-
-	/* No need for watches_lock: the suspend_mutex is sufficient. */
-	list_for_each_entry(watch, &watches, list) {
-		sprintf(token, "%lX", (long)watch);
-		xs_unwatch(watch->node, token);
-	}
-
-	mutex_lock(&xs_state.request_mutex);
-}
-
-void xs_resume(void)
-{
-	struct xenbus_watch *watch;
-	char token[sizeof(watch) * 2 + 1];
-
 	mutex_unlock(&xs_state.request_mutex);
 
-	/* No need for watches_lock: the suspend_mutex is sufficient. */
-	list_for_each_entry(watch, &watches, list) {
-		sprintf(token, "%lX", (long)watch);
-		xs_watch(watch->node, token);
+	if (reconnect) {
+		/* No need for watches_lock: the suspend_mutex is sufficient. */
+		list_for_each_entry(watch, &watches, list) {
+			sprintf(token, "%lX", (long)watch);
+			xs_watch(watch->node, token);
+		}
 	}
 
 	up_write(&xs_state.suspend_mutex);

diff -r b5414e713c97 -r b10287396924 linux-2.6-xen-sparse/include/xen/xenbus.h
@@ -159,14 +159,14 @@ int register_xenbus_watch(struct xenbus_
 int register_xenbus_watch(struct xenbus_watch *watch);
 void unregister_xenbus_watch(struct xenbus_watch *watch);
 void xs_suspend(void);
-void xs_resume(void);
+void xs_resume(int reconnect);
 
 /* Used by xenbus_dev to borrow kernel's store connection. */
 void *xenbus_dev_request_and_reply(struct xsd_sockmsg *msg);
 
 /* Called from xen core code. */
 void xenbus_suspend(void);
-void xenbus_resume(void);
+void xenbus_resume(int reconnect);
 
 #define XENBUS_IS_ERR_READ(str) ({			\
 	if (!IS_ERR(str) && strlen(str) == 0) {		\