HVM save restore: PV driver support

[PATCH 8/8] HVM save restore: PV driver support

Signed-off-by Zhai Edwin <[email protected]>

enable PV driver's save/restore in HVM domain by:
HV:
* send a pseudo PCI dev intr to guest in restore
* rebuild the shared info on behalf of HVM guest
* set a resume flag in shared info

Guest:
* check the resmume flag in the pseudo PCI dev intr handler
* if set do the PV driver resume work

diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c

linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c +4 -0

@@ -138,6 +138,10 @@ static int blkfront_resume(struct xenbus
 
 	DPRINTK("blkfront_resume: %s\n", dev->nodename);
 
+ 	/* resuming qemu disk would cause error when restore and skip next vbd resume */
+ 	if (info->connected == BLKIF_STATE_DISCONNECTED)
+ 		return 0;
+ 
 	blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
 
 	err = talk_to_backend(dev, info);

diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c

linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c +10 -1

@@ -183,7 +183,16 @@ xlbd_put_major_info(struct xlbd_major_in
 xlbd_put_major_info(struct xlbd_major_info *mi)
 {
 	mi->usage--;
-	/* XXX: release major if 0 */
+	/* release major if 0 */
+	if (mi->usage)
+		return;
+
+	printk("Unregistering block device major %i\n", mi->major);
+	if (unregister_blkdev(mi->major, mi->type->devname)) {
+		WPRINTK("can't put major %d with name %s\n",
+			mi->major, mi->type->devname);
+		kfree(mi);
+	}
 }
 
 static int

diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c

linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c +2 -0

@@ -815,6 +815,7 @@ static int xsd_port_read(char *page, cha
 }
 #endif
 
+extern void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void));
 static int __init xenbus_probe_init(void)
 {
 	int err = 0;

@@ -882,6 +883,7 @@ static int __init xenbus_probe_init(void
 		xen_store_mfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
 		xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT,
 					      PAGE_SIZE);
+ 		set_restore_handler(xenbus_suspend, xenbus_resume);
 #endif
 	}
 

diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/evtchn.c

unmodified_drivers/linux-2.6/platform-pci/evtchn.c +11 -0

@@ -165,6 +165,7 @@ void notify_remote_via_irq(int irq)
 }
 EXPORT_SYMBOL(notify_remote_via_irq);
 
+extern void pvdrv_restore(void);
 irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs)
 {
 	unsigned int l1i, port;

@@ -173,6 +174,16 @@ irqreturn_t evtchn_interrupt(int irq, vo
 	shared_info_t *s = shared_info_area;
 	vcpu_info_t *v = &s->vcpu_info[cpu];
 	unsigned long l1, l2;
+
+	/* add a check to see if need resume after restore */ 
+	if (s->pvdrv_resume == 0x58585858) {
+		printk("evtchn_interrupt:resume PV driver.\n");
+		s->pvdrv_resume= 0;
+		v->evtchn_upcall_pending = 0;
+
+		pvdrv_restore();
+		return IRQ_HANDLED;
+	}
 
 	v->evtchn_upcall_pending = 0;
 	/* NB. No need for a barrier here -- XCHG is a barrier on x86. */

diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/platform-pci.c

unmodified_drivers/linux-2.6/platform-pci/platform-pci.c +77 -0

@@ -28,6 +28,7 @@
 #include <linux/interrupt.h>
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
+#include <linux/kthread.h>
 #include <asm/system.h>
 #include <asm/io.h>
 #include <asm/irq.h>

@@ -280,9 +281,85 @@ static struct pci_driver platform_driver
 
 static int pci_device_registered;
 
+extern int gnttab_suspend(void);
+void platform_pci_suspend(void)
+{
+	gnttab_suspend();
+}
+
+extern int gnttab_resume(void);
+void platform_pci_resume(void)
+{
+	phys_to_machine_mapping = NULL;
+
+	gnttab_resume();
+}
+
+/* support for PV driver save/restore in HVM domain*/
+void (*suspend_handler)(void);
+void (*resume_handler)(void);
+void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void))
+{
+	suspend_handler = hdl1;
+	resume_handler  = hdl2;
+}
+EXPORT_SYMBOL(set_restore_handler);
+
+/* scheduled suspend&resume */
+static void restore_handler(void *unused);
+static int __do_pvdrv_restore(void *ignored);
+
+static DECLARE_WORK(restore_work, restore_handler, NULL);
+
+static int kthread_create_on_cpu(int (*f)(void *arg),
+	void *arg,
+	const char *name,
+	int cpu)
+{
+	struct task_struct *p;
+	p = kthread_create(f, arg, name);
+	if (IS_ERR(p))
+		return PTR_ERR(p);
+	kthread_bind(p, cpu);
+	wake_up_process(p);
+	return 0;
+}
+
+static int __do_pvdrv_restore(void *ignored)
+{
+	if (suspend_handler)
+		suspend_handler();
+	platform_pci_suspend();
+
+	platform_pci_resume();
+	if (resume_handler)
+		resume_handler();
+
+	return 0;
+}
+
+static void restore_handler(void *unused)
+{
+	int err;
+
+	err = kthread_create_on_cpu(__do_pvdrv_restore, NULL, "pvdrv_suspend", 0);
+	if (err < 0) {
+		printk(KERN_WARNING "error creating PV driver suspend process (%d): retrying...\n",
+			err);
+		schedule_delayed_work(&restore_work, 50/2);
+	}
+}
+
+void pvdrv_restore(void)
+{
+	schedule_work(&restore_work);
+}
+
 static int __init platform_pci_module_init(void)
 {
 	int rc;
+
+	set_restore_handler(NULL, NULL);
 
 	rc = pci_module_init(&platform_driver);
 	if (rc)

diff -r d18c6a3c676a xen/arch/x86/hvm/hvm.c

xen/arch/x86/hvm/hvm.c +3 -0

@@ -200,6 +200,9 @@ int hvm_vcpu_initialise(struct vcpu *v)
 
     /* init hvm sharepage */
     shpage_init(v->domain, get_sp(v->domain));
+
+    /* other hvm info need for save/restore */
+    hvminfo_init(v->domain);
 
     /* Init guest TSC to start from zero. */
     hvm_set_guest_time(v, 0);

diff -r d18c6a3c676a xen/arch/x86/hvm/intercept.c

xen/arch/x86/hvm/intercept.c +65 -0

@@ -445,6 +445,71 @@ void shpage_init(struct domain *d, share
     hvm_register_savevm(d, "xen_hvm_shpage", 0x10, 1, shpage_save, shpage_load, sp);
 }
 
+void hvminfo_print(struct domain* d)
+{
+    printk("********hvm other info***********\n");
+    printk("callback irq=%"PRId64".\n", d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
+    printk("shared info pfn=0x%"PRIx64".\n", d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
+}
+
+static void hvminfo_save(hvm_domain_context_t *h, void *opaque)
+{
+    struct domain *d = opaque;
+
+#ifdef HVM_DEBUG_SUSPEND
+    hvminfo_print(d);
+#endif
+
+    hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
+    hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
+}
+
+extern int rebuild_shared_info(struct domain *d, unsigned long share_info_pfn);
+static int hvminfo_load(hvm_domain_context_t *h, void *opaque, int version_id)
+{
+    struct domain *d = opaque;
+    int callback_irq = hvm_get_64u(h);
+    unsigned long share_info_pfn = hvm_get_64u(h);
+
+    if (callback_irq == 0) {
+        return 0;
+    }
+
+    d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ] = callback_irq;
+
+    /* reconstruct the share info for HVM guest */
+    rebuild_shared_info(d, share_info_pfn);
+    d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = share_info_pfn;
+
+    /* set resume state to notify HVM PV driver */
+    shared_info(d, pvdrv_resume) = 0x58585858;
+
+    /* set all the evtchn_upcall_pending on all vcpus */
+    if (callback_irq) {
+        struct vcpu *v;
+
+        for_each_vcpu(d, v) {
+            if ( !test_and_set_bit(0, &vcpu_info(v, evtchn_upcall_pending)) )
+                printk("on vcpu %d raise a vir level intr %d to restore PV driver in HVM guest!\n",
+                        v->vcpu_id,
+                        callback_irq);
+        }
+    }
+
+
+#ifdef HVM_DEBUG_SUSPEND
+    hvminfo_print(d);
+#endif
+
+    return 0;
+
+}
+
+void hvminfo_init(struct domain* d)
+{
+    hvm_register_savevm(d, "xen_hvm_other_info", 0x11, 1, hvminfo_save, hvminfo_load, d);
+}
+
 int hvm_buffered_io_intercept(ioreq_t *p)
 {
     struct vcpu *v = current;

diff -r d18c6a3c676a xen/arch/x86/mm.c

xen/arch/x86/mm.c +38 -0

@@ -3000,6 +3000,7 @@ long arch_memory_op(int op, XEN_GUEST_HA
         case XENMAPSPACE_shared_info:
             if ( xatp.idx == 0 )
                 mfn = virt_to_mfn(d->shared_info);
+            d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = xatp.gpfn;
             break;
         case XENMAPSPACE_grant_table:
             if ( xatp.idx < NR_GRANT_FRAMES )

@@ -3143,6 +3144,43 @@ long arch_memory_op(int op, XEN_GUEST_HA
     return 0;
 }
 
+/* reconstruct the share info for HVM guest */
+int rebuild_shared_info(struct domain *d, unsigned long share_info_pfn)
+{
+    unsigned long mfn, prev_mfn, gpfn = 0;
+
+    mfn = virt_to_mfn(d->shared_info);
+    if ( !shadow_mode_translate(d) || (mfn == 0)) {
+        printk("reconstruct share info for HVM guest failed!\n");
+        return -1;
+    }
+
+    LOCK_BIGLOCK(d);
+
+    /* Remove previously mapped page if it was present. */
+    prev_mfn = gmfn_to_mfn(d, share_info_pfn);
+    if ( mfn_valid(prev_mfn) )
+    {
+        if ( IS_XEN_HEAP_FRAME(mfn_to_page(prev_mfn)) )
+            /* Xen heap frames are simply unhooked from this phys slot. */
+            guest_physmap_remove_page(d, share_info_pfn, prev_mfn);
+        else
+            /* Normal domain memory is freed, to avoid leaking memory. */
+            guest_remove_page(d, share_info_pfn);
+    }
+
+    /* Unmap from old location, if any. */
+    gpfn = get_gpfn_from_mfn(mfn);
+    if ( gpfn != INVALID_M2P_ENTRY )
+        guest_physmap_remove_page(d, gpfn, mfn);
+
+    /* Map at new location. */
+    guest_physmap_add_page(d, share_info_pfn, mfn);
+
+    UNLOCK_BIGLOCK(d);
+
+    return 0;
+}
 
 /*************************
  * Writable Pagetables

diff -r d18c6a3c676a xen/include/asm-x86/hvm/support.h

xen/include/asm-x86/hvm/support.h +2 -0

@@ -248,6 +248,8 @@ extern int arch_gethvm_ctxt(struct vcpu 
 
 extern void shpage_init(struct domain *d, shared_iopage_t *sp);
 
+extern void hvminfo_init(struct domain* d);
+
 extern int hvm_enabled;
 
 int hvm_copy_to_guest_phys(paddr_t paddr, void *buf, int size);

diff -r d18c6a3c676a xen/include/public/hvm/params.h

xen/include/public/hvm/params.h +2 -1

@@ -31,6 +31,7 @@
 #define HVM_PARAM_PAE_ENABLED  4
 #define HVM_PARAM_IOREQ_PFN    5
 #define HVM_PARAM_BUFIOREQ_PFN 6
-#define HVM_NR_PARAMS          7
+#define HVM_PARAM_SHINFO_PFN   7
+#define HVM_NR_PARAMS          8
 
 #endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */

diff -r d18c6a3c676a xen/include/public/xen.h

xen/include/public/xen.h +3 -0

@@ -465,6 +465,9 @@ struct shared_info {
     uint32_t wc_sec;          /* Secs  00:00:00 UTC, Jan 1, 1970.  */
     uint32_t wc_nsec;         /* Nsecs 00:00:00 UTC, Jan 1, 1970.  */
 
+    /* flag for resume PV driver in HVM guest */
+    uint32_t pvdrv_resume;
+
     struct arch_shared_info arch;
 
 };

thanks for push in save/restore patches, but this one for save/restore PV driver on HVM seems to be missing.

if any concerns about this patch, pls. let me know.

Hi Edwin,

Quoted 4 lines · Edwin

thanks for push in save/restore patches, but this one for save/restore PV driver on HVM seems to be missing.

if any concerns about this patch, pls. let me know.

The immediate reason I didn't take this patch is that it changes the struct page_info definition. It also breaks PV save/restore (though I haven't yet investigated why).

Since we could take the basic HVM save/restore without this, and we were keen to code into the tree, I left this out for now. I plan to tidy up basic HVM save/restore first and then return to PV-on-HVM.

Cheers,

Tim.