kexec based hibernation -v6: kexec jump

@@ -20,6 +20,7 @@
 #include <asm/cpufeature.h>
 #include <asm/desc.h>
 #include <asm/system.h>
+#include <asm/cacheflush.h>
 
 #define PAGE_ALIGNED __attribute__ ((__aligned__(PAGE_SIZE)))
 static u32 kexec_pgd[1024] PAGE_ALIGNED;

@@ -83,10 +84,14 @@ static void load_segments(void)
  * reboot code buffer to allow us to avoid allocations
  * later.
  *
- * Currently nothing.
+ * Turn off NX bit for control page.
  */
 int machine_kexec_prepare(struct kimage *image)
 {
+	if (nx_enabled) {
+		change_page_attr(image->control_code_page, 1, PAGE_KERNEL_EXEC);
+		global_flush_tlb();
+	}
 	return 0;
 }
 

@@ -96,25 +101,59 @@ int machine_kexec_prepare(struct kimage 
  */
 void machine_kexec_cleanup(struct kimage *image)
 {
+	if (nx_enabled) {
+		change_page_attr(image->control_code_page, 1, PAGE_KERNEL);
+		global_flush_tlb();
+	}
+}
+
+void machine_kexec(struct kimage *image)
+{
+	machine_kexec_call(image, NULL, 0);
 }
 
 /*
  * Do not allocate memory (or fail in any way) in machine_kexec().
  * We are past the point of no return, committed to rebooting now.
  */
-NORET_TYPE void machine_kexec(struct kimage *image)
+int machine_kexec_vcall(struct kimage *image, unsigned long *ret,
+			 unsigned int argc, va_list args)
 {
 	unsigned long page_list[PAGES_NR];
 	void *control_page;
+	asmlinkage NORET_TYPE void
+		(*relocate_kernel_ptr)(unsigned long indirection_page,
+				       unsigned long control_page,
+				       unsigned long start_address,
+				       unsigned int has_pae) ATTRIB_NORET;
 
 	/* Interrupts aren't acceptable while we reboot */
 	local_irq_disable();
 
 	control_page = page_address(image->control_code_page);
-	memcpy(control_page, relocate_kernel, PAGE_SIZE);
+	memcpy(control_page, relocate_page, PAGE_SIZE/2);
+	KCALL_MAGIC(control_page) = 0;
 
+	if (image->preserve_cpu) {
+		unsigned int i;
+		KCALL_MAGIC(control_page) = KCALL_MAGIC_NUMBER;
+		KCALL_ARGC(control_page) = argc;
+		for (i = 0; i < argc; i++)
+			KCALL_ARGS(control_page)[i] = \
+				va_arg(args, unsigned long);
+
+		if (kexec_call_save_cpu(control_page)) {
+			image->start = KCALL_ENTRY(control_page);
+			if (ret)
+				*ret = KCALL_ARGS(control_page)[0];
+			return 0;
+		}
+	}
+
+	relocate_kernel_ptr = control_page +
+		((void *)relocate_kernel - (void *)relocate_page);
 	page_list[PA_CONTROL_PAGE] = __pa(control_page);
-	page_list[VA_CONTROL_PAGE] = (unsigned long)relocate_kernel;
+	page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;
 	page_list[PA_PGD] = __pa(kexec_pgd);
 	page_list[VA_PGD] = (unsigned long)kexec_pgd;
 #ifdef CONFIG_X86_PAE

@@ -127,26 +166,33 @@ NORET_TYPE void machine_kexec(struct kim
 	page_list[VA_PTE_0] = (unsigned long)kexec_pte0;
 	page_list[PA_PTE_1] = __pa(kexec_pte1);
 	page_list[VA_PTE_1] = (unsigned long)kexec_pte1;
+	page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page) << PAGE_SHIFT);
 
-	/* The segment registers are funny things, they have both a
-	 * visible and an invisible part.  Whenever the visible part is
-	 * set to a specific selector, the invisible part is loaded
-	 * with from a table in memory.  At no other time is the
-	 * descriptor table in memory accessed.
-	 *
-	 * I take advantage of this here by force loading the
-	 * segments, before I zap the gdt with an invalid value.
-	 */
-	load_segments();
-	/* The gdt & idt are now invalid.
-	 * If you want to load them you must set up your own idt & gdt.
-	 */
-	set_gdt(phys_to_virt(0),0);
-	set_idt(phys_to_virt(0),0);
+	if (image->preserve_cpu_ext) {
+		/* The segment registers are funny things, they have
+		 * both a visible and an invisible part.  Whenever the
+		 * visible part is set to a specific selector, the
+		 * invisible part is loaded with from a table in
+		 * memory.  At no other time is the descriptor table
+		 * in memory accessed.
+		 *
+		 * I take advantage of this here by force loading the
+		 * segments, before I zap the gdt with an invalid
+		 * value.
+		 */
+		load_segments();
+		/* The gdt & idt are now invalid.  If you want to load
+		 * them you must set up your own idt & gdt.
+		 */
+		set_gdt(phys_to_virt(0), 0);
+		set_idt(phys_to_virt(0), 0);
+	}
 
 	/* now call it */
-	relocate_kernel((unsigned long)image->head, (unsigned long)page_list,
-			image->start, cpu_has_pae);
+	relocate_kernel_ptr((unsigned long)image->head,
+			    (unsigned long)page_list,
+			    image->start, cpu_has_pae);
+	return 0;
 }
 
 void arch_crash_save_vmcoreinfo(void)

@@ -9,16 +9,40 @@
 #define VA_PTE_0         5
 #define PA_PTE_1         6
 #define VA_PTE_1         7
+#define PA_SWAP_PAGE     8
 #ifdef CONFIG_X86_PAE
-#define PA_PMD_0         8
-#define VA_PMD_0         9
-#define PA_PMD_1         10
-#define VA_PMD_1         11
-#define PAGES_NR         12
+#define PA_PMD_0         9
+#define VA_PMD_0         10
+#define PA_PMD_1         11
+#define VA_PMD_1         12
+#define PAGES_NR         13
 #else
-#define PAGES_NR         8
+#define PAGES_NR         9
 #endif
 
+#define KCALL_DATA_BASE		0x800
+
+#define KCALL_MAGIC_NUMBER	0xe1b6a57d
+
+#define KCALL_DATA(buf)		((__u8 *)(buf)+KCALL_DATA_BASE)
+#define KCALL_OFF(off)		(KCALL_DATA_BASE+(off))
+
+#define KCALL_MAGIC_OFF		KCALL_OFF(0x0)
+#define KCALL_MAGIC(buf)	(*(__u32 *)(KCALL_DATA(buf)+0x0))
+#define KCALL_ARGC_OFF		KCALL_OFF(0x4)
+#define KCALL_ARGC(buf)		(*(__u32 *)(KCALL_DATA(buf)+0x4))
+#define KCALL_ARGS_OFF		KCALL_OFF(0x8)
+#define KCALL_ARGS(buf)		((__u32 *)(KCALL_DATA(buf)+0x8))
+
+/*
+ * The following are not a part of jump back protocol, for internal
+ * use only
+ */
+#define KCALL_ENTRY_OFF		KCALL_OFF(0x200)
+#define KCALL_ENTRY(buf)	(*(__u32 *)(KCALL_DATA(buf)+0x200))
+/* Other internal data fields base */
+#define KCALL_OTHER_OFF		KCALL_OFF(0x204)
+
 #ifndef __ASSEMBLY__
 
 #include <asm/ptrace.h>

@@ -94,6 +118,9 @@ relocate_kernel(unsigned long indirectio
 		unsigned long start_address,
 		unsigned int has_pae) ATTRIB_NORET;
 
+extern char relocate_page[PAGE_SIZE];
+
+extern asmlinkage int kexec_call_save_cpu(void *buf);
 #endif /* __ASSEMBLY__ */
 
 #endif /* _I386_KEXEC_H */

@@ -83,6 +83,7 @@ struct kimage {
 
 	unsigned long start;
 	struct page *control_code_page;
+	struct page *swap_page;
 
 	unsigned long nr_segments;
 	struct kexec_segment segment[KEXEC_SEGMENT_MAX];

@@ -98,18 +99,37 @@ struct kimage {
 	unsigned int type : 1;
 #define KEXEC_TYPE_DEFAULT 0
 #define KEXEC_TYPE_CRASH   1
+	unsigned int preserve_cpu : 1;
+	unsigned int preserve_cpu_ext : 1;
+	unsigned int single_cpu : 1;
+	unsigned int preserve_device : 1;
+	unsigned int preserve_console : 1;
 };
 
 
 
 /* kexec interface functions */
-extern NORET_TYPE void machine_kexec(struct kimage *image) ATTRIB_NORET;
+extern void machine_kexec(struct kimage *image);
+extern int machine_kexec_vcall(struct kimage *image, unsigned long *ret,
+			       unsigned int argc, va_list args);
+extern int machine_kexec_call(struct kimage *image, unsigned long *ret,
+			      unsigned int argc, ...);
 extern int machine_kexec_prepare(struct kimage *image);
 extern void machine_kexec_cleanup(struct kimage *image);
 extern asmlinkage long sys_kexec_load(unsigned long entry,
 					unsigned long nr_segments,
 					struct kexec_segment __user *segments,
 					unsigned long flags);
+extern void kimage_free(struct kimage *image);
+extern int kexec_call(struct kimage *image, unsigned long *ret,
+		      unsigned int argc, ...);
+extern int kexec_vcall(struct kimage *image, unsigned long *ret,
+		       unsigned int argc, va_list args);
+extern int machine_kexec_jump(struct kimage *image, unsigned long *cmd_ret,
+			      unsigned long cmd);
+extern int kexec_jump(struct kimage *image, unsigned long *cmd_ret,
+		      unsigned long cmd);
+#define KJUMP_CMD_NONE 0
 #ifdef CONFIG_COMPAT
 extern asmlinkage long compat_sys_kexec_load(unsigned long entry,
 				unsigned long nr_segments,

@@ -152,8 +172,13 @@ extern struct kimage *kexec_crash_image;
 #define kexec_flush_icache_page(page)
 #endif
 
-#define KEXEC_ON_CRASH  0x00000001
-#define KEXEC_ARCH_MASK 0xffff0000
+#define KEXEC_ON_CRASH		0x00000001
+#define KEXEC_PRESERVE_CPU	0x00000002
+#define KEXEC_PRESERVE_CPU_EXT	0x00000004
+#define KEXEC_SINGLE_CPU	0x00000008
+#define KEXEC_PRESERVE_DEVICE	0x00000010
+#define KEXEC_PRESERVE_CONSOLE	0x00000020
+#define KEXEC_ARCH_MASK		0xffff0000
 
 /* These values match the ELF architecture values.
  * Unless there is a good reason that should continue to be the case.

@@ -170,7 +195,13 @@ extern struct kimage *kexec_crash_image;
 #define KEXEC_ARCH_MIPS_LE (10 << 16)
 #define KEXEC_ARCH_MIPS    ( 8 << 16)
 
-#define KEXEC_FLAGS    (KEXEC_ON_CRASH)  /* List of defined/legal kexec flags */
+/* List of defined/legal kexec flags */
+#define KEXEC_FLAGS    (KEXEC_ON_CRASH |		\
+			KEXEC_PRESERVE_CPU |		\
+			KEXEC_PRESERVE_CPU_EXT |	\
+			KEXEC_SINGLE_CPU |		\
+			KEXEC_PRESERVE_DEVICE |		\
+			KEXEC_PRESERVE_CONSOLE)
 
 #define VMCOREINFO_BYTES           (4096)
 #define VMCOREINFO_NOTE_NAME       "VMCOREINFO"

158 lines not on this page · raw message

@@ -301,18 +301,24 @@ EXPORT_SYMBOL_GPL(kernel_restart);
  *	Move into place and start executing a preloaded standalone
  *	executable.  If nothing was preloaded return an error.
  */
-static void kernel_kexec(void)
+static int kernel_kexec(unsigned long cmd)
 {
+	int ret = -ENOSYS;
 #ifdef CONFIG_KEXEC
 	struct kimage *image;
 	image = xchg(&kexec_image, NULL);
 	if (!image)
-		return;
-	kernel_restart_prepare(NULL);
-	printk(KERN_EMERG "Starting new kernel\n");
-	machine_shutdown();
-	machine_kexec(image);
+		return -EINVAL;
+	if (!image->preserve_cpu) {
+		kernel_restart_prepare(NULL);
+		printk(KERN_EMERG "Starting new kernel\n");
+		machine_shutdown();
+	}
+	ret = kexec_jump(image, NULL, cmd);
+	image = xchg(&kexec_image, image);
+	kimage_free(image);
 #endif
+	return ret;
 }
 
 static void kernel_shutdown_prepare(enum system_states state)

@@ -420,9 +426,12 @@ asmlinkage long sys_reboot(int magic1, i
 		break;
 
 	case LINUX_REBOOT_CMD_KEXEC:
-		kernel_kexec();
-		unlock_kernel();
-		return -EINVAL;
+		{
+			int ret;
+			ret = kernel_kexec((unsigned long)arg);
+			unlock_kernel();
+			return ret;
+		}
 
 #ifdef CONFIG_HIBERNATION
 	case LINUX_REBOOT_CMD_SW_SUSPEND:

@@ -66,7 +66,7 @@ void machine_kexec_cleanup(struct kimage
  * Do not allocate memory (or fail in any way) in machine_kexec().
  * We are past the point of no return, committed to rebooting now.
  */
-NORET_TYPE void machine_kexec(struct kimage *image)
+void machine_kexec(struct kimage *image)
 {
 	if (ppc_md.machine_kexec)
 		ppc_md.machine_kexec(image);

@@ -179,7 +179,7 @@ void machine_kexec_cleanup(struct kimage
  * Do not allocate memory (or fail in any way) in machine_kexec().
  * We are past the point of no return, committed to rebooting now.
  */
-NORET_TYPE void machine_kexec(struct kimage *image)
+void machine_kexec(struct kimage *image)
 {
 	unsigned long page_list[PAGES_NR];
 	void *control_page;

@@ -70,7 +70,7 @@ static void kexec_info(struct kimage *im
  * Do not allocate memory (or fail in any way) in machine_kexec().
  * We are past the point of no return, committed to rebooting now.
  */
-NORET_TYPE void machine_kexec(struct kimage *image)
+void machine_kexec(struct kimage *image)
 {
 
 	unsigned long page_list;

@@ -47,7 +47,7 @@ void machine_kexec_cleanup(struct kimage
  * Do not allocate memory (or fail in any way) in machine_kexec().
  * We are past the point of no return, committed to rebooting now.
  */
-NORET_TYPE void machine_kexec(struct kimage *image)
+void machine_kexec(struct kimage *image)
 {
 	if (ppc_md.machine_kexec)
 		ppc_md.machine_kexec(image);