arm64: kernel: Add support for hibernate/suspend-to-disk

By enabling the MMU early in cpu_resume(), the sleep_save_sp and stack can be accessed by VA, which avoids the need to convert-addresses and clean to PoC on the suspend path.

MMU setup is shared with the boot path, meaning the swapper_pg_dir is restored directly: ttbr1_el1 is no longer saved/restored.

struct sleep_save_sp is removed, replacing it with a single array of pointers.

cpu_do_{suspend,resume} could be further reduced to not restore: cpacr_el1, mdscr_el1, tcr_el1, vbar_el1 and sctlr_el1, all of which are set by __cpu_setup(). However these values all contain res0 bits that may be used to enable future features.

Signed-off-by James Morse <[email protected]>

---

arch/arm64/kernel/sleep.S +2 -2

index dca81612fe90..0e2b36f1fb44 100644
@@ -73,8 +73,8 @@ ENTRY(__cpu_suspend_enter)
 	str	x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]
 
 	/* find the mpidr_hash */
-	ldr	x1, =sleep_save_sp
-	ldr	x1, [x1, #SLEEP_SAVE_SP_VIRT]
+	ldr	x1, =sleep_save_stash
+	ldr	x1, [x1]
 	mrs	x7, mpidr_el1
 	ldr	x9, =mpidr_hash
 	ldr	x10, [x9, #MPIDR_HASH_MASK]

arch/arm64/kernel/sleep.S +15 -33

 	push	x29, lr
-	bl	__cpu_suspend_save
+	bl	cpu_do_suspend
 	pop	x29, lr
 	mov	x0, #1
 	ret
 ENDPROC(__cpu_suspend_enter)
 	.ltorg
 
-/*
- * x0 must contain the sctlr value retrieved from restored context
- */
-	.pushsection	".idmap.text", "ax"
-ENTRY(cpu_resume_mmu)
-	ldr	x3, =cpu_resume_after_mmu
-	msr	sctlr_el1, x0		// restore sctlr_el1
-	isb
-	/*
-	 * Invalidate the local I-cache so that any instructions fetched
-	 * speculatively from the PoC are discarded, since they may have
-	 * been dynamically patched at the PoU.
-	 */
-	ic	iallu
-	dsb	nsh
-	isb
-	br	x3			// global jump to virtual address
-ENDPROC(cpu_resume_mmu)
-	.popsection
-cpu_resume_after_mmu:
-	mov	x0, #0			// return zero on success
-	ret
-ENDPROC(cpu_resume_after_mmu)
-
 ENTRY(cpu_resume)
 	bl	el2_setup		// if in EL2 drop to EL1 cleanly
+	/* enable the MMU early - so we can access sleep_save_stash by va */
+	adr_l	lr, __enable_mmu	/* __cpu_setup will return here */
+	ldr	x27, =_cpu_resume	/* __enable_mmu will branch here */
+	adrp	x25, idmap_pg_dir
+	adrp	x26, swapper_pg_dir
+	b	__cpu_setup
+
+ENTRY(_cpu_resume)
 	mrs	x1, mpidr_el1
 	adrp	x8, mpidr_hash
 	add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address

@@ -130,29 +116,27 @@ ENTRY(cpu_resume)
 	ldp	w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
 	compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
         /* x7 contains hash index, let's use it to grab context pointer */
-	ldr_l	x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS
+	ldr_l	x0, sleep_save_stash
 	ldr	x0, [x0, x7, lsl #3]
 	add	x29, x0, #SLEEP_STACK_DATA_CALLEE_REGS
 	add	x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
 	/* load sp from context */
 	ldr	x2, [x0, #CPU_CTX_SP]
-	/* load physical address of identity map page table in x1 */
-	adrp	x1, idmap_pg_dir
 	mov	sp, x2
 	/* save thread_info */
 	and	x2, x2, #~(THREAD_SIZE - 1)
 	msr	sp_el0, x2
 	/*
-	 * cpu_do_resume expects x0 to contain context physical address
-	 * pointer and x1 to contain physical address of 1:1 page tables
+	 * cpu_do_resume expects x0 to contain context address pointer
 	 */
-	bl	cpu_do_resume		// PC relative jump, MMU off
-	/* Can't access these by physical address once the MMU is on */
+	bl	cpu_do_resume
+
 	ldp	x19, x20, [x29, #16]
 	ldp	x21, x22, [x29, #32]
 	ldp	x23, x24, [x29, #48]
 	ldp	x25, x26, [x29, #64]
 	ldp	x27, x28, [x29, #80]
 	ldp	x29, lr, [x29]
-	b	cpu_resume_mmu		// Resume MMU, never returns
+	mov	x0, #0
+	ret
 ENDPROC(cpu_resume)

arch/arm64/kernel/suspend.c +0 -0

index 6fe46100685a..2ec2f94d1690 100644
@@ -10,30 +10,12 @@
 #include <asm/suspend.h>
 #include <asm/tlbflush.h>
 

arch/arm64/mm/proc.S +1 -2

index db832c42ae30..a2ef1256a329 100644
@@ -24,6 +24,7 @@
 #include <asm/asm-offsets.h>
 #include <asm/hwcap.h>
 #include <asm/pgtable.h>
+#include <asm/pgtable-hwdef.h>
 
 #ifdef CONFIG_ARM64_64K_PAGES
 #define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K

@@ -61,62 +62,50 @@ ENTRY(cpu_do_suspend)
 	mrs	x2, tpidr_el0
 	mrs	x3, tpidrro_el0
 	mrs	x4, contextidr_el1
-	mrs	x5, mair_el1
 	mrs	x6, cpacr_el1
-	mrs	x7, ttbr1_el1
 	mrs	x8, tcr_el1
 	mrs	x9, vbar_el1
 	mrs	x10, mdscr_el1
 	mrs	x11, oslsr_el1
 	mrs	x12, sctlr_el1

arch/arm64/mm/proc.S +10 -19

 	ret
 ENDPROC(cpu_do_suspend)
 
 /**
  * cpu_do_resume - restore CPU register context
  *
- * x0: Physical address of context pointer
- * x1: ttbr0_el1 to be restored
- *
- * Returns:
- *	sctlr_el1 value in x0
+ * x0: Address of context pointer
  */
 ENTRY(cpu_do_resume)
-	/*
-	 * Invalidate local tlb entries before turning on MMU
-	 */
-	tlbi	vmalle1
 	ldp	x2, x3, [x0]
 	ldp	x4, x5, [x0, #16]
-	ldp	x6, x7, [x0, #32]
-	ldp	x8, x9, [x0, #48]
-	ldp	x10, x11, [x0, #64]
-	ldr	x12, [x0, #80]
+	ldp	x6, x8, [x0, #32]
+	ldp	x9, x10, [x0, #48]
+	ldp	x11, x12, [x0, #64]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
-	msr	mair_el1, x5
 	msr	cpacr_el1, x6
-	msr	ttbr0_el1, x1
-	msr	ttbr1_el1, x7
-	tcr_set_idmap_t0sz x8, x7
+
+	/* Don't change t0sz here, mask those bits when restoring */
+	mrs	x5, tcr_el1
+	bfi	x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
+
 	msr	tcr_el1, x8
 	msr	vbar_el1, x9
 	msr	mdscr_el1, x10
+	msr	sctlr_el1, x12
 	/*
 	 * Restore oslsr_el1 by writing oslar_el1
 	 */
 	ubfx	x11, x11, #1, #1
 	msr	oslar_el1, x11
 	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
-	mov	x0, x12
-	dsb	nsh		// Make sure local tlb invalidation completed
 	isb
 	ret
 ENDPROC(cpu_do_resume)

Signature · 1 line

2.6.2

Quoted 17 lines · James Morse

By enabling the MMU early in cpu_resume(), the sleep_save_sp and stack can be accessed by VA, which avoids the need to convert-addresses and clean to PoC on the suspend path.

MMU setup is shared with the boot path, meaning the swapper_pg_dir is restored directly: ttbr1_el1 is no longer saved/restored.

struct sleep_save_sp is removed, replacing it with a single array of pointers.

cpu_do_{suspend,resume} could be further reduced to not restore: cpacr_el1, mdscr_el1, tcr_el1, vbar_el1 and sctlr_el1, all of which are set by __cpu_setup(). However these values all contain res0 bits that may be used to enable future features.

Signed-off-by James Morse <[email protected]>

---

[...]

Quoted 19 lines

arch/arm64/kernel/sleep.S +2 -2

index dca81612fe90..0e2b36f1fb44 100644
@@ -73,8 +73,8 @@ ENTRY(__cpu_suspend_enter)
 	str	x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]
 
 	/* find the mpidr_hash */
-	ldr	x1, =sleep_save_sp
-	ldr	x1, [x1, #SLEEP_SAVE_SP_VIRT]
+	ldr	x1, =sleep_save_stash
+	ldr	x1, [x1]
 	mrs	x7, mpidr_el1
 	ldr	x9, =mpidr_hash
 	ldr	x10, [x9, #MPIDR_HASH_MASK]

@@ -87,40 +87,26 @@ ENTRY(__cpu_suspend_enter)
 	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
 	add	x1, x1, x8, lsl #3
 

arch/arm64/kernel/sleep.S ENTRY(__cpu_suspend_enter) · line 91

Quoted 2 lines

arch/arm64/kernel/sleep.S +2 -0

+	str	x0, [x1]
+	add	x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS

Mmm...this instruction does not really belong in this patch, it should be part of patch 6, correct ? What I mean is, the new struct to stash system regs (struct sleep_stack_data) was added in patch 6, if the offset #SLEEP_STACK_DATA_SYSTEM_REGS (which is 0) had to be added it had to be added in patch 6 too, it does not belong in this patch, am I right ?

arch/arm64/kernel/sleep.S ENTRY(__cpu_suspend_enter) · line 98

Quoted 9 lines

arch/arm64/kernel/sleep.S +1 -1

 	push	x29, lr
-	bl	__cpu_suspend_save
+	bl	cpu_do_suspend
 	pop	x29, lr
 	mov	x0, #1
 	ret
 ENDPROC(__cpu_suspend_enter)
 	.ltorg

With the last updates it seems fine by me, so:

Reviewed-by Lorenzo Pieralisi <[email protected]>

Quoted 19 lines

Quoted 19 lines

arch/arm64/kernel/sleep.S +2 -2

index dca81612fe90..0e2b36f1fb44 100644
@@ -73,8 +73,8 @@ ENTRY(__cpu_suspend_enter)
 	str	x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]
 
 	/* find the mpidr_hash */
-	ldr	x1, =sleep_save_sp
-	ldr	x1, [x1, #SLEEP_SAVE_SP_VIRT]
+	ldr	x1, =sleep_save_stash
+	ldr	x1, [x1]
 	mrs	x7, mpidr_el1
 	ldr	x9, =mpidr_hash
 	ldr	x10, [x9, #MPIDR_HASH_MASK]

@@ -87,40 +87,26 @@ ENTRY(__cpu_suspend_enter)
 	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
 	add	x1, x1, x8, lsl #3
 

Quoted 9 lines

arch/arm64/kernel/sleep.S ENTRY(__cpu_suspend_enter) · line 91

Quoted 2 lines

arch/arm64/kernel/sleep.S +2 -0

+	str	x0, [x1]
+	add	x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS

Mmm...this instruction does not really belong in this patch, it should be part of patch 6, correct ? What I mean is, the new struct to stash system regs (struct sleep_stack_data) was added in patch 6, if the offset #SLEEP_STACK_DATA_SYSTEM_REGS (which is 0) had to be added it had to be added in patch 6 too, it does not belong in this patch, am I right ?

In the previous patch __cpu_suspend_save() was changed to take a struct sleep_stack_data, this then passes system_regs to
cpu_do_suspend(). The 'add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS'
is being done in C, (and hopefully optimised out by the compiler).

In this patch, we removed __cpu_suspend_save() and call cpu_do_suspend() directly, so need to account for system_regs's position in struct sleep_stack_data ourselves.

I'm paranoid, but if you prefer I can remove the 'add 0', and dump a comment next to the struct definition making it someone elses problem to fix it if they ever change the layout of the struct...

Quoted 11 lines

arch/arm64/kernel/sleep.S ENTRY(__cpu_suspend_enter) · line 98

Quoted 11 lines

arch/arm64/kernel/sleep.S +1 -1

 	push	x29, lr
-	bl	__cpu_suspend_save
+	bl	cpu_do_suspend
 	pop	x29, lr
 	mov	x0, #1
 	ret
 ENDPROC(__cpu_suspend_enter)
 	.ltorg

[...2x Nits fixed...]

Quoted 3 lines

With the last updates it seems fine by me, so:

Reviewed-by Lorenzo Pieralisi <[email protected]>

Thanks!

James

Today el2_setup() returns its result in w20. This means we can't call it like any normal function. Change this to w0.

All three callers of el2_setup() call set_cpu_boot_mode_flag() immediately afterwards, and don't rely on x0 in later code. Boot cpu code calls preserve_boot_args() before el2_setup(), so we don't clobber the DT pointer.

Signed-off-by James Morse <[email protected]>

---
 arch/arm64/include/asm/processor.h |  2 ++
 arch/arm64/kernel/head.S           | 18 +++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)

arch/arm64/include/asm/processor.h +0 -0

index 4acb7ca94fcd..07ac4351538e 100644

arch/arm64/include/asm/processor.h +1 -0

+
 #endif /* __ASM_PROCESSOR_H */

arch/arm64/kernel/head.S +9 -9

index 410a2dc04775..5894c7e91e6b 100644
@@ -206,7 +206,7 @@ section_table:
 
 ENTRY(stext)
 	bl	preserve_boot_args
-	bl	el2_setup			// Drop to EL1, w20=cpu_boot_mode
+	bl	el2_setup			// Drop to EL1, w0=cpu_boot_mode
 	adrp	x24, __PHYS_OFFSET
 	bl	set_cpu_boot_mode_flag
 	bl	__create_page_tables		// x25=TTBR0, x26=TTBR1

@@ -446,8 +446,8 @@ ENDPROC(__mmap_switched)
  * If we're fortunate enough to boot at EL2, ensure that the world is
  * sane before dropping to EL1.
  *
- * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if
- * booted in EL1 or EL2 respectively.
+ * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 if booted in
+ * EL1 or EL2 respectively.
  */
 ENTRY(el2_setup)
 	mrs	x0, CurrentEL

@@ -462,7 +462,7 @@ CPU_LE(	bic	x0, x0, #(1 << 25)	)	// Clear the EE bit for EL2
 CPU_BE(	orr	x0, x0, #(3 << 24)	)	// Set the EE and E0E bits for EL1
 CPU_LE(	bic	x0, x0, #(3 << 24)	)	// Clear the EE and E0E bits for EL1
 	msr	sctlr_el1, x0
-	mov	w20, #BOOT_CPU_MODE_EL1		// This cpu booted in EL1
+	mov	w0, #BOOT_CPU_MODE_EL1		// This cpu booted in EL1
 	isb
 	ret
 

@@ -538,20 +538,20 @@ CPU_LE(	movk	x0, #0x30d0, lsl #16	)	// Clear EE and E0E on LE systems
 		      PSR_MODE_EL1h)
 	msr	spsr_el2, x0
 	msr	elr_el2, lr
-	mov	w20, #BOOT_CPU_MODE_EL2		// This CPU booted in EL2
+	mov	w0, #BOOT_CPU_MODE_EL2		// This CPU booted in EL2
 	eret
 ENDPROC(el2_setup)
 
 /*
  * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
- * in x20. See arch/arm64/include/asm/virt.h for more info.
+ * in w0. See arch/arm64/include/asm/virt.h for more info.
  */
 ENTRY(set_cpu_boot_mode_flag)
 	adr_l	x1, __boot_cpu_mode
-	cmp	w20, #BOOT_CPU_MODE_EL2
+	cmp	w0, #BOOT_CPU_MODE_EL2
 	b.ne	1f
 	add	x1, x1, #4
-1:	str	w20, [x1]			// This CPU has booted in EL1
+1:	str	w0, [x1]			// This CPU has booted in EL1
 	dmb	sy
 	dc	ivac, x1			// Invalidate potentially stale cache line
 	ret

@@ -576,7 +576,7 @@ ENTRY(__boot_cpu_mode)
 	 * cores are held until we're ready for them to initialise.
 	 */
 ENTRY(secondary_holding_pen)
-	bl	el2_setup			// Drop to EL1, w20=cpu_boot_mode
+	bl	el2_setup			// Drop to EL1, w0=cpu_boot_mode
 	bl	set_cpu_boot_mode_flag
 	mrs	x0, mpidr_el1
 	ldr     x1, =MPIDR_HWID_BITMASK

Signature · 1 line

2.6.2

Hi James,

Quoted 19 lines · James Morse

Today el2_setup() returns its result in w20. This means we can't call it like any normal function. Change this to w0.

All three callers of el2_setup() call set_cpu_boot_mode_flag() immediately afterwards, and don't rely on x0 in later code. Boot cpu code calls preserve_boot_args() before el2_setup(), so we don't clobber the DT pointer.

Signed-off-by James Morse <[email protected]>

---
 arch/arm64/include/asm/processor.h |  2 ++
 arch/arm64/kernel/head.S           | 18 +++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)

arch/arm64/include/asm/processor.h +0 -0

index 4acb7ca94fcd..07ac4351538e 100644
@@ -188,4 +188,6 @@ static inline void spin_lock_prefetch(const void *x)
 

arch/arm64/include/asm/processor.h static inline void spin_lock_prefetch(const void *x) · line 191

Quoted 3 lines

arch/arm64/include/asm/processor.h +1 -0

 void cpu_enable_pan(void *__unused);
 
+int el2_setup(void);

Do we really need to have this declaration (and this patch) ? We never treat (and call) el2_setup as a C function anyway, as long as the registers used to pass in values (and return value) are documented I do not think we have an issue anyway, am I missing something here ?

Quoted 19 lines · James Morse

Today el2_setup() returns its result in w20. This means we can't call it like any normal function. Change this to w0.

All three callers of el2_setup() call set_cpu_boot_mode_flag() immediately afterwards, and don't rely on x0 in later code. Boot cpu code calls preserve_boot_args() before el2_setup(), so we don't clobber the DT pointer.

Signed-off-by James Morse <[email protected]>

---
 arch/arm64/include/asm/processor.h |  2 ++
 arch/arm64/kernel/head.S           | 18 +++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)

arch/arm64/include/asm/processor.h +0 -0

index 4acb7ca94fcd..07ac4351538e 100644
@@ -188,4 +188,6 @@ static inline void spin_lock_prefetch(const void *x)
 

arch/arm64/include/asm/processor.h static inline void spin_lock_prefetch(const void *x) · line 191

Quoted 3 lines

arch/arm64/include/asm/processor.h +1 -0

 void cpu_enable_pan(void *__unused);
 
+int el2_setup(void);

Do we really need to have this declaration (and this patch) ? We never treat (and call) el2_setup as a C function anyway, as long as the registers used to pass in values (and return value) are documented I do not think we have an issue anyway, am I missing something here ?

Some architectures require code written to memory as if it were data to be 'cleaned' from any data caches before the processor can fetch them as new instructions.

During resume from hibernate, the snapshot code copies some pages directly, meaning these architectures do not get a chance to perform their cache maintenance. Modify the read and decompress code to call flush_icache_range() on all pages that are restored, so that the restored in-place pages are guaranteed to be executable on these architectures.

Signed-off-by James Morse <[email protected]>

Acked-by Pavel Machek <[email protected]>

Add support for hibernate/suspend-to-disk.

Suspend borrows code from cpu_suspend() to write cpu state onto the stack, before calling swsusp_save() to save the memory image.

Restore creates a set of temporary page tables, covering only the linear map, copies the restore code to a 'safe' page, then uses the copy to restore the memory image. The copied code executes in the lower half of the address space, and once complete, restores the original kernel's page tables. It then calls into cpu_resume(), and follows the normal cpu_suspend() path back into the suspend code.

To restore a kernel using KASLR, the address of the page tables, and cpu_resume() are stored in the hibernate arch-header and the el2 vectors are pivotted via the 'safe' page in low memory. This also permits us to resume using a different version of the kernel to the version that hibernated, but because the MMU isn't turned off during resume, the MMU settings must be the same between both kernels. To ensure this, the value of the translation control register (TCR_EL1) is also included in the hibernate arch-header, this means your resume kernel must have the same page size, and virtual address space size.

Signed-off-by James Morse <[email protected]>

Tested-by Kevin Hilman <[email protected]> # Tested on Juno R2

---

arch/arm64/kernel/hibernate-asm.S new file +14 -0

+/*
+ * el2_setup() moves lr into elr and erets. Copy lr to another register to
+ * preserve it.
+ *
+ * int swsusp_el2_setup_helper(phys_addr_t el2_setup_pa);
+ */
+ENTRY(swsusp_el2_setup_helper)
+	mov	x16, x30
+	hvc	#0
+	mov	x30, x16
+	ret
+ENDPROC(swsusp_el2_setup_helper)
+
+/*

arch/arm64/kernel/hibernate.c new file +10 -0

+int swsusp_arch_suspend(void)
+{
+	int ret = 0;
+	unsigned long flags;
+	struct sleep_stack_data state;
+
+	local_dbg_save(flags);
+
+	if (__cpu_suspend_enter(&state)) {
+		ret = swsusp_save();

arch/arm64/kernel/hibernate.c new file +219 -0

+
+		/*
+		 * Tell the hibernation core that we've just restored
+		 * the memory
+		 */
+		in_suspend = 0;
+
+		__cpu_suspend_exit();
+	}
+
+	local_dbg_restore(flags);
+
+	return ret;
+}
+
+static int copy_pte(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long start,
+		    unsigned long end)
+{
+	unsigned long next;
+	unsigned long addr = start;
+	pte_t *src_pte = pte_offset_kernel(src_pmd, start);
+	pte_t *dst_pte = pte_offset_kernel(dst_pmd, start);
+
+	do {
+		next = addr + PAGE_SIZE;
+		if (pte_val(*src_pte))
+			set_pte(dst_pte,
+				__pte(pte_val(*src_pte) & ~PTE_RDONLY));
+	} while (dst_pte++, src_pte++, addr = next, addr != end);
+
+	return 0;
+}
+
+static int copy_pmd(pud_t *dst_pud, pud_t *src_pud, unsigned long start,
+		    unsigned long end)
+{
+	int rc = 0;
+	pte_t *dst_pte;
+	unsigned long next;
+	unsigned long addr = start;
+	pmd_t *src_pmd = pmd_offset(src_pud, start);
+	pmd_t *dst_pmd = pmd_offset(dst_pud, start);
+
+	do {
+		next = pmd_addr_end(addr, end);
+		if (!pmd_val(*src_pmd))
+			continue;
+
+		if (pmd_table(*(src_pmd))) {
+			dst_pte = (pte_t *)get_safe_page(GFP_ATOMIC);
+			if (!dst_pte) {
+				rc = -ENOMEM;
+				break;
+			}
+
+			set_pmd(dst_pmd, __pmd(virt_to_phys(dst_pte)
+					       | PMD_TYPE_TABLE));
+
+			rc = copy_pte(dst_pmd, src_pmd, addr, next);
+			if (rc)
+				break;
+		} else {
+			set_pmd(dst_pmd,
+				__pmd(pmd_val(*src_pmd) & ~PMD_SECT_RDONLY));
+		}
+	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
+
+	return rc;
+}
+
+static int copy_pud(pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long start,
+		    unsigned long end)
+{
+	int rc = 0;
+	pmd_t *dst_pmd;
+	unsigned long next;
+	unsigned long addr = start;
+	pud_t *src_pud = pud_offset(src_pgd, start);
+	pud_t *dst_pud = pud_offset(dst_pgd, start);
+
+	do {
+		next = pud_addr_end(addr, end);
+		if (!pud_val(*src_pud))
+			continue;
+
+		if (pud_table(*(src_pud))) {
+			if (PTRS_PER_PMD != 1) {
+				dst_pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
+				if (!dst_pmd) {
+					rc = -ENOMEM;
+					break;
+				}
+
+				set_pud(dst_pud, __pud(virt_to_phys(dst_pmd)
+						       | PUD_TYPE_TABLE));
+			}
+
+			rc = copy_pmd(dst_pud, src_pud, addr, next);
+			if (rc)
+				break;
+		} else {
+			set_pud(dst_pud,
+				__pud(pud_val(*src_pud) & ~PMD_SECT_RDONLY));
+		}
+	} while (dst_pud++, src_pud++, addr = next, addr != end);
+
+	return rc;
+}
+
+static int copy_page_tables(pgd_t *dst_pgd, unsigned long start,
+			    unsigned long end)
+{
+	int rc = 0;
+	pud_t *dst_pud;
+	unsigned long next;
+	unsigned long addr = start;
+	pgd_t *src_pgd = pgd_offset_k(start);
+
+	dst_pgd += pgd_index(start);
+
+	do {
+		next = pgd_addr_end(addr, end);
+		if (!pgd_val(*src_pgd))
+			continue;
+
+		if (PTRS_PER_PUD != 1) {
+			dst_pud = (pud_t *)get_safe_page(GFP_ATOMIC);
+			if (!dst_pud) {
+				rc = -ENOMEM;
+				break;
+			}
+
+			set_pgd(dst_pgd, __pgd(virt_to_phys(dst_pud)
+					       | PUD_TYPE_TABLE));
+		}
+
+		rc = copy_pud(dst_pgd, src_pgd, addr, next);
+		if (rc)
+			break;
+	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
+
+	return rc;
+}
+
+/*
+ * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit().
+ *
+ * Memory allocated by get_safe_page() will be dealt with by the hibernate code,
+ * we don't need to free it here.
+ */
+int swsusp_arch_resume(void)
+{
+	int rc = 0;
+	size_t exit_size;
+	pgd_t *tmp_pg_dir;
+	void *lm_restore_pblist;
+	phys_addr_t phys_hibernate_exit;
+	void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, void *);
+
+	/*
+	 * Copy swsusp_arch_suspend_exit() to a safe page. This will generate
+	 * a new set of ttbr0 page tables and load them.
+	 */
+	exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start;
+	rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size,
+				   (void **)&hibernate_exit,
+				   &phys_hibernate_exit,
+				   get_safe_page, GFP_ATOMIC);
+	if (rc) {
+		pr_err("Failed to create safe executable page for hibernate_exit code.");
+		goto out;
+	}
+
+	/*
+	 * The hibernate exit text contains a set of el2 vectors, that will
+	 * be executed at el2 with the mmu off in order to reload hyp-stub.
+	 */
+	__flush_dcache_area(hibernate_exit, exit_size);
+
+	/*
+	 * Restoring the memory image will overwrite the ttbr1 page tables.
+	 * Create a second copy of just the linear map, and use this when
+	 * restoring.
+	 */
+	tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
+	if (!tmp_pg_dir) {
+		pr_err("Failed to allocate memory for temporary page tables.");
+		rc = -ENOMEM;
+		goto out;
+	}
+	rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, 0);
+	if (rc)
+		goto out;
+
+	/*
+	 * Since we only copied the linear map, we need to find restore_pblist's
+	 * linear map address.
+	 */
+	lm_restore_pblist = phys_to_virt(virt_to_phys(restore_pblist));
+
+	/*
+	 * Both KASLR and restoring with a different kernel version will cause
+	 * the el2 vectors to be in a different location in the resumed kernel.
+	 * Load hibernate's temporary copy into el2.
+	 */
+	if (is_hyp_mode_available()) {
+		phys_addr_t el2_vectors = phys_hibernate_exit;  /* base */
+		el2_vectors += hibernate_el2_vectors -
+			       __hibernate_exit_text_start;     /* offset */
+
+		__hyp_set_vectors(el2_vectors);
+	}
+
+	hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1,
+		       resume_hdr.reenter_kernel, lm_restore_pblist);
+
+out:
+	return rc;
+}

arch/arm64/kernel/vmlinux.lds.S +15 -0

index 282e3e64a17e..02184c39f7e2 100644
@@ -46,6 +46,16 @@ jiffies = jiffies_64;
 	*(.idmap.text)					\
 	VMLINUX_SYMBOL(__idmap_text_end) = .;
 
+#ifdef CONFIG_HIBERNATION
+#define HIBERNATE_TEXT					\
+	. = ALIGN(SZ_4K);				\
+	VMLINUX_SYMBOL(__hibernate_exit_text_start) = .;\
+	*(.hibernate_exit.text)				\
+	VMLINUX_SYMBOL(__hibernate_exit_text_end) = .;
+#else
+#define HIBERNATE_TEXT
+#endif
+
 /*
  * The size of the PE/COFF section that covers the kernel image, which
  * runs from stext to _edata, must be a round multiple of the PE/COFF

@@ -107,6 +117,7 @@ SECTIONS
 			LOCK_TEXT
 			HYPERVISOR_TEXT
 			IDMAP_TEXT
+			HIBERNATE_TEXT
 			*(.fixup)
 			*(.gnu.warning)
 		. = ALIGN(16);

@@ -182,6 +193,10 @@ ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
 	"HYP init code too big or misaligned")
 ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
 	"ID map text too big or misaligned")
+#ifdef CONFIG_HIBERNATION
+ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
+	<= SZ_4K, "Hibernate exit text too big or misaligned")
+#endif
 
 /*
  * If padding is applied before .head.text, virt<->phys conversions will fail.

Signature · 1 line

2.6.2

PM: Image mismatch: memory size

Resuming using a different kernel version is fragile, while there are sufficient details in the hibernate arch-header to perform the restore, changes in the boot process can have a long-lasting impact on the system. In particular, if the EFI stub causes more memory to be allocated, the amount of memory left for linux is reduced. If we are lucky, this will cause restore to fail with the message:

-----Original Message-----
From: linux-pm-owner at vger.kernel.org [mailto:linux-pm-
owner at vger.kernel.org] On Behalf Of Pavel Machek
Sent: Wednesday, February 17, 2016 4:16 AM
To: James Morse
Cc: linux-arm-kernel at lists.infradead.org; Will Deacon; Sudeep Holla; Geoff Levand; Catalin Marinas; Lorenzo Pieralisi; Mark Rutland; AKASHI Takahiro; Marc Zyngier; Rafael J . Wysocki; linux-pm at vger.kernel.org
Subject: Re: [PATCH v5 15/15] arm64: hibernate: Prevent resume from a different kernel version

Quoted 3 lines · James Morse

Resuming using a different kernel version is fragile, while there are sufficient details in the hibernate arch-header to perform the restore, changes in the boot process can have a long-lasting impact on the

system.

Quoted 3 lines

In particular, if the EFI stub causes more memory to be allocated, the amount of memory left for linux is reduced. If we are lucky, this will cause restore to fail with the message:

Well, this does not close the door completely. 4.6-rc0 is going to be very different from 4.6-rc1. Better solution would be to increase version every time EFI stub changes, or maybe record ammount of memory reserved for EFI.

Quoted 5 lines · James Morse

Quoted 3 lines

Resuming using a different kernel version is fragile, while there are sufficient details in the hibernate arch-header to perform the restore, changes in the boot process can have a long-lasting impact on the

system.

Quoted 8 lines

Quoted 3 lines

In particular, if the EFI stub causes more memory to be allocated, the amount of memory left for linux is reduced. If we are lucky, this will cause restore to fail with the message:

Well, this does not close the door completely. 4.6-rc0 is going to be very different from 4.6-rc1. Better solution would be to increase version every time EFI stub changes, or maybe record ammount of memory reserved for EFI.

This reminds me a similar problem I once encountered on x86 : - )
The efi memory layout should be strictly the same before/after hibernation, right?

-----Original Message-----
From: James Morse [mailto:james.morse at arm.com]
Sent: Thursday, February 18, 2016 8:00 PM
To: Chen, Yu C
Cc: Pavel Machek; linux-arm-kernel at lists.infradead.org; Will Deacon; Sudeep Holla; Geoff Levand; Catalin Marinas; Lorenzo Pieralisi; Mark Rutland; AKASHI Takahiro; Marc Zyngier; Rafael J . Wysocki; linux-pm at vger.kernel.org
Subject: Re: [PATCH v5 15/15] arm64: hibernate: Prevent resume from a different kernel version

Quoted 3 lines · Yu C

Quoted 1 line · James Morse

This reminds me a similar problem I once encountered on x86 : - ) The efi memory layout should be strictly the same before/after hibernation,

right?

The kernel hopes it is the same, as the page-tables it uses for runtime services calls are restored along with the rest of memory, but there is the risk that these don't match the EFI memory map any more.

Even if the amount of memory is the same, the layout might be different. (Core hibernate code already has a counter of the number of physical pages.)

I'm still digging through the efi code (and spec), but it is fairly easy to cause the memory map to change before entry to linux. This doesn't seem to be a problem, as linux happily overwrites most of the allocated areas, so it may not be as fragile as I thought.

Quoted 15 lines

-----Original Message-----
From: James Morse [mailto:james.morse at arm.com]
Sent: Thursday, February 18, 2016 8:00 PM
To: Chen, Yu C
Cc: Pavel Machek; linux-arm-kernel at lists.infradead.org; Will Deacon; Sudeep Holla; Geoff Levand; Catalin Marinas; Lorenzo Pieralisi; Mark Rutland; AKASHI Takahiro; Marc Zyngier; Rafael J . Wysocki; linux-pm at vger.kernel.org
Subject: Re: [PATCH v5 15/15] arm64: hibernate: Prevent resume from a different kernel version

Quoted 3 lines · Yu C

Quoted 1 line · James Morse

This reminds me a similar problem I once encountered on x86 : - ) The efi memory layout should be strictly the same before/after hibernation,

Quoted 8 lines

right?

The kernel hopes it is the same, as the page-tables it uses for runtime services calls are restored along with the rest of memory, but there is the risk that these don't match the EFI memory map any more.

Even if the amount of memory is the same, the layout might be different. (Core hibernate code already has a counter of the number of physical pages.)

I've encountered this situation before.

Quoted 5 lines

I'm still digging through the efi code (and spec), but it is fairly easy to cause the memory map to change before entry to linux. This doesn't seem to be a problem, as linux happily overwrites most of the allocated areas, so it may not be as fragile as I thought.

If I did some peripherals plug/unplug after suspend to hibernation,
then the next time it boots up there would likely be a slightly different efi memory map. The efi provides a callback of get_memory_map to Linux for retrieving the initial map, which depends on the bios/bootloader probing/enumeration IMO. The difference of efi map might lead to kernel panic during resume sometimes.

-----Original Message-----
From: Pavel Machek [mailto:pavel at ucw.cz]
Sent: Sunday, February 21, 2016 3:57 AM
To: Chen, Yu C
Cc: James Morse; linux-arm-kernel at lists.infradead.org; Will Deacon; Sudeep Holla; Geoff Levand; Catalin Marinas; Lorenzo Pieralisi; Mark Rutland; AKASHI Takahiro; Marc Zyngier; Rafael J . Wysocki; linux-pm at vger.kernel.org
Subject: Re: [PATCH v5 15/15] arm64: hibernate: Prevent resume from a different kernel version

Quoted 14 lines · Yu C

Quoted 14 lines

-----Original Message-----
From: James Morse [mailto:james.morse at arm.com]
Sent: Thursday, February 18, 2016 8:00 PM
To: Chen, Yu C
Cc: Pavel Machek; linux-arm-kernel at lists.infradead.org; Will Deacon; Sudeep Holla; Geoff Levand; Catalin Marinas; Lorenzo Pieralisi; Mark Rutland; AKASHI Takahiro; Marc Zyngier; Rafael J . Wysocki; linux-pm at vger.kernel.org
Subject: Re: [PATCH v5 15/15] arm64: hibernate: Prevent resume from a different kernel version

Quoted 1 line · Yu C

Quoted 1 line · James Morse

Quoted 8 lines

Quoted 8 lines

Quoted 3 lines

This reminds me a similar problem I once encountered on x86 : - ) The efi memory layout should be strictly the same before/after hibernation,

right?

The kernel hopes it is the same, as the page-tables it uses for runtime services calls are restored along with the rest of memory, but there is the risk that these don't match the EFI memory map any

more.

Quoted 4 lines

Quoted 4 lines

Even if the amount of memory is the same, the layout might be different. (Core hibernate code already has a counter of the number of physical pages.)

Quoted 8 lines

I've encountered this situation before.

Quoted 5 lines

I'm still digging through the efi code (and spec), but it is fairly easy to cause the memory map to change before entry to linux. This doesn't seem to be a problem, as linux happily overwrites most of the allocated areas, so it may not be as fragile as I thought.

If I did some peripherals plug/unplug after suspend to hibernation, then the next time it boots up there would likely be a slightly different efi

memory map.

Quoted 3 lines

The efi provides a callback of get_memory_map to Linux for retrieving the initial map, which depends on the bios/bootloader probing/enumeration IMO. The difference of efi map might lead to kernel

panic during resume sometimes.

md5hash of the efi map, and complain to bios vendors if it happens? Because iirc it is not supposed to.

This version of hibernate is based on v4.5-rc4, with the contents of the arm64 for-next/pgtable branch and Ard's v6sub1 "arm64: split linear and kernel mappings" series [0]. The patches shared with kexec still come from v13 [1].