arm64: kernel: Add support for hibernate/suspend-to-disk

Hibernate could make use of the cpu_suspend() code to save/restore cpu state, however it needs to be able to return '0' from the 'finisher'.

Rework cpu_suspend() so that the finisher is called from C code, independently from the save/restore of cpu state. Space to save the context in is allocated in the caller's stack frame, and passed into __cpu_suspend_enter().

Hibernate's use of this API will look like a copy of the cpu_suspend() function.

Signed-off-by James Morse <[email protected]>

---
 arch/arm64/include/asm/suspend.h |  8 ++++
 arch/arm64/kernel/asm-offsets.c  |  2 +
 arch/arm64/kernel/sleep.S        | 86 +++++++++++++---------------------------
 arch/arm64/kernel/suspend.c      | 81 ++++++++++++++++++++++---------------
 4 files changed, 86 insertions(+), 91 deletions(-)

arch/arm64/include/asm/suspend.h +1 -0

index 59a5b0f1e81c..a9de0d3f543f 100644
@@ -2,6 +2,7 @@
 #define __ASM_SUSPEND_H
 
 #define NR_CTX_REGS 11
+#define NR_CALLEE_SAVED_REGS 12
 
 /*
  * struct cpu_suspend_ctx must be 16-byte aligned since it is allocated on

arch/arm64/include/asm/suspend.h +4 -0

+};
+
 extern int cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
 extern void cpu_resume(void);
+int __cpu_suspend_enter(struct sleep_stack_data *state);
+void __cpu_suspend_exit(struct mm_struct *mm);
 #endif

arch/arm64/kernel/asm-offsets.c +2 -0

index 8d89cf8dae55..5daa4e692932 100644
@@ -160,6 +160,8 @@ int main(void)
   DEFINE(SLEEP_SAVE_SP_SZ,	sizeof(struct sleep_save_sp));
   DEFINE(SLEEP_SAVE_SP_PHYS,	offsetof(struct sleep_save_sp, save_ptr_stash_phys));
   DEFINE(SLEEP_SAVE_SP_VIRT,	offsetof(struct sleep_save_sp, save_ptr_stash));
+  DEFINE(SLEEP_STACK_DATA_SYSTEM_REGS,	offsetof(struct sleep_stack_data, system_regs));
+  DEFINE(SLEEP_STACK_DATA_CALLEE_REGS,	offsetof(struct sleep_stack_data, callee_saved_regs));
 #endif
   return 0;
 }

arch/arm64/kernel/sleep.S +0 -0

index f586f7c875e2..6182388e32a5 100644

arch/arm64/kernel/sleep.S +25 -52

  *
- *  x0 = suspend finisher argument
- *  x1 = suspend finisher function pointer
+ *  x0 = struct sleep_stack_data area
  */
 ENTRY(__cpu_suspend_enter)
-	stp	x29, lr, [sp, #-96]!
-	stp	x19, x20, [sp,#16]
-	stp	x21, x22, [sp,#32]
-	stp	x23, x24, [sp,#48]
-	stp	x25, x26, [sp,#64]
-	stp	x27, x28, [sp,#80]
-	/*
-	 * Stash suspend finisher and its argument in x20 and x19
-	 */
-	mov	x19, x0
-	mov	x20, x1
+	stp	x29, lr, [x0, #SLEEP_STACK_DATA_CALLEE_REGS]
+	stp	x19, x20, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+16]
+	stp	x21, x22, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+32]
+	stp	x23, x24, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+48]
+	stp	x25, x26, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+64]
+	stp	x27, x28, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+80]
+
+	/* save the sp in cpu_suspend_ctx */
 	mov	x2, sp
-	sub	sp, sp, #CPU_SUSPEND_SZ	// allocate cpu_suspend_ctx
-	mov	x0, sp
-	/*
-	 * x0 now points to struct cpu_suspend_ctx allocated on the stack
-	 */
-	str	x2, [x0, #CPU_CTX_SP]
+	str	x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]
+
+	/* find the mpidr_hash */
 	ldr	x1, =sleep_save_sp
 	ldr	x1, [x1, #SLEEP_SAVE_SP_VIRT]
 	mrs	x7, mpidr_el1

@@ -93,34 +81,11 @@ ENTRY(__cpu_suspend_enter)
 	ldp	w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
 	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
 	add	x1, x1, x8, lsl #3
+
+	push	x29, lr
 	bl	__cpu_suspend_save
-	/*
-	 * Grab suspend finisher in x20 and its argument in x19
-	 */
-	mov	x0, x19
-	mov	x1, x20
-	/*
-	 * We are ready for power down, fire off the suspend finisher
-	 * in x1, with argument in x0
-	 */
-	blr	x1
-        /*
-	 * Never gets here, unless suspend finisher fails.
-	 * Successful cpu_suspend should return from cpu_resume, returning
-	 * through this code path is considered an error
-	 * If the return value is set to 0 force x0 = -EOPNOTSUPP
-	 * to make sure a proper error condition is propagated
-	 */
-	cmp	x0, #0
-	mov	x3, #-EOPNOTSUPP
-	csel	x0, x3, x0, eq
-	add	sp, sp, #CPU_SUSPEND_SZ	// rewind stack pointer
-	ldp	x19, x20, [sp, #16]
-	ldp	x21, x22, [sp, #32]
-	ldp	x23, x24, [sp, #48]
-	ldp	x25, x26, [sp, #64]
-	ldp	x27, x28, [sp, #80]
-	ldp	x29, lr, [sp], #96
+	pop	x29, lr
+	mov	x0, #1
 	ret
 ENDPROC(__cpu_suspend_enter)
 	.ltorg

@@ -146,12 +111,6 @@ ENDPROC(cpu_resume_mmu)
 	.popsection
 cpu_resume_after_mmu:
 	mov	x0, #0			// return zero on success
-	ldp	x19, x20, [sp, #16]
-	ldp	x21, x22, [sp, #32]
-	ldp	x23, x24, [sp, #48]
-	ldp	x25, x26, [sp, #64]
-	ldp	x27, x28, [sp, #80]
-	ldp	x29, lr, [sp], #96
 	ret
 ENDPROC(cpu_resume_after_mmu)
 

@@ -168,6 +127,8 @@ ENTRY(cpu_resume)
         /* x7 contains hash index, let's use it to grab context pointer */
 	ldr_l	x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS
 	ldr	x0, [x0, x7, lsl #3]
+	add	x29, x0, #SLEEP_STACK_DATA_CALLEE_REGS
+	add	x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
 	/* load sp from context */
 	ldr	x2, [x0, #CPU_CTX_SP]
 	/* load physical address of identity map page table in x1 */

@@ -178,5 +139,12 @@ ENTRY(cpu_resume)
 	 * pointer and x1 to contain physical address of 1:1 page tables
 	 */
 	bl	cpu_do_resume		// PC relative jump, MMU off
+	/* Can't access these by physical address once the MMU is on */
+	ldp	x19, x20, [x29, #16]
+	ldp	x21, x22, [x29, #32]
+	ldp	x23, x24, [x29, #48]
+	ldp	x25, x26, [x29, #64]
+	ldp	x27, x28, [x29, #80]
+	ldp	x29, lr, [x29]
 	b	cpu_resume_mmu		// Resume MMU, never returns
 ENDPROC(cpu_resume)

arch/arm64/kernel/suspend.c +49 -32

index 8297d502217e..2c1a1fd0b4bb 100644
@@ -9,22 +9,22 @@
 #include <asm/suspend.h>
 #include <asm/tlbflush.h>
 
-extern int __cpu_suspend_enter(unsigned long arg, int (*fn)(unsigned long));
+
 /*
  * This is called by __cpu_suspend_enter() to save the state, and do whatever
  * flushing is required to ensure that when the CPU goes to sleep we have
  * the necessary data available when the caches are not searched.
  *
- * ptr: CPU context virtual address
+ * ptr: sleep_stack_data containing cpu state virtual address.
  * save_ptr: address of the location where the context physical address
  *           must be saved
  */
-void notrace __cpu_suspend_save(struct cpu_suspend_ctx *ptr,
+void notrace __cpu_suspend_save(struct sleep_stack_data *ptr,
 				phys_addr_t *save_ptr)
 {
 	*save_ptr = virt_to_phys(ptr);
 
-	cpu_do_suspend(ptr);
+	cpu_do_suspend(&ptr->system_regs);
 	/*
 	 * Only flush the context that must be retrieved with the MMU
 	 * off. VA primitives ensure the flush is applied to all

@@ -50,6 +50,37 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
 	hw_breakpoint_restore = hw_bp_restore;
 }
 
+void notrace __cpu_suspend_exit(struct mm_struct *mm)
+{
+	/*
+	 * We are resuming from reset with TTBR0_EL1 set to the
+	 * idmap to enable the MMU; restore the active_mm mappings in
+	 * TTBR0_EL1 unless the active_mm == &init_mm, in which case
+	 * the thread entered cpu_suspend with TTBR0_EL1 set to
+	 * reserved TTBR0 page tables and should be restored as such.
+	 */
+	if (mm == &init_mm)
+		cpu_set_reserved_ttbr0();
+	else
+		cpu_switch_mm(mm->pgd, mm);
+
+	flush_tlb_all();
+
+	/*
+	 * Restore per-cpu offset before any kernel
+	 * subsystem relying on it has a chance to run.
+	 */
+	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
+
+	/*
+	 * Restore HW breakpoint registers to sane values
+	 * before debug exceptions are possibly reenabled
+	 * through local_dbg_restore.
+	 */
+	if (hw_breakpoint_restore)
+		hw_breakpoint_restore(NULL);
+}
+
 /*
  * cpu_suspend
  *

@@ -60,8 +91,9 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
 int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 {
 	struct mm_struct *mm = current->active_mm;
-	int ret;
+	int ret = 0;
 	unsigned long flags;
+	struct sleep_stack_data state;
 
 	/*
 	 * From this point debug exceptions are disabled to prevent

@@ -76,36 +108,21 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
 	 * page tables, so that the thread address space is properly
 	 * set-up on function return.
 	 */
-	ret = __cpu_suspend_enter(arg, fn);
-	if (ret == 0) {
-		/*
-		 * We are resuming from reset with TTBR0_EL1 set to the
-		 * idmap to enable the MMU; restore the active_mm mappings in
-		 * TTBR0_EL1 unless the active_mm == &init_mm, in which case
-		 * the thread entered cpu_suspend with TTBR0_EL1 set to
-		 * reserved TTBR0 page tables and should be restored as such.
-		 */
-		if (mm == &init_mm)
-			cpu_set_reserved_ttbr0();
-		else
-			cpu_switch_mm(mm->pgd, mm);
-
-		flush_tlb_all();
-
-		/*
-		 * Restore per-cpu offset before any kernel
-		 * subsystem relying on it has a chance to run.
-		 */
-		set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
+	if (__cpu_suspend_enter(&state)) {
+		/* Call the suspend finisher */
+		ret = fn(arg);
 
 		/*
-		 * Restore HW breakpoint registers to sane values
-		 * before debug exceptions are possibly reenabled
-		 * through local_dbg_restore.
+		 * Never gets here, unless suspend finisher fails.
+		 * Successful cpu_suspend should return from cpu_resume,
+		 * returning through this code path is considered an error
+		 * If the return value is set to 0 force ret = -EOPNOTSUPP
+		 * to make sure a proper error condition is propagated
 		 */
-		if (hw_breakpoint_restore)
-			hw_breakpoint_restore(NULL);
-	}
+		if (!ret)
+			ret = -EOPNOTSUPP;
+	} else
+		__cpu_suspend_exit(mm);
 
 	/*
 	 * Restore pstate flags. OS lock and mdscr have been already

Signature · 1 line

2.1.4

arch/arm64/kernel/hibernate.c new file +76 -0

+/*
+ * Walk the provided mm's page tables, from start_addr to end_addr. Translate
+ * each page to its alias in the linear map, and clean that to the PoU.
+ * This is safe to call on user-space mm's, as all the access is to page tables
+ * and kernel linear-map addresses.
+ *
+ * Uses __clean_dcache_pou(), which does not provide any barriers or icache
+ * maintenance. Ensure start_addr is page aligned.
+ */
+static void clean_mapped_range(struct mm_struct *mm, unsigned long start_addr,
+			       unsigned long end_addr)
+{
+	pte_t *pte;
+	size_t length;
+	unsigned long map_addr;
+	unsigned long linear_addr;
+
+	for (map_addr = start_addr; map_addr < end_addr; map_addr += length) {
+		pte = lookup_address(pgd_offset(mm, map_addr), map_addr,
+				     &length);
+		/* length is valid, even if pte is NULL */
+		if (!pte || !pte_valid(*pte))
+			continue;
+
+		linear_addr = (unsigned long)pfn_to_kaddr(pte_pfn(*pte));
+		__clean_dcache_pou(linear_addr, linear_addr+length);
+	}
+}
+
+int swsusp_arch_suspend(void)
+{
+	int ret = 0;
+	unsigned long flags;
+	struct task_struct *p;
+	struct vm_area_struct *vma;
+	struct sleep_stack_data state;
+	struct mm_struct *mm = current->active_mm;
+
+	local_dbg_save(flags);
+
+	if (__cpu_suspend_enter(&state))
+		ret = swsusp_save();
+	else {
+		__cpu_suspend_exit(mm);
+
+		pr_info("Performing cache maintenance.\n");
+
+		/*
+		 * We clean the 'tricky' cache ranges here. Modules and user
+		 * space executable code may have been written to via its
+		 * alias in the kernel linear mapping.
+		 *
+		 * To clean these ranges, we walk the page tables to find the
+		 * physical pages, and then their position in the linear map.
+		 *
+		 * The restore_pblist used during restore only contains pages
+		 * that were in use - other pages containing executable code
+		 * may have been written by core hibernate code.
+		 */
+		clean_mapped_range(&init_mm, MODULES_VADDR, MODULES_END);
+
+		/*
+		 * Any user space executable code that isn't going to be
+		 * reloaded from disk (e.g. jit code) is now potentially
+		 * in the data cache, and needs cleaning.
+		 *
+		 * TODO: Some pages are mapped to user-space many times.
+		 *       Implement a 'cleaned' bitmap so we only clean each
+		 *       page once.
+		 */
+		read_lock(&tasklist_lock);
+		for_each_process(p) {
+			if (!p->mm || p->mm == &init_mm)
+				continue;
+
+			down_read(&p->mm->mmap_sem);

arch/arm64/kernel/hibernate.c new file +236 -0

+
+		/* page tables may still be cached -how does this affect dma? */
+
+		/* all cache cleaning should have finished */
+		dsb(ish);
+		__flush_icache_all();
+	}
+
+	local_dbg_restore(flags);
+
+	return ret;
+}
+
+static int copy_pte(pmd_t *dst, pmd_t *src, unsigned long start_addr)
+{
+	int i;
+	pte_t *old_pte = pte_offset_kernel(src, start_addr);
+	pte_t *new_pte = pte_offset_kernel(dst, start_addr);
+
+	for (i = pte_index(start_addr); i < PTRS_PER_PTE;
+	     i++, old_pte++, new_pte++) {
+		if (pte_val(*old_pte))
+			set_pte(new_pte,
+				__pte(pte_val(*old_pte) & ~PTE_RDONLY));
+	}
+
+	return 0;
+}
+
+static int copy_pmd(pud_t *dst, pud_t *src, unsigned long start_addr)
+{
+	int i;
+	int rc = 0;
+	pte_t *new_pte;
+	pmd_t *old_pmd = pmd_offset(src, start_addr);
+	pmd_t *new_pmd = pmd_offset(dst, start_addr);
+
+	for (i = pmd_index(start_addr); i < PTRS_PER_PMD;
+	     i++, start_addr += PMD_SIZE, old_pmd++, new_pmd++) {
+		if (!pmd_val(*old_pmd))
+			continue;
+
+		if (pmd_table(*(old_pmd))) {
+			new_pte = (pte_t *)get_safe_page(GFP_ATOMIC);
+			if (!new_pte) {
+				rc = -ENOMEM;
+				break;
+			}
+
+			set_pmd(new_pmd, __pmd(virt_to_phys(new_pte)
+					       | PMD_TYPE_TABLE));
+
+			rc = copy_pte(new_pmd, old_pmd, start_addr);
+			if (rc)
+				break;
+		} else
+			set_pmd(new_pmd,
+				__pmd(pmd_val(*old_pmd) & ~PMD_SECT_RDONLY));
+	}
+
+	return rc;
+}
+
+static int copy_pud(pgd_t *dst, pgd_t *src, unsigned long start_addr)
+{
+	int i;
+	int rc = 0;
+	pmd_t *new_pmd;
+	pud_t *old_pud = pud_offset(src, start_addr);
+	pud_t *new_pud = pud_offset(dst, start_addr);
+
+	for (i = pud_index(start_addr); i < PTRS_PER_PUD;
+	     i++, start_addr += PUD_SIZE, old_pud++, new_pud++) {
+		if (!pud_val(*old_pud))
+			continue;
+
+		if (pud_table(*(old_pud))) {
+			if (PTRS_PER_PMD != 1) {
+				new_pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
+				if (!new_pmd) {
+					rc = -ENOMEM;
+					break;
+				}
+
+				set_pud(new_pud, __pud(virt_to_phys(new_pmd)
+						       | PUD_TYPE_TABLE));
+			}
+
+			rc = copy_pmd(new_pud, old_pud, start_addr);
+			if (rc)
+				break;
+		} else
+			set_pud(new_pud,
+				__pud(pud_val(*old_pud) & ~PMD_SECT_RDONLY));
+	}
+
+	return rc;
+}
+
+static int copy_linear_map(pgd_t *new_pgd)
+{
+	int i;
+	int rc = 0;
+	pud_t *new_pud;
+	unsigned long start_addr = PAGE_OFFSET;
+	pgd_t *old_pgd = pgd_offset_k(start_addr);
+
+	new_pgd += pgd_index(start_addr);
+
+	for (i = pgd_index(start_addr); i < PTRS_PER_PGD;
+	     i++, start_addr += PGDIR_SIZE, old_pgd++, new_pgd++) {
+		if (!pgd_val(*old_pgd))
+			continue;
+
+		if (PTRS_PER_PUD != 1) {
+			new_pud = (pud_t *)get_safe_page(GFP_ATOMIC);
+			if (!new_pud) {
+				rc = -ENOMEM;
+				break;
+			}
+
+			set_pgd(new_pgd, __pgd(virt_to_phys(new_pud)
+					       | PUD_TYPE_TABLE));
+		}
+
+		rc = copy_pud(new_pgd, old_pgd, start_addr);
+		if (rc)
+			break;
+	}
+
+	return rc;
+}
+
+/*
+ * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit().
+ *
+ * Memory allocated by get_safe_page() will be dealt with by the hibernate code,
+ * we don't need to free it here.
+ *
+ * Allocate a safe zero page to use as ttbr0, as all existing page tables, and
+ * even the empty_zero_page will be overwritten.
+ */
+int swsusp_arch_resume(void)
+{
+	int rc = 0;
+	pgd_t *pgd;
+	size_t length;
+	size_t exit_size;
+	pgd_t *tmp_pg_dir;
+	pte_t *exit_page_pte;
+	pte_t exit_page_pte_orig;
+	unsigned long exit_page;
+	void *safe_zero_page_mem;
+	void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t);
+
+	/* Copy swsusp_arch_suspend_exit() to a safe page. */
+	exit_page = get_safe_page(GFP_ATOMIC);
+	if (!exit_page) {
+		pr_err("Failed to allocate memory for hibernate_exit code.");
+		rc = -ENOMEM;
+		goto out;
+	}
+	exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start;
+	memcpy((void *)exit_page, __hibernate_exit_text_start, exit_size);
+	flush_icache_range(exit_page, exit_page + exit_size);
+	if (IS_ENABLED(CONFIG_DEBUG_RODATA)) {
+		/*
+		 * set_memory_x() is only for the module ranges. We only have
+		 * the linear-map mapped - so need to make the copied page
+		 * executable now, and when we run with the copied page tables.
+		 * The process of restoring the hibernate kernel will undo
+		 * this change.
+		 */
+		pgd = pgd_offset(&init_mm, exit_page);
+		exit_page_pte = lookup_address(pgd, exit_page, &length);
+		if (exit_page_pte) {
+			exit_page_pte_orig = pte_val(*exit_page_pte);
+			set_pte_at(&init_mm, exit_page, exit_page_pte,
+				  __pte(pte_val(*exit_page_pte) & ~PTE_PXN));
+			flush_tlb_kernel_range(exit_page, exit_page + PAGE_SIZE);
+		}
+		else {
+			pr_err("Failed to find page table entry for hibernate_exit code!");
+			rc = -EFAULT;
+			goto out;
+		}
+	}
+	hibernate_exit = (void *)exit_page;
+
+	/*
+	 * Even the zero page may get overwritten during restore.
+	 * get_safe_page() only returns zero'd pages.
+	 */
+	safe_zero_page_mem = (void *)get_safe_page(GFP_ATOMIC);
+	if (!safe_zero_page_mem) {
+		pr_err("Failed to allocate memory for zero page.");
+		rc = -ENOMEM;
+		goto pte_undo;
+	}
+	empty_zero_page = virt_to_page(safe_zero_page_mem);
+	cpu_set_reserved_ttbr0();
+
+	/*
+	 * Restoring the memory image will overwrite the ttbr1 page tables.
+	 * Create a second copy, of just the linear map, and use this when
+	 * restoring.
+	 */
+	tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
+	if (!tmp_pg_dir) {
+		pr_err("Failed to allocate memory for temporary page tables.");
+		rc = -ENOMEM;
+		goto pte_undo;
+	}
+	rc = copy_linear_map(tmp_pg_dir);
+	if (rc)
+		goto pte_undo;
+
+	/*
+	 * EL2 may get upset if we overwrite its page-tables/stack.
+	 * kvm_reset_cpu() returns EL2 to the hyp stub. This isn't needed
+	 * on normal suspend/resume as PSCI prevents us from ruining EL2.
+	 */
+	if (IS_ENABLED(CONFIG_KVM_ARM_HOST))
+		kvm_reset_cpu();
+
+	hibernate_exit(virt_to_phys(tmp_pg_dir), virt_to_phys(swapper_pg_dir));
+
+pte_undo:
+	if (IS_ENABLED(CONFIG_DEBUG_RODATA)) {
+		set_pte_at(&init_mm, exit_page, exit_page_pte,
+			   exit_page_pte_orig);
+		flush_tlb_kernel_range(exit_page, exit_page + PAGE_SIZE);
+	}
+out:
+	return rc;
+}

arch/arm64/kernel/sleep.S +1 -0

index da4405062d83..f58008d6dadf 100644
@@ -2,6 +2,7 @@
 #include <linux/linkage.h>
 #include <asm/asm-offsets.h>
 #include <asm/assembler.h>
+#include <asm/memory.h>
 
 	.text
 /*

arch/arm64/kernel/vmlinux.lds.S +15 -0

index 98073332e2d0..3d8284d91f4c 100644
@@ -44,6 +44,16 @@ jiffies = jiffies_64;
 	*(.idmap.text)					\
 	VMLINUX_SYMBOL(__idmap_text_end) = .;
 
+#ifdef CONFIG_HIBERNATION
+#define HIBERNATE_TEXT					\
+	. = ALIGN(SZ_4K);				\
+	VMLINUX_SYMBOL(__hibernate_exit_text_start) = .;\
+	*(.hibernate_exit.text)				\
+	VMLINUX_SYMBOL(__hibernate_exit_text_end) = .;
+#else
+#define HIBERNATE_TEXT
+#endif
+
 /*
  * The size of the PE/COFF section that covers the kernel image, which
  * runs from stext to _edata, must be a round multiple of the PE/COFF

@@ -102,6 +112,7 @@ SECTIONS
 			LOCK_TEXT
 			HYPERVISOR_TEXT
 			IDMAP_TEXT
+			HIBERNATE_TEXT
 			*(.fixup)
 			*(.gnu.warning)
 		. = ALIGN(16);

@@ -181,6 +192,10 @@ ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
 	"HYP init code too big or misaligned")
 ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
 	"ID map text too big or misaligned")
+#ifdef CONFIG_HIBERNATION
+ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
+	<= SZ_4K, "Hibernate exit text too big or misaligned")
+#endif
 
 /*
  * If padding is applied before .head.text, virt<->phys conversions will fail.

Signature · 1 line

2.1.4