MMU disabling code and kexec fixes

Hi,

This is version 2 of the patches originally posted here:

http://lists.infradead.org/pipermail/linux-arm-kernel/2011-June/052157.html

Thanks to Dave and Frank for feedback on those patches.

Changes from v1 include:

* Implementation of cpu_reset for ARMv6 cores.
* Introduction of stack-switching code to allow us to include more of the address space in the 1:1 mapping.
* Exposure of non-static identity mappings functions via a new header file.
* Changes to the cpu_* macro definitions when MULTI_CPU is defined.
* Introduction of a new reboot mode, 'k', for mapping (most of) kernelspace as 1:1.
* Use of swapper_pg_dir to create the identity mapping.

The idea is to remap all of the address space apart from the kernel image and a small amount of memory (up to a page) immediately following it as part of the identity map. This means that we can guarantee the reset code will be available at its physical address (unless the difference between virtual and physical space is unusually small. The 1:1 scenario is a special case that I haven't handled here).

I've tested this with kexec on a Realview-PBX (Cortex-A9) and it appears to work well, although kexecing an SMP kernel exposes issues with other parts of the code (timer initialisation for example).

Once again, all feedback welcome!

Will

Will Deacon (10):
ARM: l2x0: fix disabling function to avoid livelock
ARM: l2x0: fix invalidate-all function to avoid livelock
ARM: proc: add definition of cpu_reset for ARMv6 and ARMv7 cores
ARM: lib: add switch_stack function for safely changing stack
ARM: idmap: add header file for identity mapping functions
ARM: reset: allow kernelspace mappings to be flat mapped during reset
ARM: multi-cpu: remove arguments from CPU proc macros
ARM: reset: add reset functionality for jumping to a physical address
ARM: kexec: use arm_machine_reset for branching to the reboot buffer
ARM: stop: execute platform callback from cpu_stop code

 arch/arm/Kconfig                |    2 +-
 arch/arm/include/asm/idmap.h    |   23 ++++++++++++
 arch/arm/include/asm/pgtable.h  |    3 --
 arch/arm/include/asm/proc-fns.h |   14 ++++----
 arch/arm/include/asm/system.h   |    1 +
 arch/arm/kernel/machine_kexec.c |   14 +------
 arch/arm/kernel/process.c       |   75 +++++++++++++++++++++++++++++++++-----
 arch/arm/kernel/smp.c           |    5 +++
 arch/arm/lib/Makefile           |    3 +-
 arch/arm/lib/switch_stack.S     |   44 +++++++++++++++++++++++
 arch/arm/mm/cache-l2x0.c        |   13 ++++---
 arch/arm/mm/idmap.c             |   31 +++++++++++-----
 arch/arm/mm/proc-v6.S           |    5 +++
 arch/arm/mm/proc-v7.S           |    7 ++++
 14 files changed, 191 insertions(+), 49 deletions(-)
 create mode 100644 arch/arm/include/asm/idmap.h
 create mode 100644 arch/arm/lib/switch_stack.S

The l2x0_disable function attempts to writel with the l2x0_lock held. This results in livelock when the writel contains an outer_sync call for the platform.

This patch replaces the writel with a call to writel_relaxed in the disabling code, preventing livelock from occurring.

---
 arch/arm/mm/cache-l2x0.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

index ef59099..2bce3be 100644
@@ -266,7 +266,7 @@ static void l2x0_disable(void)
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
-	writel(0, l2x0_base + L2X0_CTRL);
+	writel_relaxed(0, l2x0_base + L2X0_CTRL);
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 

1.7.0.4

With the L2 cache disabled, exclusive memory access instructions may cease to function correctly, leading to livelock when trying to acquire a spinlock.

The l2x0 invalidate-all routine *must* run with the cache disabled and so needs to take extra care not to take any locks along the way.

This patch modifies the invalidation routine to avoid locking. Since the cache is disabled, we make the assumption that other CPUs are not executing background maintenance tasks on the L2 cache whilst we are invalidating it.

---
 arch/arm/mm/cache-l2x0.c |   11 ++++++-----
 1 files changed, 6 insertions(+), 5 deletions(-)

index 2bce3be..fe5630f 100644
@@ -148,16 +148,17 @@ static void l2x0_clean_all(void)
 
 static void l2x0_inv_all(void)
 {
-	unsigned long flags;
-
-	/* invalidate all ways */
-	spin_lock_irqsave(&l2x0_lock, flags);
 	/* Invalidating when L2 is enabled is a nono */
 	BUG_ON(readl(l2x0_base + L2X0_CTRL) & 1);
+
+	/*
+	 * invalidate all ways
+	 * Since the L2 is disabled, exclusive accessors may not be
+	 * available to us, so avoid taking any locks.
+	 */
 	writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY);
 	cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask);
 	cache_sync();
-	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_inv_range(unsigned long start, unsigned long end)

1.7.0.4

This patch adds simple definitions of cpu_reset for ARMv6 and ARMv7 cores, which disable the MMU via the SCTLR.

---
 arch/arm/mm/proc-v6.S |    5 +++++
 arch/arm/mm/proc-v7.S |    7 +++++++
 2 files changed, 12 insertions(+), 0 deletions(-)

index 1d2b845..f3b5232 100644
@@ -56,6 +56,11 @@ ENTRY(cpu_v6_proc_fin)
  */
 	.align	5
 ENTRY(cpu_v6_reset)
+	mrc	p15, 0, r1, c1, c0, 0		@ ctrl register
+	bic	r1, r1, #0x1			@ ...............m
+	mcr	p15, 0, r1, c1, c0, 0		@ disable MMU
+	mov	r1, #0
+	mcr	p15, 0, r1, c7, c5, 4		@ ISB
 	mov	pc, r0
 
 /*

index b3b566e..776443a 100644
@@ -58,9 +58,16 @@ ENDPROC(cpu_v7_proc_fin)
  *	to what would be the reset vector.
  *
  *	- loc   - location to jump to for soft reset
+ *
+ *	This code must be executed using a flat identity mapping with
+ *      caches disabled.
  */
 	.align	5
 ENTRY(cpu_v7_reset)
+	mrc	p15, 0, r1, c1, c0, 0		@ ctrl register
+	bic	r1, r1, #0x1			@ ...............m
+	mcr	p15, 0, r1, c1, c0, 0		@ disable MMU
+	isb
 	mov	pc, r0
 ENDPROC(cpu_v7_reset)
 

1.7.0.4

When disabling the MMU, it is necessary to take out a 1:1 identity map of the reset code so that it can safely be executed with and without the MMU active. To avoid the situation where the physical address of the reset code aliases with the virtual address of the active stack (which cannot be included in the 1:1 mapping), it is desirable to change to a new stack at a location which is less likely to alias.

This code adds a new lib function, switch_stack:

void switch_stack(void (*fn)(void *), void *arg, void *sp);

which changes the stack to point at the sp parameter, before invoking fn(arg) with the new stack selected.

---
 arch/arm/lib/Makefile       |    3 +-
 arch/arm/lib/switch_stack.S |   44 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletions(-)
 create mode 100644 arch/arm/lib/switch_stack.S

index 59ff42d..5fa67de 100644
@@ -13,7 +13,8 @@ lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 		   testchangebit.o testclearbit.o testsetbit.o        \
 		   ashldi3.o ashrdi3.o lshrdi3.o muldi3.o             \
 		   ucmpdi2.o lib1funcs.o div64.o sha1.o               \
-		   io-readsb.o io-writesb.o io-readsl.o io-writesl.o
+		   io-readsb.o io-writesb.o io-readsl.o io-writesl.o  \
+		   switch_stack.o
 
 mmu-y	:= clear_user.o copy_page.o getuser.o putuser.o
 

1.7.0.4

Argh

I really meant:

	ldr	lr, [sp], #4

Apologies for the confusion!

Cheers
---Dave

The identity mappings functions are useful outside of SMP booting, so expose them through their own header file.

---
 arch/arm/include/asm/idmap.h   |   16 ++++++++++++++++
 arch/arm/include/asm/pgtable.h |    3 ---
 arch/arm/kernel/process.c      |    3 +--
 arch/arm/kernel/smp.c          |    1 +
 arch/arm/mm/idmap.c            |    1 +
 5 files changed, 19 insertions(+), 5 deletions(-)
 create mode 100644 arch/arm/include/asm/idmap.h

new file mode 100644
index 0000000..ea9517e
@@ -0,0 +1,16 @@
+#ifndef _ARM_IDMAP_H
+#define _ARM_IDMAP_H
+
+#include <asm/page.h>
+
+void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end);
+
+#ifdef CONFIG_SMP
+void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end);
+#else
+void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end) {};
+#endif
+
+void setup_mm_for_reboot(char mode);
+
+#endif	/* _ARM_IDMAP_H */

index 5750704..9d559a8 100644
@@ -474,9 +474,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 
 #define pgtable_cache_init() do { } while (0)
 
-void identity_mapping_add(pgd_t *, unsigned long, unsigned long);
-void identity_mapping_del(pgd_t *, unsigned long, unsigned long);
-
 #endif /* !__ASSEMBLY__ */
 
 #endif /* CONFIG_MMU */

index 5e1e541..8bd9d94 100644
@@ -32,6 +32,7 @@
 #include <linux/hw_breakpoint.h>
 
 #include <asm/cacheflush.h>
+#include <asm/idmap.h>
 #include <asm/leds.h>
 #include <asm/processor.h>
 #include <asm/system.h>

@@ -56,8 +57,6 @@ static const char *isa_modes[] = {
   "ARM" , "Thumb" , "Jazelle", "ThumbEE"
 };
 
-extern void setup_mm_for_reboot(char mode);
-
 static volatile int hlt_counter;
 
 #include <mach/system.h>

index 344e52b..dfc76aa 100644
@@ -39,6 +39,7 @@
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
 #include <asm/localtimer.h>
+#include <asm/idmap.h>
 
 /*
  * as from 2.5, kernels no longer have an init_tasks structure

index 2be9139..4ae0f09 100644
@@ -1,6 +1,7 @@
 #include <linux/kernel.h>
 
 #include <asm/cputype.h>
+#include <asm/idmap.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 

1.7.0.4

Hi Will,

I'm wondering why identity_mapping_del() is restricted to SMP.

Yes have seen that the only current _user_ is SMP initialization, but does that mean it has to be restricted to that ?

I'd want to use it in hibernation resume code, to "cleanse" swapper_pg_dir of the identity mappings _after_ resume. That task doesn't depend on SMP or not.

Thanks,
FrankH.

Currently, switch_mm_for_reboot only takes out a 1:1 mapping from 0x0 to TASK_SIZE during reboot. For situations where we actually want to turn off the MMU (e.g. kexec, hibernate, CPU hotplug) we want to map as much memory as possible using the identity mapping so that we increase the chance of mapping our reset code.

This patch introduces a new reboot mode, 'k', which remaps all of memory apart from the kernel (PAGE_OFFSET - _end) and an additional page immediately following it, which can be used as a temporary stack if valid memory is available there. Note that this change makes it necessary to manipulate and switch to the swapper page tables rather than hijack the current task.

---
 arch/arm/include/asm/idmap.h |    7 +++++++
 arch/arm/mm/idmap.c          |   30 +++++++++++++++++++++---------
 2 files changed, 28 insertions(+), 9 deletions(-)

index ea9517e..ae0697d 100644
@@ -2,6 +2,7 @@
 #define _ARM_IDMAP_H
 
 #include <asm/page.h>
+#include <asm/sections.h>
 
 void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end);
 

@@ -11,6 +12,12 @@ void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end);
 void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end) {};
 #endif
 
+/* Modes understood from arm_machine_{restart,reset}. */
+#define MODE_REMAP_KERNEL	'k'
+
+/* Page reserved after the kernel image. */
+#define RESERVE_STACK_PAGE	ALIGN((unsigned long)_end + PAGE_SIZE, SZ_1M)
+
 void setup_mm_for_reboot(char mode);
 
 #endif	/* _ARM_IDMAP_H */

index 4ae0f09..e4ae3c5 100644
@@ -75,17 +75,29 @@ void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
 #endif
 
 /*
- * In order to soft-boot, we need to insert a 1:1 mapping in place of
- * the user-mode pages.  This will then ensure that we have predictable
- * results when turning the mmu off
+ * In order to soft-boot, we need to insert a 1:1 mapping of memory.
+ * This will then ensure that we have predictable results when turning
+ * the mmu off.
  */
 void setup_mm_for_reboot(char mode)
 {
-	/*
-	 * We need to access to user-mode page tables here. For kernel threads
-	 * we don't have any user-mode mappings so we use the context that we
-	 * "borrowed".
-	 */
-	identity_mapping_add(current->active_mm->pgd, 0, TASK_SIZE);
+
+	identity_mapping_add(swapper_pg_dir, 0, TASK_SIZE);
+	if (mode == MODE_REMAP_KERNEL) {
+		/*
+		 * Extend the flat mapping into kernelspace.
+		 * We leave room for the kernel image and a `reboot stack'.
+		 */
+		identity_mapping_add(swapper_pg_dir, TASK_SIZE, PAGE_OFFSET);
+		identity_mapping_add(swapper_pg_dir, RESERVE_STACK_PAGE, 0);
+	}
+
+	/* Clean and invalidate L1. */
+	flush_cache_all();
+
+	/* Switch exclusively to kernel mappings. */
+	cpu_switch_mm(swapper_pg_dir, &init_mm);
+
+	/* Flush the TLB. */
 	local_flush_tlb_all();
 }

1.7.0.4

The macros for invoking functions via the processor struct in the MULTI_CPU case define the arguments as part of the macros, making it impossible to take the address of those functions.

This patch removes the arguments from the macro definitions so that we can take the address of these functions like we can for the !MULTI_CPU case.

---
 arch/arm/include/asm/proc-fns.h |   14 +++++++-------
 1 files changed, 7 insertions(+), 7 deletions(-)

index 8ec535e..633d1cb 100644
@@ -82,13 +82,13 @@ extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
 extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
 extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
 #else
-#define cpu_proc_init()			processor._proc_init()
-#define cpu_proc_fin()			processor._proc_fin()
-#define cpu_reset(addr)			processor.reset(addr)
-#define cpu_do_idle()			processor._do_idle()
-#define cpu_dcache_clean_area(addr,sz)	processor.dcache_clean_area(addr,sz)
-#define cpu_set_pte_ext(ptep,pte,ext)	processor.set_pte_ext(ptep,pte,ext)
-#define cpu_do_switch_mm(pgd,mm)	processor.switch_mm(pgd,mm)
+#define cpu_proc_init			processor._proc_init
+#define cpu_proc_fin			processor._proc_fin
+#define cpu_reset			processor.reset
+#define cpu_do_idle			processor._do_idle
+#define cpu_dcache_clean_area		processor.dcache_clean_area
+#define cpu_set_pte_ext			processor.set_pte_ext
+#define cpu_do_switch_mm		processor.switch_mm
 #endif
 
 extern void cpu_resume(void);

1.7.0.4

Tools such as kexec and CPU hotplug require a way to reset the processor and branch to some code in physical space. This requires various bits of jiggery pokery with the caches and MMU which, when it goes wrong, tends to lock up the system.

This patch implements a new function, arm_machine_reset, for consolidating this code in one place where it can be used by multiple subsystems.

---
 arch/arm/include/asm/system.h |    1 +
 arch/arm/kernel/process.c     |   72 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 65 insertions(+), 8 deletions(-)

index 832888d..cd2a3cd 100644
@@ -108,6 +108,7 @@ extern int cpu_architecture(void);
 extern void cpu_init(void);
 
 void arm_machine_restart(char mode, const char *cmd);
+void arm_machine_reset(unsigned long reset_code_phys);
 extern void (*arm_pm_restart)(char str, const char *cmd);
 
 #define UDBG_UNDEFINED	(1 << 0)

1.7.0.4

Now that there is a common way to reset the machine, let's use it instead of reinventing the wheel in the kexec backend.

---
 arch/arm/kernel/machine_kexec.c |   14 ++------------
 1 files changed, 2 insertions(+), 12 deletions(-)

index e59bbd4..f84567c 100644
@@ -16,8 +16,6 @@
 extern const unsigned char relocate_new_kernel[];
 extern const unsigned int relocate_new_kernel_size;
 
-extern void setup_mm_for_reboot(char mode);
-
 extern unsigned long kexec_start_address;
 extern unsigned long kexec_indirection_page;
 extern unsigned long kexec_mach_type;

@@ -111,14 +109,6 @@ void machine_kexec(struct kimage *image)
 
 	if (kexec_reinit)
 		kexec_reinit();
-	local_irq_disable();
-	local_fiq_disable();
-	setup_mm_for_reboot(0); /* mode is not used, so just pass 0*/
-	flush_cache_all();
-	outer_flush_all();
-	outer_disable();
-	cpu_proc_fin();
-	outer_inv_all();
-	flush_cache_all();
-	cpu_reset(reboot_code_buffer_phys);
+
+	arm_machine_reset(reboot_code_buffer_phys);
 }

1.7.0.4

Sending IPI_CPU_STOP to a CPU causes it to execute a busy cpu_relax loop forever. This makes it impossible to kexec successfully on an SMP system since the secondary CPUs do not reset.

This patch adds a callback to platform_cpu_kill, defined when CONFIG_HOTPLUG_CPU=y, from the ipi_cpu_stop handling code. This function currently just returns 1 on all platforms that define it but allows them to do something more sophisticated in the future.

---
 arch/arm/Kconfig      |    2 +-
 arch/arm/kernel/smp.c |    4 ++++
 2 files changed, 5 insertions(+), 1 deletions(-)

index 9adc278..c26d8f6 100644
@@ -1795,7 +1795,7 @@ config XIP_PHYS_ADDR
 
 config KEXEC
 	bool "Kexec system call (EXPERIMENTAL)"
-	depends on EXPERIMENTAL
+	depends on EXPERIMENTAL && (!SMP || HOTPLUG_CPU)
 	help
 	  kexec is a system call that implements the ability to shutdown your
 	  current kernel, and to start another kernel.  It is like a reboot

index dfc76aa..39189bf 100644
@@ -548,6 +548,10 @@ static void ipi_cpu_stop(unsigned int cpu)
 	local_fiq_disable();
 	local_irq_disable();
 
+#ifdef CONFIG_HOTPLUG_CPU
+	platform_cpu_kill(cpu);
+#endif
+
 	while (1)
 		cpu_relax();
 }

1.7.0.4