arm64 kexec kernel patches V5

Some of the macros defined in kvm_arm.h are useful in assembly files, but are not compatible with the assembler. Change any C language integer constant definitions using appended U, UL, or ULL to the UL() preprocessor macro. Also, add a preprocessor include of the asm/memory.h file which defines the UL() macro.

Fixes build errors like these when using kvm_arm.h in assembly source files:

Error: unexpected characters following instruction at operand 3 -- `and x0,x1,#((1U<<25)-1)'

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/include/asm/kvm_arm.h | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

arch/arm64/include/asm/kvm_arm.h +11 -10

index cc83520..9038a26 100644
@@ -18,6 +18,7 @@
 #ifndef __ARM64_KVM_ARM_H__
 #define __ARM64_KVM_ARM_H__
 
+#include <asm/memory.h>
 #include <asm/types.h>
 
 /* Hyp Configuration Register (HCR) bits */

@@ -149,9 +150,9 @@
 #endif
 
 #define VTTBR_BADDR_SHIFT (VTTBR_X - 1)
-#define VTTBR_BADDR_MASK  (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)
-#define VTTBR_VMID_SHIFT  (48LLU)
-#define VTTBR_VMID_MASK	  (0xffLLU << VTTBR_VMID_SHIFT)
+#define VTTBR_BADDR_MASK  (((UL(1) << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)
+#define VTTBR_VMID_SHIFT  (UL(48))
+#define VTTBR_VMID_MASK	  (UL(0xFF) << VTTBR_VMID_SHIFT)
 
 /* Hyp System Trap Register */
 #define HSTR_EL2_TTEE	(1 << 16)

@@ -174,13 +175,13 @@
 
 /* Exception Syndrome Register (ESR) bits */
 #define ESR_EL2_EC_SHIFT	(26)
-#define ESR_EL2_EC		(0x3fU << ESR_EL2_EC_SHIFT)
-#define ESR_EL2_IL		(1U << 25)
+#define ESR_EL2_EC		(UL(0x3f) << ESR_EL2_EC_SHIFT)
+#define ESR_EL2_IL		(UL(1) << 25)
 #define ESR_EL2_ISS		(ESR_EL2_IL - 1)
 #define ESR_EL2_ISV_SHIFT	(24)
-#define ESR_EL2_ISV		(1U << ESR_EL2_ISV_SHIFT)
+#define ESR_EL2_ISV		(UL(1) << ESR_EL2_ISV_SHIFT)
 #define ESR_EL2_SAS_SHIFT	(22)
-#define ESR_EL2_SAS		(3U << ESR_EL2_SAS_SHIFT)
+#define ESR_EL2_SAS		(UL(3) << ESR_EL2_SAS_SHIFT)
 #define ESR_EL2_SSE		(1 << 21)
 #define ESR_EL2_SRT_SHIFT	(16)
 #define ESR_EL2_SRT_MASK	(0x1f << ESR_EL2_SRT_SHIFT)

@@ -194,16 +195,16 @@
 #define ESR_EL2_FSC_TYPE	(0x3c)
 
 #define ESR_EL2_CV_SHIFT	(24)
-#define ESR_EL2_CV		(1U << ESR_EL2_CV_SHIFT)
+#define ESR_EL2_CV		(UL(1) << ESR_EL2_CV_SHIFT)
 #define ESR_EL2_COND_SHIFT	(20)
-#define ESR_EL2_COND		(0xfU << ESR_EL2_COND_SHIFT)
+#define ESR_EL2_COND		(UL(0xf) << ESR_EL2_COND_SHIFT)
 
 
 #define FSC_FAULT	(0x04)
 #define FSC_PERM	(0x0c)
 
 /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
-#define HPFAR_MASK	(~0xFUL)
+#define HPFAR_MASK	(~UL(0xf))
 
 #define ESR_EL2_EC_UNKNOWN	(0x00)
 #define ESR_EL2_EC_WFI		(0x01)

Signature · 8 lines

1.9.1



_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Quoted 9 lines · Geoff Levand

Some of the macros defined in kvm_arm.h are useful in assembly files, but are not compatible with the assembler. Change any C language integer constant definitions using appended U, UL, or ULL to the UL() preprocessor macro. Also, add a preprocessor include of the asm/memory.h file which defines the UL() macro.

Fixes build errors like these when using kvm_arm.h in assembly source files:

Quoted 3 lines

Error: unexpected characters following instruction at operand 3 -- `and x0,x1,#((1U<<25)-1)'

Signed-off-by Geoff Levand <[email protected]>

Thanks for putting this together, Geoff.

Acked-by Mark Rutland <[email protected]>

Quoted 16 lines · Mark Rutland

Quoted 6 lines · Geoff Levand

Some of the macros defined in kvm_arm.h are useful in assembly files, but are not compatible with the assembler. Change any C language integer constant definitions using appended U, UL, or ULL to the UL() preprocessor macro. Also, add a preprocessor include of the asm/memory.h file which defines the UL() macro.

Quoted 6 lines

Fixes build errors like these when using kvm_arm.h in assembly source files:

Error: unexpected characters following instruction at operand 3 -- `and x0,x1,#((1U<<25)-1)'

Signed-off-by Geoff Levand <[email protected]>

Thanks for putting this together, Geoff.

Quoted 1 line

Acked-by Mark Rutland <[email protected]>

This doesn't apply against -rc2 (see below). If you send a rebased version of this patch, I'm happy to queue it for 3.19.

Alternatively, it could go via the kvm tree. Christoffer, Marc, do you have a preference? (the kexec stuff depends on this, so that's why I was thinking of queuing it in the arm64 tree).

When a CPU is reset it needs to be put into the exception level it had when it entered the kernel. Update cpu_reset() to accept an argument el2_switch which signals cpu_reset() to enter the soft reset address at EL2. If el2_switch is not set the soft reset address will be entered at EL1.

Update cpu_soft_restart() and soft_restart() to pass the return of is_hyp_mode_available() as the el2_switch value to cpu_reset(). Also update the comments of cpu_reset(), cpu_soft_restart() and soft_restart() to reflect this change.

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/include/asm/proc-fns.h |  4 ++--
 arch/arm64/kernel/process.c       |  6 ++++-
 arch/arm64/mm/proc.S              | 47 +++++++++++++++++++++++++++++----------
 3 files changed, 42 insertions(+), 15 deletions(-)

arch/arm64/include/asm/proc-fns.h +2 -2

index 9a8fd84..339394d 100644
@@ -32,8 +32,8 @@ extern void cpu_cache_off(void);
 extern void cpu_do_idle(void);
 extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
 extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
-void cpu_soft_restart(phys_addr_t cpu_reset,
-		unsigned long addr) __attribute__((noreturn));
+void cpu_soft_restart(phys_addr_t cpu_reset, unsigned long el2_switch,
+		      unsigned long addr) __attribute__((noreturn));
 extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr);
 extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr);
 

arch/arm64/kernel/process.c +5 -1

index bf66922..0a3414b 100644
@@ -50,6 +50,7 @@
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/stacktrace.h>
+#include <asm/virt.h>
 
 #ifdef CONFIG_CC_STACKPROTECTOR
 #include <linux/stackprotector.h>

@@ -60,7 +61,10 @@ EXPORT_SYMBOL(__stack_chk_guard);
 void soft_restart(unsigned long addr)
 {
 	setup_mm_for_reboot();
-	cpu_soft_restart(virt_to_phys(cpu_reset), addr);
+
+	cpu_soft_restart(virt_to_phys(cpu_reset), is_hyp_mode_available(),
+			 addr);
+
 	/* Should never get here */
 	BUG();
 }

arch/arm64/mm/proc.S +35 -12

index 4e778b1..7467199 100644
@@ -25,6 +25,7 @@
 #include <asm/hwcap.h>
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
+#include <asm/virt.h>
 
 #include "proc-macros.S"
 

@@ -59,27 +60,48 @@ ENTRY(cpu_cache_off)
 ENDPROC(cpu_cache_off)
 
 /*
- *	cpu_reset(loc)
+ * cpu_reset(el2_switch, loc) - Helper for cpu_soft_restart.
  *
- *	Perform a soft reset of the system.  Put the CPU into the same state
- *	as it would be if it had been reset, and branch to what would be the
- *	reset vector. It must be executed with the flat identity mapping.
+ * @cpu_reset: Physical address of the cpu_reset routine.
+ * @el2_switch: Flag to indicate a swich to EL2 is needed.
+ * @addr: Location to jump to for soft reset.
  *
- *	- loc   - location to jump to for soft reset
+ * Put the CPU into the same state as it would be if it had been reset, and
+ * branch to what would be the reset vector. It must be executed with the
+ * flat identity mapping.
  */
+
 	.align	5
+
 ENTRY(cpu_reset)
-	mrs	x1, sctlr_el1
-	bic	x1, x1, #1
-	msr	sctlr_el1, x1			// disable the MMU
+	mrs	x2, sctlr_el1
+	bic	x2, x2, #1
+	msr	sctlr_el1, x2			// disable the MMU
 	isb
-	ret	x0
+
+	cbz	x0, 1f				// el2_switch?
+	mov	x0, x1
+	mov	x1, xzr
+	mov	x2, xzr
+	mov	x3, xzr
+	hvc	#HVC_CALL_FUNC			// no return
+
+1:	ret	x1
 ENDPROC(cpu_reset)
 
+/*
+ * cpu_soft_restart(cpu_reset, el2_switch, addr) - Perform a cpu soft reset.
+ *
+ * @cpu_reset: Physical address of the cpu_reset routine.
+ * @el2_switch: Flag to indicate a swich to EL2 is needed, passed to cpu_reset.
+ * @addr: Location to jump to for soft reset, passed to cpu_reset.
+ *
+ */
+
 ENTRY(cpu_soft_restart)
-	/* Save address of cpu_reset() and reset address */
-	mov	x19, x0
-	mov	x20, x1
+	mov	x19, x0				// cpu_reset
+	mov	x20, x1				// el2_switch
+	mov	x21, x2				// addr
 
 	/* Turn D-cache off */
 	bl	cpu_cache_off

@@ -88,6 +110,7 @@ ENTRY(cpu_soft_restart)
 	bl	flush_cache_all
 
 	mov	x0, x20
+	mov	x1, x21
 	ret	x19
 ENDPROC(cpu_soft_restart)
 

Signature · 8 lines

1.9.1



_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Quoted 5 lines · Geoff Levand

When a CPU is reset it needs to be put into the exception level it had when it entered the kernel. Update cpu_reset() to accept an argument el2_switch which signals cpu_reset() to enter the soft reset address at EL2. If el2_switch is not set the soft reset address will be entered at EL1.

Quoted 4 lines

Update cpu_soft_restart() and soft_restart() to pass the return of is_hyp_mode_available() as the el2_switch value to cpu_reset(). Also update the comments of cpu_reset(), cpu_soft_restart() and soft_restart() to reflect this change.

This will blow up without warning with KVM, and I think we need to address that first.

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

This change is required for kexec re-boot to work properly when a user does not specify a second stage DTB via the kexec --dtb option. When the --dtb option is not specified kexec will prepare the second stage DTB using data from /proc/device-tree.

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/boot/dts/foundation-v8.dts  | 13 +++++++++++--
 arch/arm64/boot/dts/rtsm_ve-aemv8a.dts | 13 +++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

arch/arm64/boot/dts/foundation-v8.dts +1 -2

index 4a06090..2b76c2d 100644
@@ -6,8 +6,6 @@
 
 /dts-v1/;
 
-/memreserve/ 0x80000000 0x00010000;
-
 / {
 	model = "Foundation-v8A";
 	compatible = "arm,foundation-aarch64", "arm,vexpress";

@@ -64,6 +62,17 @@
 		      <0x00000008 0x80000000 0 0x80000000>;
 	};
 
+	reserved-memory {

arch/arm64/boot/dts/foundation-v8.dts +0 -0

 	gic: interrupt-controller@2c001000 {
 		compatible = "arm,cortex-a15-gic", "arm,cortex-a9-gic";
 		#interrupt-cells = <3>;

arch/arm64/boot/dts/rtsm_ve-aemv8a.dts +11 -2

index 572005e..0f80807 100644
@@ -9,8 +9,6 @@
 
 /dts-v1/;
 
-/memreserve/ 0x80000000 0x00010000;
-
 / {
 	model = "RTSM_VE_AEMv8A";
 	compatible = "arm,rtsm_ve,aemv8a", "arm,vexpress";

@@ -67,6 +65,17 @@
 		      <0x00000008 0x80000000 0 0x80000000>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		firmware-memory@0x80000000 {
+			no-map;
+			reg = <0x0 0x80000000 0x0 0x00010000>;
+		};
+	};
+
 	gic: interrupt-controller@2c001000 {
 		compatible = "arm,cortex-a15-gic", "arm,cortex-a9-gic";
 		#interrupt-cells = <3>;

Signature · 8 lines

1.9.1



_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Quoted 4 lines · Geoff Levand

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

The limitation here is in the kernel (and a partially in userspace), so modifying the dts files is a workaround rather than a fix.

It's perfectly valid for people to remain using /memreserve/, so this isn't sufficient. There are also existing DTBs using /memreserve/ which we can't rely on being modified to use reserved-memory.

I think we need to expose memreserves to userspace somehow, potentially along with other DTB header fields. Grant, ideas?

Quoted 16 lines

Quoted 4 lines · Geoff Levand

I did some work on this and I think we just need to convert all the arm64 dts from /memreserve/ to reserved-memory nodes and update the arm64 booting.txt to specify using reserved-memory. I'll prepare a patch for it.

I don't think that is going to be entirely sufficient. There will be platforms that don't get converted over, and this is a generic problem that covers all architectures using DT, not just aarch64. The solution probably needs to include exposing the /memreserve/ sections to userspace. I can see two ways to do this:

1) Create a new property in /sysfs with all the memreserve sctions
2) Live-modify the device tree to put the memreserve data into a node at boot time.

Quoted 5 lines

Option 2 is probably the most generic solution, but it will require some care to make sure there aren't any overlaps if a reserved-memory node already exists.

g.

For reference, here is an old conversation about this exact thing. Reading through it, the opinions I expressed then don't necessarily match what I think now. I still don't think it is a good idea to expose the physical address of the old .dtb blob, but I do agree that the memreserve sections need to be exposed.

https://lists.ozlabs.org/pipermail/linuxppc-dev/2010-July/083993.html

g.

Quoted 13 lines · Mark Rutland

Quoted 4 lines · Geoff Levand

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

The limitation here is in the kernel (and a partially in userspace), so modifying the dts files is a workaround rather than a fix.

It's perfectly valid for people to remain using /memreserve/, so this isn't sufficient. There are also existing DTBs using /memreserve/ which we can't rely on being modified to use reserved-memory.

Quoted 2 lines

I think we need to expose memreserves to userspace somehow, potentially along with other DTB header fields. Grant, ideas?

Yes, I suggested the same thing to Geoff in a separate thread. Here's

Quoted 19 lines · what I

Quoted 19 lines

Quoted 4 lines · Geoff Levand

I did some work on this and I think we just need to convert all the arm64 dts from /memreserve/ to reserved-memory nodes and update the arm64 booting.txt to specify using reserved-memory. I'll prepare a patch for it.

I don't think that is going to be entirely sufficient. There will be platforms that don't get converted over, and this is a generic problem that covers all architectures using DT, not just aarch64. The solution probably needs to include exposing the /memreserve/ sections to userspace. I can see two ways to do this:

1) Create a new property in /sysfs with all the memreserve sctions
2) Live-modify the device tree to put the memreserve data into a node at boot time.

Option 2 is probably the most generic solution, but it will require some care to make sure there aren't any overlaps if a reserved-memory node already exists.

Quoted 10 lines

Quoted 2 lines

g.

For reference, here is an old conversation about this exact thing. Reading through it, the opinions I expressed then don't necessarily match what I think now. I still don't think it is a good idea to expose the physical address of the old .dtb blob, but I do agree that the memreserve sections need to be exposed.

https://lists.ozlabs.org/pipermail/linuxppc-dev/2010-July/083993.html

Quoted 36 lines · Grant Likely

Quoted 9 lines · Mark Rutland

Quoted 4 lines · Geoff Levand

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

The limitation here is in the kernel (and a partially in userspace), so modifying the dts files is a workaround rather than a fix.

Quoted 6 lines

It's perfectly valid for people to remain using /memreserve/, so this isn't sufficient. There are also existing DTBs using /memreserve/ which we can't rely on being modified to use reserved-memory.

I think we need to expose memreserves to userspace somehow, potentially along with other DTB header fields. Grant, ideas?

Yes, I suggested the same thing to Geoff in a separate thread. Here's

Quoted 16 lines · what I

Quoted 16 lines

Quoted 4 lines · Geoff Levand

I did some work on this and I think we just need to convert all the arm64 dts from /memreserve/ to reserved-memory nodes and update the arm64 booting.txt to specify using reserved-memory. I'll prepare a patch for it.

I don't think that is going to be entirely sufficient. There will be platforms that don't get converted over, and this is a generic problem that covers all architectures using DT, not just aarch64. The solution probably needs to include exposing the /memreserve/ sections to userspace. I can see two ways to do this:

1) Create a new property in /sysfs with all the memreserve sctions
2) Live-modify the device tree to put the memreserve data into a node at boot time.

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Option 2 is probably the most generic solution, but it will require some care to make sure there aren't any overlaps if a reserved-memory node already exists.

I would prefer the former currently. While I currently believe that modifying the tree is something we're going to have to do for stateful properties, it's not someting I want to have to do unless absolutely necessary.

Quoted 36 lines · Grant Likely

Quoted 9 lines · Mark Rutland

Quoted 4 lines · Geoff Levand

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

The limitation here is in the kernel (and a partially in userspace), so modifying the dts files is a workaround rather than a fix.

Quoted 6 lines

It's perfectly valid for people to remain using /memreserve/, so this isn't sufficient. There are also existing DTBs using /memreserve/ which we can't rely on being modified to use reserved-memory.

I think we need to expose memreserves to userspace somehow, potentially along with other DTB header fields. Grant, ideas?

Yes, I suggested the same thing to Geoff in a separate thread. Here's

Quoted 16 lines · what I

Quoted 16 lines

Quoted 4 lines · Geoff Levand

I did some work on this and I think we just need to convert all the arm64 dts from /memreserve/ to reserved-memory nodes and update the arm64 booting.txt to specify using reserved-memory. I'll prepare a patch for it.

I don't think that is going to be entirely sufficient. There will be platforms that don't get converted over, and this is a generic problem that covers all architectures using DT, not just aarch64. The solution probably needs to include exposing the /memreserve/ sections to userspace. I can see two ways to do this:

1) Create a new property in /sysfs with all the memreserve sctions
2) Live-modify the device tree to put the memreserve data into a node at boot time.

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Option 2 is probably the most generic solution, but it will require some care to make sure there aren't any overlaps if a reserved-memory node already exists.

I would prefer the former currently. While I currently believe that modifying the tree is something we're going to have to do for stateful properties, it's not someting I want to have to do unless absolutely necessary.

Quoted 7 lines

Quoted 7 lines

For reference, here is an old conversation about this exact thing. Reading through it, the opinions I expressed then don't necessarily match what I think now. I still don't think it is a good idea to expose the physical address of the old .dtb blob, but I do agree that the memreserve sections need to be exposed.

https://lists.ozlabs.org/pipermail/linuxppc-dev/2010-July/083993.html

Another option would be to expose the original DTB as a (read-only) binary file somewhere. That might interact poorly with live tree modification in future, however.

Hi,

Quoted 37 lines · Mark Rutland

Quoted 36 lines · Grant Likely

Quoted 9 lines · Mark Rutland

Quoted 4 lines · Geoff Levand

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

The limitation here is in the kernel (and a partially in userspace), so modifying the dts files is a workaround rather than a fix.

Quoted 6 lines

It's perfectly valid for people to remain using /memreserve/, so this isn't sufficient. There are also existing DTBs using /memreserve/ which we can't rely on being modified to use reserved-memory.

I think we need to expose memreserves to userspace somehow, potentially along with other DTB header fields. Grant, ideas?

Yes, I suggested the same thing to Geoff in a separate thread. Here's

Quoted 16 lines · what I

Quoted 16 lines

Quoted 4 lines · Geoff Levand

I did some work on this and I think we just need to convert all the arm64 dts from /memreserve/ to reserved-memory nodes and update the arm64 booting.txt to specify using reserved-memory. I'll prepare a patch for it.

I don't think that is going to be entirely sufficient. There will be platforms that don't get converted over, and this is a generic problem that covers all architectures using DT, not just aarch64. The solution probably needs to include exposing the /memreserve/ sections to userspace. I can see two ways to do this:

1) Create a new property in /sysfs with all the memreserve sctions
2) Live-modify the device tree to put the memreserve data into a node at boot time.

Quoted 8 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Option 2 is probably the most generic solution, but it will require some care to make sure there aren't any overlaps if a reserved-memory node already exists.

I would prefer the former currently. While I currently believe that modifying the tree is something we're going to have to do for stateful properties, it's not someting I want to have to do unless absolutely necessary.

Current user space kexec utilities use /proc/device-tree and nothing else. The intension of the device tree is to describe the system sufficiently for a kernel to boot, so I think we should put the /memreserve/ info into /proc/device-tree.

We could put the /memreserve/ entries in there directly, or convert to reserved-memory nodes. At the moment I like the idea to convert to reserved-memory nodes.

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

For reference, here is an old conversation about this exact thing. Reading through it, the opinions I expressed then don't necessarily match what I think now. I still don't think it is a good idea to expose the physical address of the old .dtb blob, but I do agree that the memreserve sections need to be exposed.

Quoted 5 lines

Quoted 1 line

Quoted 1 line

https://lists.ozlabs.org/pipermail/linuxppc-dev/2010-July/083993.html

Another option would be to expose the original DTB as a (read-only) binary file somewhere. That might interact poorly with live tree modification in future, however.

I don't like the idea of having two interfaces to get essentially the same info. I think it will be a maintenance problem over time.

Hi,

Quoted 37 lines · Mark Rutland

Quoted 36 lines · Grant Likely

Quoted 9 lines · Mark Rutland

Quoted 4 lines · Geoff Levand

Device tree regions described by /memreserve/ entries are not available in /proc/device-tree, and hence are not available to user space utilities that use /proc/device-tree. In order for these regions to be available, convert any arm64 DTS files using /memreserve/ entries to use reserved-memory nodes.

The limitation here is in the kernel (and a partially in userspace), so modifying the dts files is a workaround rather than a fix.

Quoted 6 lines

It's perfectly valid for people to remain using /memreserve/, so this isn't sufficient. There are also existing DTBs using /memreserve/ which we can't rely on being modified to use reserved-memory.

I think we need to expose memreserves to userspace somehow, potentially along with other DTB header fields. Grant, ideas?

Yes, I suggested the same thing to Geoff in a separate thread. Here's

Quoted 16 lines · what I

Quoted 16 lines

Quoted 4 lines · Geoff Levand

I did some work on this and I think we just need to convert all the arm64 dts from /memreserve/ to reserved-memory nodes and update the arm64 booting.txt to specify using reserved-memory. I'll prepare a patch for it.

I don't think that is going to be entirely sufficient. There will be platforms that don't get converted over, and this is a generic problem that covers all architectures using DT, not just aarch64. The solution probably needs to include exposing the /memreserve/ sections to userspace. I can see two ways to do this:

1) Create a new property in /sysfs with all the memreserve sctions
2) Live-modify the device tree to put the memreserve data into a node at boot time.

Quoted 8 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Option 2 is probably the most generic solution, but it will require some care to make sure there aren't any overlaps if a reserved-memory node already exists.

I would prefer the former currently. While I currently believe that modifying the tree is something we're going to have to do for stateful properties, it's not someting I want to have to do unless absolutely necessary.

Current user space kexec utilities use /proc/device-tree and nothing else. The intension of the device tree is to describe the system sufficiently for a kernel to boot, so I think we should put the /memreserve/ info into /proc/device-tree.

We could put the /memreserve/ entries in there directly, or convert to reserved-memory nodes. At the moment I like the idea to convert to reserved-memory nodes.

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

For reference, here is an old conversation about this exact thing. Reading through it, the opinions I expressed then don't necessarily match what I think now. I still don't think it is a good idea to expose the physical address of the old .dtb blob, but I do agree that the memreserve sections need to be exposed.

Quoted 5 lines

Quoted 1 line

Quoted 1 line

https://lists.ozlabs.org/pipermail/linuxppc-dev/2010-July/083993.html

Another option would be to expose the original DTB as a (read-only) binary file somewhere. That might interact poorly with live tree modification in future, however.

I don't like the idea of having two interfaces to get essentially the same info. I think it will be a maintenance problem over time.

-Geoff

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Adding Leif...

I'm just wondering does UEFI being used for the memory information have any impact here as the DT would not have valid memory nodes either?

I'd assume reserved memory comes from UEFI (or both) in that case?

Perhaps we need to expose memory layout independent of DT, UEFI or anything else.

Quoted 6 lines · Geoff Levand

On Fri, 2014-10-24 at 13:27 +0100, Mark Rutland wrote:
Current user space kexec utilities use /proc/device-tree and nothing else. The intension of the device tree is to describe the system sufficiently for a kernel to boot, so I think we should put the /memreserve/ info into /proc/device-tree.

Quoted 3 lines

We could put the /memreserve/ entries in there directly, or convert to reserved-memory nodes. At the moment I like the idea to convert to reserved-memory nodes.

I'm just wondering does UEFI being used for the memory information have any impact here as the DT would not have valid memory nodes either? I'd assume reserved memory comes from UEFI (or both) in that case?

Change any reference of device tree '/memreserve/' entries in the arm64 booting.txt to refer to 'reserved-memory nodes'. Reserved-memory nodes are the preferred method of specifying reserved memory.

Signed-off-by Geoff Levand <[email protected]>

---
 Documentation/arm64/booting.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

Documentation/arm64/booting.txt +2 -2

index f3c05b5..7446822 100644
@@ -196,7 +196,7 @@ following manner:
   naturally-aligned 64-bit zero-initalised memory location.
 
   These CPUs should spin outside of the kernel in a reserved area of
-  memory (communicated to the kernel by a /memreserve/ region in the
+  memory (communicated to the kernel by a reserved-memory node in the
   device tree) polling their cpu-release-addr location, which must be
   contained in the reserved region.  A wfe instruction may be inserted
   to reduce the overhead of the busy-loop and a sev will be issued by

@@ -209,7 +209,7 @@ following manner:
 - CPUs with a "psci" enable method should remain outside of
   the kernel (i.e. outside of the regions of memory described to the
   kernel in the memory node, or in a reserved area of memory described
-  to the kernel by a /memreserve/ region in the device tree).  The
+  to the kernel by a reserved-memory node in the device tree).  The
   kernel will issue CPU_ON calls as described in ARM document number ARM
   DEN 0022A ("Power State Coordination Interface System Software on ARM
   processors") to bring CPUs into the kernel.

Signature · 8 lines

1.9.1



_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Quoted 3 lines · Geoff Levand

Change any reference of device tree '/memreserve/' entries in the arm64 booting.txt to refer to 'reserved-memory nodes'. Reserved-memory nodes are the preferred method of specifying reserved memory.

Per my comments on patch 5, I don't think this change is sufficient.

However, we should probably update the document to allow reserved-memory nodes.

On an unrelated note we probably need to work out how reserved-memory interacts with the UEFI memory map -- unmappable regions shouldn't be described by UEFI and I hope people don't use reserved-memory as a workaround for broken UEFI tables.

Quoted 8 lines · Mark Rutland

Quoted 3 lines · Geoff Levand

Change any reference of device tree '/memreserve/' entries in the arm64 booting.txt to refer to 'reserved-memory nodes'. Reserved-memory nodes are the preferred method of specifying reserved memory.

Per my comments on patch 5, I don't think this change is sufficient.

Quoted 7 lines

However, we should probably update the document to allow reserved-memory nodes.

On an unrelated note we probably need to work out how reserved-memory interacts with the UEFI memory map -- unmappable regions shouldn't be described by UEFI and I hope people don't use reserved-memory as a workaround for broken UEFI tables.

When booting with UEFI, the boot stub will clear out all memory nodes and (should) clear out reserved regions so that the kernel can use the UEFI memory map as authoritative.

Quoted 18 lines · Grant Likely

Quoted 15 lines · Mark Rutland

Quoted 3 lines · Geoff Levand

Change any reference of device tree '/memreserve/' entries in the arm64 booting.txt to refer to 'reserved-memory nodes'. Reserved-memory nodes are the preferred method of specifying reserved memory.

Per my comments on patch 5, I don't think this change is sufficient.

However, we should probably update the document to allow reserved-memory nodes.

On an unrelated note we probably need to work out how reserved-memory interacts with the UEFI memory map -- unmappable regions shouldn't be described by UEFI and I hope people don't use reserved-memory as a workaround for broken UEFI tables.

Quoted 3 lines

When booting with UEFI, the boot stub will clear out all memory nodes and (should) clear out reserved regions so that the kernel can use the UEFI memory map as authoritative.

We clear memory nodes and memreserves currently.

I was thinking about reserved-memory nodes (for CMA and such), which we don't currently clear.

Quoted 23 lines · Mark Rutland

Quoted 21 lines · Grant Likely

Quoted 8 lines · Mark Rutland

Quoted 3 lines · Geoff Levand

Change any reference of device tree '/memreserve/' entries in the arm64 booting.txt to refer to 'reserved-memory nodes'. Reserved-memory nodes are the preferred method of specifying reserved memory.

Per my comments on patch 5, I don't think this change is sufficient.

Quoted 7 lines

However, we should probably update the document to allow reserved-memory nodes.

On an unrelated note we probably need to work out how reserved-memory interacts with the UEFI memory map -- unmappable regions shouldn't be described by UEFI and I hope people don't use reserved-memory as a workaround for broken UEFI tables.

When booting with UEFI, the boot stub will clear out all memory nodes and (should) clear out reserved regions so that the kernel can use the UEFI memory map as authoritative.

Quoted 4 lines

We clear memory nodes and memreserves currently.

I was thinking about reserved-memory nodes (for CMA and such), which we don't currently clear.

We should either clear them, or make sure that they will coexist happily with the UEFI map. I can think of a few situations, like CMA, where still having the reserved-memory node would be a good idea.

Quoted 24 lines · Grant Likely

Quoted 23 lines · Mark Rutland

Quoted 21 lines · Grant Likely

Quoted 8 lines · Mark Rutland

Quoted 3 lines · Geoff Levand

Change any reference of device tree '/memreserve/' entries in the arm64 booting.txt to refer to 'reserved-memory nodes'. Reserved-memory nodes are the preferred method of specifying reserved memory.

Per my comments on patch 5, I don't think this change is sufficient.

Quoted 7 lines

However, we should probably update the document to allow reserved-memory nodes.

On an unrelated note we probably need to work out how reserved-memory interacts with the UEFI memory map -- unmappable regions shouldn't be described by UEFI and I hope people don't use reserved-memory as a workaround for broken UEFI tables.

When booting with UEFI, the boot stub will clear out all memory nodes and (should) clear out reserved regions so that the kernel can use the UEFI memory map as authoritative.

Quoted 8 lines

Quoted 4 lines

We clear memory nodes and memreserves currently.

I was thinking about reserved-memory nodes (for CMA and such), which we don't currently clear.

We should either clear them, or make sure that they will coexist happily with the UEFI map. I can think of a few situations, like CMA, where still having the reserved-memory node would be a good idea.

My thinking would be that reservations which the kernel could choose to ignore for whatever reason and use for general allocation (e.g. CMA) make sense, but anything else is nonsense if it overlaps with available memory in the UEFI memory map -- it shouldn't have been there to begin with.

I don't know what the best thing to do in that case is. I'd like to explode very loudly during bringup to get vendors to fix their UEFI tables, but in the long run we might just have to reconcile the two and align with the most stringent requirement. Good luck getting another OS working in that case...

Add three new files, kexec.h, machine_kexec.c and relocate_kernel.S to the arm64 architecture that add support for the kexec re-boot mechanism (CONFIG_KEXEC) on arm64 platforms.

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/Kconfig                  |   9 ++
 arch/arm64/include/asm/kexec.h      |  47 +++++++++
 arch/arm64/kernel/Makefile          |   1 +
 arch/arm64/kernel/machine_kexec.c   | 169 +++++++++++++++++++++++++++++++++
 arch/arm64/kernel/relocate_kernel.S | 184 ++++++++++++++++++++++++++++++++++++
 include/uapi/linux/kexec.h          |   1 +
 6 files changed, 411 insertions(+)
 create mode 100644 arch/arm64/include/asm/kexec.h
 create mode 100644 arch/arm64/kernel/machine_kexec.c
 create mode 100644 arch/arm64/kernel/relocate_kernel.S

arch/arm64/Kconfig +0 -0

index f0d3a2d..af03449 100644

arch/arm64/Kconfig +7 -0

+       bool "kexec system call"
+       ---help---
+         kexec is a system call that implements the ability to shutdown your
+         current kernel, and to start another kernel.  It is like a reboot
+         but it is independent of the system firmware.   And like a reboot
+         you can start any kernel with it, not just Linux.
+
 config XEN_DOM0
        def_bool y
        depends on XEN

arch/arm64/include/asm/kexec.h new file +47 -0

new file mode 100644
index 0000000..e7bd7ab
@@ -0,0 +1,47 @@
+/*
+ * kexec for arm64
+ *
+ * Copyright (C) Linaro.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#if !defined(_ARM64_KEXEC_H)
+#define _ARM64_KEXEC_H
+
+/* Maximum physical address we can use pages from */
+
+#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL)
+
+/* Maximum address we can reach in physical address mode */
+
+#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL)
+
+/* Maximum address we can use for the control code buffer */
+
+#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL)
+
+#define KEXEC_CONTROL_PAGE_SIZE        4096
+
+#define KEXEC_ARCH KEXEC_ARCH_ARM64
+
+#if !defined(__ASSEMBLY__)
+
+/**
+ * crash_setup_regs() - save registers for the panic kernel
+ *
+ * @newregs: registers are saved here
+ * @oldregs: registers to be saved (may be %NULL)
+ */
+
+static inline void crash_setup_regs(struct pt_regs *newregs,
+                                   struct pt_regs *oldregs)
+{
+       /* Empty routine needed to avoid build errors. */
+}
+
+#endif /* !defined(__ASSEMBLY__) */
+
+#endif

arch/arm64/kernel/Makefile +1 -0

index 6e9538c..77a7351 100644
@@ -30,6 +30,7 @@ arm64-obj-$(CONFIG_CPU_IDLE)          += cpuidle.o
 arm64-obj-$(CONFIG_JUMP_LABEL)         += jump_label.o
 arm64-obj-$(CONFIG_KGDB)               += kgdb.o
 arm64-obj-$(CONFIG_EFI)                        += efi.o efi-stub.o efi-entry.o
+arm64-obj-$(CONFIG_KEXEC)              += machine_kexec.o relocate_kernel.o
 
 obj-y                                  += $(arm64-obj-y) vdso/
 obj-m                                  += $(arm64-obj-m)

arch/arm64/kernel/machine_kexec.c new file +68 -0

new file mode 100644
index 0000000..95bc8d9
@@ -0,0 +1,169 @@
+/*
+ * kexec for arm64
+ *
+ * Copyright (C) Linaro.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/kexec.h>
+#include <linux/of_fdt.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+#include <asm/cacheflush.h>
+#include <asm/system_misc.h>
+
+/* Global variables for the relocate_kernel routine. */
+
+extern const unsigned char relocate_new_kernel[];
+extern const unsigned long relocate_new_kernel_size;
+extern unsigned long arm64_kexec_dtb_addr;
+extern unsigned long arm64_kexec_kimage_head;
+extern unsigned long arm64_kexec_kimage_start;
+
+/**
+ * kexec_is_dtb - Helper routine to check the device tree header signature.
+ */
+
+static bool kexec_is_dtb(const void *dtb)
+{
+       __be32 magic;
+
+       return get_user(magic, (__be32 *)dtb) ? false :
+               (be32_to_cpu(magic) == OF_DT_HEADER);
+}
+
+/**
+ * kexec_find_dtb_seg - Helper routine to find the dtb segment.
+ */
+
+static const struct kexec_segment *kexec_find_dtb_seg(
+       const struct kimage *image)
+{
+       int i;
+
+       for (i = 0; i < image->nr_segments; i++) {
+               if (kexec_is_dtb(image->segment[i].buf))
+                       return &image->segment[i];
+       }
+
+       return NULL;
+}
+
+void machine_kexec_cleanup(struct kimage *image)
+{
+       /* Empty routine needed to avoid build errors. */
+}
+
+/**
+ * machine_kexec_prepare - Prepare for a kexec reboot.
+ *
+ * Called from the core kexec code when a kernel image is loaded.
+ */
+
+int machine_kexec_prepare(struct kimage *image)
+{

arch/arm64/kernel/machine_kexec.c new file +22 -0

+
+/**
+ * kexec_list_flush - Helper to flush the kimage list to PoC.
+ */
+
+static void kexec_list_flush(unsigned long kimage_head)
+{
+       void *dest;
+       unsigned long *entry;
+
+       for (entry = &kimage_head, dest = NULL; ; entry++) {
+               unsigned int flag = *entry &
+                       (IND_DESTINATION | IND_INDIRECTION | IND_DONE |
+                       IND_SOURCE);
+               void *addr = phys_to_virt(*entry & PAGE_MASK);
+
+               switch (flag) {
+               case IND_INDIRECTION:
+                       entry = (unsigned long *)addr - 1;
+                       __flush_dcache_area(addr, PAGE_SIZE);
+                       break;
+               case IND_DESTINATION:

arch/arm64/kernel/machine_kexec.c new file +19 -0

+               }
+       }
+}
+
+/**
+ * machine_kexec - Do the kexec reboot.
+ *
+ * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC.
+ */
+
+void machine_kexec(struct kimage *image)
+{
+       phys_addr_t reboot_code_buffer_phys;
+       void *reboot_code_buffer;
+
+       BUG_ON(num_online_cpus() > 1);
+
+       arm64_kexec_kimage_head = image->head;
+

arch/arm64/kernel/machine_kexec.c new file +10 -0

+
+       /* Flush the kimage list. */
+
+       kexec_list_flush(image->head);
+
+       pr_info("Bye!\n");
+
+       /* Disable all DAIF exceptions. */
+
+       asm volatile ("msr daifset, #0xf" : : : "memory");

arch/arm64/kernel/machine_kexec.c new file +6 -0

+}
+
+void machine_crash_shutdown(struct pt_regs *regs)
+{
+       /* Empty routine needed to avoid build errors. */
+}

arch/arm64/kernel/relocate_kernel.S new file +12 -0

new file mode 100644
index 0000000..49cf9a0
@@ -0,0 +1,184 @@
+/*
+ * kexec for arm64
+ *
+ * Copyright (C) Linaro.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/assembler.h>
+#include <asm/kexec.h>

arch/arm64/kernel/relocate_kernel.S new file +98 -0

+
+/*
+ * relocate_new_kernel - Put a 2nd stage kernel image in place and boot it.
+ *
+ * The memory that the old kernel occupies may be overwritten when coping the
+ * new image to its final location.  To assure that the relocate_new_kernel
+ * routine which does that copy is not overwritten all code and data needed
+ * by relocate_new_kernel must be between the symbols relocate_new_kernel and
+ * relocate_new_kernel_end.  The machine_kexec() routine will copy
+ * relocate_new_kernel to the kexec control_code_page, a special page which
+ * has been set up to be preserved during the copy operation.
+ */
+
+.globl relocate_new_kernel
+relocate_new_kernel:
+
+       /* Setup the list loop variables. */
+
+       ldr     x18, arm64_kexec_kimage_head    /* x18 = list entry */
+       dcache_line_size x17, x0                /* x17 = dcache line size */
+       mov     x16, xzr                        /* x16 = segment start */
+       mov     x15, xzr                        /* x15 = entry ptr */
+       mov     x14, xzr                        /* x14 = copy dest */
+
+       /* Check if the new image needs relocation. */
+
+       cbz     x18, .Ldone
+       tbnz    x18, IND_DONE_BIT, .Ldone
+
+.Lloop:
+       and     x13, x18, PAGE_MASK             /* x13 = addr */
+
+       /* Test the entry flags. */
+
+.Ltest_source:
+       tbz     x18, IND_SOURCE_BIT, .Ltest_indirection
+
+       /* copy_page(x20 = dest, x21 = src) */
+
+       mov x20, x14
+       mov x21, x13
+
+1:     ldp     x22, x23, [x21]
+       ldp     x24, x25, [x21, #16]
+       ldp     x26, x27, [x21, #32]
+       ldp     x28, x29, [x21, #48]
+       add     x21, x21, #64
+       stnp    x22, x23, [x20]
+       stnp    x24, x25, [x20, #16]
+       stnp    x26, x27, [x20, #32]
+       stnp    x28, x29, [x20, #48]
+       add     x20, x20, #64
+       tst     x21, #(PAGE_SIZE - 1)
+       b.ne    1b
+
+       /* dest += PAGE_SIZE */
+
+       add     x14, x14, PAGE_SIZE
+       b       .Lnext
+
+.Ltest_indirection:
+       tbz     x18, IND_INDIRECTION_BIT, .Ltest_destination
+
+       /* ptr = addr */
+
+       mov     x15, x13
+       b       .Lnext
+
+.Ltest_destination:
+       tbz     x18, IND_DESTINATION_BIT, .Lnext
+
+       /* flush segment */
+
+       bl      .Lflush
+       mov     x16, x13
+
+       /* dest = addr */
+
+       mov     x14, x13
+
+.Lnext:
+       /* entry = *ptr++ */
+
+       ldr     x18, [x15], #8
+
+       /* while (!(entry & DONE)) */
+
+       tbz     x18, IND_DONE_BIT, .Lloop
+
+.Ldone:
+       /* flush last segment */
+
+       bl      .Lflush
+
+       dsb     sy
+       isb
+       ic      ialluis
+       dsb     sy

arch/arm64/kernel/relocate_kernel.S new file +54 -0

+
+/* flush - x17 = line size, x16 = start addr, x14 = end addr. */
+
+.Lflush:
+       cbz     x16, 2f
+       mov     x0, x16
+       sub     x1, x17, #1
+       bic     x0, x0, x1
+1:     dc      civac, x0
+       add     x0, x0, x17
+       cmp     x0, x14
+       b.lo    1b
+2:     ret
+
+.align 3       /* To keep the 64-bit values below naturally aligned. */
+
+/* The machine_kexec routines set these variables. */
+
+/*
+ * arm64_kexec_kimage_start - Copy of image->start, the entry point of the new
+ * image.
+ */
+
+.globl arm64_kexec_kimage_start
+arm64_kexec_kimage_start:
+       .quad   0x0
+
+/*
+ * arm64_kexec_dtb_addr - Physical address of a device tree.
+ */
+
+.globl arm64_kexec_dtb_addr
+arm64_kexec_dtb_addr:
+       .quad   0x0
+
+/*
+ * arm64_kexec_kimage_head - Copy of image->head, the list of kimage entries.
+ */
+
+.globl arm64_kexec_kimage_head
+arm64_kexec_kimage_head:
+       .quad   0x0
+
+.Lrelocate_new_kernel_end:
+
+/*
+ * relocate_new_kernel_size - Number of bytes to copy to the control_code_page.
+ */
+
+.globl relocate_new_kernel_size
+relocate_new_kernel_size:
+       .quad .Lrelocate_new_kernel_end - relocate_new_kernel
+
+.org   KEXEC_CONTROL_PAGE_SIZE

include/uapi/linux/kexec.h +1 -1

index 6925f5b..04626b9 100644
@@ -39,6 +39,7 @@
 #define KEXEC_ARCH_SH      (42 << 16)
 #define KEXEC_ARCH_MIPS_LE (10 << 16)
 #define KEXEC_ARCH_MIPS    ( 8 << 16)
+#define KEXEC_ARCH_ARM64   (183 << 16)
 
 /* The artificial cap on the number of segments passed to kexec_load. */
 #define KEXEC_SEGMENT_MAX 16
--

1.9.1

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

arch/arm64/Kconfig config ARCH_HAS_CACHE_LINE_SIZE · line 317

Quoted 7 lines

+++ b/arch/arm64/Kconfig

arch/arm64/Kconfig +2 -0

@@ -313,6 +313,15 @@ config ARCH_HAS_CACHE_LINE_SIZE
 
 source "mm/Kconfig"
 
+config KEXEC
+       depends on (!SMP || PM_SLEEP_SMP)

In its current state this also depends on !KVM && !EFI (technically you could detect those cases at runtime, but I don't see that in this series).

Quoted 12 lines

arch/arm64/kernel/machine_kexec.c new file +12 -0

+/**
+ * machine_kexec_prepare - Prepare for a kexec reboot.
+ *
+ * Called from the core kexec code when a kernel image is loaded.
+ */
+
+int machine_kexec_prepare(struct kimage *image)
+{
+       const struct kexec_segment *dtb_seg = kexec_find_dtb_seg(image);
+
+       if (!dtb_seg)
+               pr_warn("%s: No device tree segment found.\n", __func__);

arch/arm64/kernel/machine_kexec.c line 78

Quoted 6 lines

arch/arm64/kernel/machine_kexec.c new file +6 -0

+
+       arm64_kexec_dtb_addr = dtb_seg ? dtb_seg->mem : 0;
+       arm64_kexec_kimage_start = image->start;
+
+       return 0;
+}

I thought all of the DTB handling was moving to purgatory?

Quoted 26 lines

arch/arm64/kernel/machine_kexec.c new file +26 -0

+/**
+ * kexec_list_flush - Helper to flush the kimage list to PoC.
+ */
+
+static void kexec_list_flush(unsigned long kimage_head)
+{
+       void *dest;
+       unsigned long *entry;
+
+       for (entry = &kimage_head, dest = NULL; ; entry++) {
+               unsigned int flag = *entry &
+                       (IND_DESTINATION | IND_INDIRECTION | IND_DONE |
+                       IND_SOURCE);
+               void *addr = phys_to_virt(*entry & PAGE_MASK);
+
+               switch (flag) {
+               case IND_INDIRECTION:
+                       entry = (unsigned long *)addr - 1;
+                       __flush_dcache_area(addr, PAGE_SIZE);
+                       break;
+               case IND_DESTINATION:
+                       dest = addr;
+                       break;
+               case IND_SOURCE:
+                       __flush_dcache_area(addr, PAGE_SIZE);
+                       dest += PAGE_SIZE;

arch/arm64/kernel/machine_kexec.c line 110

Quoted 5 lines

arch/arm64/kernel/machine_kexec.c new file +5 -0

+                       break;
+               case IND_DONE:
+                       return;
+               default:
+                       break;

Can an image ever have no flags? Given the presence of IND_NONE I'd assume not, so this looks like a candidate for a BUG().

Quoted 26 lines

arch/arm64/kernel/machine_kexec.c new file +25 -0

+               }
+       }
+}
+
+/**
+ * machine_kexec - Do the kexec reboot.
+ *
+ * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC.
+ */
+
+void machine_kexec(struct kimage *image)
+{
+       phys_addr_t reboot_code_buffer_phys;
+       void *reboot_code_buffer;
+
+       BUG_ON(num_online_cpus() > 1);
+
+       arm64_kexec_kimage_head = image->head;
+
+       reboot_code_buffer_phys = page_to_phys(image->control_code_page);
+       reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys);
+
+       /*
+        * Copy relocate_new_kernel to the reboot_code_buffer for use
+        * after the kernel is shut down.

arch/arm64/kernel/machine_kexec.c line 139

Quoted 4 lines

arch/arm64/kernel/machine_kexec.c new file +4 -0

+        */
+
+       memcpy(reboot_code_buffer, relocate_new_kernel,
+               relocate_new_kernel_size);

Can we get rid of the line gaps between comments and the single function calls they apply to, please? I realise it's a minor thing, but this looks rather inconsistent with the rest of arch/arm64/.

arch/arm64/kernel/machine_kexec.c line 143

Quoted 4 lines

arch/arm64/kernel/machine_kexec.c new file +4 -0

+
+       /* Flush the reboot_code_buffer in preparation for its execution. */
+
+       __flush_dcache_area(reboot_code_buffer, relocate_new_kernel_size);

That code should already be at the PoC per the boot protocol (the entire kernel image should have been clean to the PoC at boot, so the instructions forming relocate_new_kernel are globally visible).

From the looks of it you only need to flush the variables at the very end.

Quoted 21 lines

arch/arm64/kernel/relocate_kernel.S new file +18 -0

@@ -0,0 +1,184 @@
+/*
+ * kexec for arm64
+ *
+ * Copyright (C) Linaro.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/assembler.h>
+#include <asm/kexec.h>
+#include <asm/memory.h>
+#include <asm/page.h>
+#include <asm/proc-macros.S>
+
+/* The list entry flags. */
+

arch/arm64/kernel/relocate_kernel.S line 22

Quoted 4 lines

arch/arm64/kernel/relocate_kernel.S new file +4 -0

+#define IND_DESTINATION_BIT 0
+#define IND_INDIRECTION_BIT 1
+#define IND_DONE_BIT        2
+#define IND_SOURCE_BIT      3

As previously, I think these need to be moved into a common header, and defined in terms of the existing IND_* macros (or vice-versa). I believe you had a patch doing so; what's the status of that?

Quoted 14 lines

arch/arm64/kernel/relocate_kernel.S new file +14 -0

+/*
+ * relocate_new_kernel - Put a 2nd stage kernel image in place and boot it.
+ *
+ * The memory that the old kernel occupies may be overwritten when coping the
+ * new image to its final location.  To assure that the relocate_new_kernel
+ * routine which does that copy is not overwritten all code and data needed
+ * by relocate_new_kernel must be between the symbols relocate_new_kernel and
+ * relocate_new_kernel_end.  The machine_kexec() routine will copy
+ * relocate_new_kernel to the kexec control_code_page, a special page which
+ * has been set up to be preserved during the copy operation.
+ */
+
+.globl relocate_new_kernel
+relocate_new_kernel:

Quoted 6 lines

arch/arm64/kernel/relocate_kernel.S new file +6 -0

+
+       /* start_new_image */
+
+       ldr     x4, arm64_kexec_kimage_start
+       ldr     x0, arm64_kexec_dtb_addr
+       mov     x1, xzr

arch/arm64/kernel/relocate_kernel.S line 130

Quoted 3 lines

arch/arm64/kernel/relocate_kernel.S new file +3 -0

+       mov     x2, xzr
+       mov     x3, xzr
+       br      x4

This last part should be in userspace-provided purgatory. If you have purgatory code which does this then we should be able to rely on that, and we don't have to try to maintain this DTB handling in kernelspace (which I suspect may become painful as the boot protocol evolves).

Hi Mark,

Quoted 11 lines

arch/arm64/Kconfig config ARCH_HAS_CACHE_LINE_SIZE · line 317

Quoted 7 lines

+++ b/arch/arm64/Kconfig

arch/arm64/Kconfig +2 -0

@@ -313,6 +313,15 @@ config ARCH_HAS_CACHE_LINE_SIZE
 
 source "mm/Kconfig"
 
+config KEXEC
+       depends on (!SMP || PM_SLEEP_SMP)

In its current state this also depends on !KVM && !EFI (technically you could detect those cases at runtime, but I don't see that in this series).

A kernel built with CONFIG_EFI is OK if run on a non-EFI system or without using a system's EFI support. I added a patch that adds runtime checks in the kexec_load syscall path to print a message and return failure for situations where KVM or EFI won't work.

Quoted 12 lines

Quoted 12 lines

arch/arm64/kernel/machine_kexec.c new file +12 -0

+/**
+ * machine_kexec_prepare - Prepare for a kexec reboot.
+ *
+ * Called from the core kexec code when a kernel image is loaded.
+ */
+
+int machine_kexec_prepare(struct kimage *image)
+{
+       const struct kexec_segment *dtb_seg = kexec_find_dtb_seg(image);
+
+       if (!dtb_seg)
+               pr_warn("%s: No device tree segment found.\n", __func__);

Quoted 8 lines

arch/arm64/kernel/machine_kexec.c line 78

Quoted 6 lines

arch/arm64/kernel/machine_kexec.c new file +6 -0

+
+       arm64_kexec_dtb_addr = dtb_seg ? dtb_seg->mem : 0;
+       arm64_kexec_kimage_start = image->start;
+
+       return 0;
+}

I thought all of the DTB handling was moving to purgatory?

Non-purgatory booting is needed for kexec-lite. We can do this simple check here which optionally sets x0 to the dtb address to support that. The other solution is to have a trampoline in kexec-lite that sets x0 (basically an absolute minimal purgatory), but I think to do it here is nicer, and is also the same way that the arm arch code does it.

Maybe removing this pr_warn message and just relying on the kexec_image_info() output would be better.

Quoted 26 lines

Quoted 26 lines

arch/arm64/kernel/machine_kexec.c new file +26 -0

+/**
+ * kexec_list_flush - Helper to flush the kimage list to PoC.
+ */
+
+static void kexec_list_flush(unsigned long kimage_head)
+{
+       void *dest;
+       unsigned long *entry;
+
+       for (entry = &kimage_head, dest = NULL; ; entry++) {
+               unsigned int flag = *entry &
+                       (IND_DESTINATION | IND_INDIRECTION | IND_DONE |
+                       IND_SOURCE);
+               void *addr = phys_to_virt(*entry & PAGE_MASK);
+
+               switch (flag) {
+               case IND_INDIRECTION:
+                       entry = (unsigned long *)addr - 1;
+                       __flush_dcache_area(addr, PAGE_SIZE);
+                       break;
+               case IND_DESTINATION:
+                       dest = addr;
+                       break;
+               case IND_SOURCE:
+                       __flush_dcache_area(addr, PAGE_SIZE);
+                       dest += PAGE_SIZE;

Quoted 8 lines

arch/arm64/kernel/machine_kexec.c line 110

Quoted 5 lines

arch/arm64/kernel/machine_kexec.c new file +5 -0

+                       break;
+               case IND_DONE:
+                       return;
+               default:
+                       break;

Can an image ever have no flags? Given the presence of IND_NONE I'd assume not, so this looks like a candidate for a BUG().

Sure, I guess things will blow up before it ever gets here though.

Quoted 27 lines

Quoted 27 lines

arch/arm64/kernel/machine_kexec.c new file +25 -0

+               }
+       }
+}
+
+/**
+ * machine_kexec - Do the kexec reboot.
+ *
+ * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC.
+ */
+
+void machine_kexec(struct kimage *image)
+{
+       phys_addr_t reboot_code_buffer_phys;
+       void *reboot_code_buffer;
+
+       BUG_ON(num_online_cpus() > 1);
+
+       arm64_kexec_kimage_head = image->head;
+
+       reboot_code_buffer_phys = page_to_phys(image->control_code_page);
+       reboot_code_buffer = phys_to_virt(reboot_code_buffer_phys);
+
+       /*
+        * Copy relocate_new_kernel to the reboot_code_buffer for use
+        * after the kernel is shut down.

Quoted 8 lines

arch/arm64/kernel/machine_kexec.c line 139

Quoted 4 lines

arch/arm64/kernel/machine_kexec.c new file +4 -0

+        */
+
+       memcpy(reboot_code_buffer, relocate_new_kernel,
+               relocate_new_kernel_size);

Can we get rid of the line gaps between comments and the single function calls they apply to, please? I realise it's a minor thing, but this looks rather inconsistent with the rest of arch/arm64/.

checkpatch doesn't seem to mind, but sure, I can do that.

Quoted 11 lines

arch/arm64/kernel/machine_kexec.c line 143

Quoted 4 lines

arch/arm64/kernel/machine_kexec.c new file +4 -0

+
+       /* Flush the reboot_code_buffer in preparation for its execution. */
+
+       __flush_dcache_area(reboot_code_buffer, relocate_new_kernel_size);

That code should already be at the PoC per the boot protocol (the entire kernel image should have been clean to the PoC at boot, so the instructions forming relocate_new_kernel are globally visible).

From the looks of it you only need to flush the variables at the very end.

We copy the relocate_new_kernel routine to reboot_code_buffer, which is a buffer allocated by the kexec core with alloc_pages(). That copy of relocate_new_kernel is what we are flushing here.

I believe we need to flush that buffer out to PoC so we can execute the code it contains after cpu_soft_restart().

Quoted 22 lines

Quoted 22 lines

arch/arm64/kernel/relocate_kernel.S new file +18 -0

@@ -0,0 +1,184 @@
+/*
+ * kexec for arm64
+ *
+ * Copyright (C) Linaro.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/assembler.h>
+#include <asm/kexec.h>
+#include <asm/memory.h>
+#include <asm/page.h>
+#include <asm/proc-macros.S>
+
+/* The list entry flags. */
+

Quoted 8 lines

arch/arm64/kernel/relocate_kernel.S line 22

Quoted 4 lines

arch/arm64/kernel/relocate_kernel.S new file +4 -0

+#define IND_DESTINATION_BIT 0
+#define IND_INDIRECTION_BIT 1
+#define IND_DONE_BIT        2
+#define IND_SOURCE_BIT      3

As previously, I think these need to be moved into a common header, and defined in terms of the existing IND_* macros (or vice-versa). I believe you had a patch doing so; what's the status of that?

Still working to get it merged:

https://lkml.org/lkml/2014/11/12/675

Quoted 15 lines

Quoted 15 lines

arch/arm64/kernel/relocate_kernel.S new file +14 -0

+/*
+ * relocate_new_kernel - Put a 2nd stage kernel image in place and boot it.
+ *
+ * The memory that the old kernel occupies may be overwritten when coping the
+ * new image to its final location.  To assure that the relocate_new_kernel
+ * routine which does that copy is not overwritten all code and data needed
+ * by relocate_new_kernel must be between the symbols relocate_new_kernel and
+ * relocate_new_kernel_end.  The machine_kexec() routine will copy
+ * relocate_new_kernel to the kexec control_code_page, a special page which
+ * has been set up to be preserved during the copy operation.
+ */
+
+.globl relocate_new_kernel
+relocate_new_kernel:

...

Quoted 6 lines

Quoted 6 lines

arch/arm64/kernel/relocate_kernel.S new file +6 -0

+
+       /* start_new_image */
+
+       ldr     x4, arm64_kexec_kimage_start
+       ldr     x0, arm64_kexec_dtb_addr
+       mov     x1, xzr

Quoted 8 lines

arch/arm64/kernel/relocate_kernel.S line 130

Quoted 3 lines

arch/arm64/kernel/relocate_kernel.S new file +3 -0

+       mov     x2, xzr
+       mov     x3, xzr
+       br      x4

This last part should be in userspace-provided purgatory. If you have purgatory code which does this then we should be able to rely on that, and we don't have to try to maintain this DTB handling in kernelspace (which I suspect may become painful as the boot protocol evolves).

I think the putting the dtb address in x0 is already fixed. There are users with firmware that does this and any change to the boot protocol will have to work with it.

As I mentioned above, we need a solution for non-purgatory re-boot and I think this is the best way.

Quoted 29 lines · Mark Rutland

Quoted 12 lines

Quoted 12 lines

arch/arm64/kernel/machine_kexec.c new file +12 -0

+/**
+ * machine_kexec_prepare - Prepare for a kexec reboot.
+ *
+ * Called from the core kexec code when a kernel image is loaded.
+ */
+
+int machine_kexec_prepare(struct kimage *image)
+{
+       const struct kexec_segment *dtb_seg = kexec_find_dtb_seg(image);
+
+       if (!dtb_seg)
+               pr_warn("%s: No device tree segment found.\n", __func__);

Quoted 8 lines

arch/arm64/kernel/machine_kexec.c line 78

Quoted 6 lines

arch/arm64/kernel/machine_kexec.c new file +6 -0

+
+       arm64_kexec_dtb_addr = dtb_seg ? dtb_seg->mem : 0;
+       arm64_kexec_kimage_start = image->start;
+
+       return 0;
+}

I thought all of the DTB handling was moving to purgatory?

Non-purgatory booting is needed for kexec-lite. We can do this simple check here which optionally sets x0 to the dtb address to support that. The other solution is to have a trampoline in kexec-lite that sets x0 (basically an absolute minimal purgatory), but I think to do it here is nicer, and is also the same way that the arm arch code does it.

Quoted 2 lines

Maybe removing this pr_warn message and just relying on the kexec_image_info() output would be better.

I mentioned previously that I don't think the "kexec-lite" approach is a good one, especially if we're going to have userspace purgatory code anyway. It embeds a policy w.r.t. the segment handling within the kernel, on the assumption of a specific use-case for what is a more general mechanism.

Unfortunately secureboot with kexec_file_load will require a kernelspace purgatory and likely special DT handling, but it's already a far more limited interface.

Quoted 32 lines

Quoted 14 lines

Quoted 14 lines

arch/arm64/kernel/relocate_kernel.S new file +14 -0

+/*
+ * relocate_new_kernel - Put a 2nd stage kernel image in place and boot it.
+ *
+ * The memory that the old kernel occupies may be overwritten when coping the
+ * new image to its final location.  To assure that the relocate_new_kernel
+ * routine which does that copy is not overwritten all code and data needed
+ * by relocate_new_kernel must be between the symbols relocate_new_kernel and
+ * relocate_new_kernel_end.  The machine_kexec() routine will copy
+ * relocate_new_kernel to the kexec control_code_page, a special page which
+ * has been set up to be preserved during the copy operation.
+ */
+
+.globl relocate_new_kernel
+relocate_new_kernel:

...

Quoted 6 lines

Quoted 6 lines

arch/arm64/kernel/relocate_kernel.S new file +6 -0

+
+       /* start_new_image */
+
+       ldr     x4, arm64_kexec_kimage_start
+       ldr     x0, arm64_kexec_dtb_addr
+       mov     x1, xzr

Quoted 8 lines

arch/arm64/kernel/relocate_kernel.S line 130

Quoted 3 lines

arch/arm64/kernel/relocate_kernel.S new file +3 -0

+       mov     x2, xzr
+       mov     x3, xzr
+       br      x4

This last part should be in userspace-provided purgatory. If you have purgatory code which does this then we should be able to rely on that, and we don't have to try to maintain this DTB handling in kernelspace (which I suspect may become painful as the boot protocol evolves).

Quoted 3 lines

I think the putting the dtb address in x0 is already fixed. There are users with firmware that does this and any change to the boot protocol will have to work with it.

Sure, but that is the _Linux_ boot protocol, and the Kconfig description of kexec stats "you can start any kernel with it, not just Linux". Why should we embed Linux-specific details into a supposedly generic mechanism?

We may also extend the boot protocol, and I would rather not have to manage the complexity of each possible extension within the kernel, especially given that the only context we can pass in kexec is segments.

Quoted 2 lines

As I mentioned above, we need a solution for non-purgatory re-boot and I think this is the best way.

Why do we need a solution for "non-purgatory re-boot"? As far as I can see this is a non-problem.

arch/arm64/kernel/relocate_kernel.S new file +134 -0

new file mode 100644
index 0000000..49cf9a0
@@ -0,0 +1,184 @@
+/*
+ * kexec for arm64
+ *
+ * Copyright (C) Linaro.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/assembler.h>
+#include <asm/kexec.h>
+#include <asm/memory.h>
+#include <asm/page.h>
+#include <asm/proc-macros.S>
+
+/* The list entry flags. */
+
+#define IND_DESTINATION_BIT 0
+#define IND_INDIRECTION_BIT 1
+#define IND_DONE_BIT        2
+#define IND_SOURCE_BIT      3
+
+/*
+ * relocate_new_kernel - Put a 2nd stage kernel image in place and boot it.
+ *
+ * The memory that the old kernel occupies may be overwritten when coping the
+ * new image to its final location.  To assure that the relocate_new_kernel
+ * routine which does that copy is not overwritten all code and data needed
+ * by relocate_new_kernel must be between the symbols relocate_new_kernel and
+ * relocate_new_kernel_end.  The machine_kexec() routine will copy
+ * relocate_new_kernel to the kexec control_code_page, a special page which
+ * has been set up to be preserved during the copy operation.
+ */
+
+.globl relocate_new_kernel
+relocate_new_kernel:
+
+       /* Setup the list loop variables. */
+
+       ldr     x18, arm64_kexec_kimage_head    /* x18 = list entry */
+       dcache_line_size x17, x0                /* x17 = dcache line size */
+       mov     x16, xzr                        /* x16 = segment start */
+       mov     x15, xzr                        /* x15 = entry ptr */
+       mov     x14, xzr                        /* x14 = copy dest */
+
+       /* Check if the new image needs relocation. */
+
+       cbz     x18, .Ldone
+       tbnz    x18, IND_DONE_BIT, .Ldone
+
+.Lloop:
+       and     x13, x18, PAGE_MASK             /* x13 = addr */
+
+       /* Test the entry flags. */
+
+.Ltest_source:
+       tbz     x18, IND_SOURCE_BIT, .Ltest_indirection
+
+       /* copy_page(x20 = dest, x21 = src) */
+
+       mov x20, x14
+       mov x21, x13
+
+1:     ldp     x22, x23, [x21]
+       ldp     x24, x25, [x21, #16]
+       ldp     x26, x27, [x21, #32]
+       ldp     x28, x29, [x21, #48]
+       add     x21, x21, #64
+       stnp    x22, x23, [x20]
+       stnp    x24, x25, [x20, #16]
+       stnp    x26, x27, [x20, #32]
+       stnp    x28, x29, [x20, #48]
+       add     x20, x20, #64
+       tst     x21, #(PAGE_SIZE - 1)
+       b.ne    1b
+
+       /* dest += PAGE_SIZE */
+
+       add     x14, x14, PAGE_SIZE
+       b       .Lnext
+
+.Ltest_indirection:
+       tbz     x18, IND_INDIRECTION_BIT, .Ltest_destination
+
+       /* ptr = addr */
+
+       mov     x15, x13
+       b       .Lnext
+
+.Ltest_destination:
+       tbz     x18, IND_DESTINATION_BIT, .Lnext
+
+       /* flush segment */
+
+       bl      .Lflush
+       mov     x16, x13
+
+       /* dest = addr */
+
+       mov     x14, x13
+
+.Lnext:
+       /* entry = *ptr++ */
+
+       ldr     x18, [x15], #8
+
+       /* while (!(entry & DONE)) */
+
+       tbz     x18, IND_DONE_BIT, .Lloop
+
+.Ldone:
+       /* flush last segment */
+
+       bl      .Lflush
+
+       dsb     sy
+       isb
+       ic      ialluis
+       dsb     sy
+       isb
+
+       /* start_new_image */
+
+       ldr     x4, arm64_kexec_kimage_start
+       ldr     x0, arm64_kexec_dtb_addr
+       mov     x1, xzr
+       mov     x2, xzr
+       mov     x3, xzr
+       br      x4
+
+/* flush - x17 = line size, x16 = start addr, x14 = end addr. */
+
+.Lflush:

arch/arm64/kernel/relocate_kernel.S new file +40 -0

+.align 3       /* To keep the 64-bit values below naturally aligned. */
+
+/* The machine_kexec routines set these variables. */
+
+/*
+ * arm64_kexec_kimage_start - Copy of image->start, the entry point of the new
+ * image.
+ */
+
+.globl arm64_kexec_kimage_start
+arm64_kexec_kimage_start:
+       .quad   0x0
+
+/*
+ * arm64_kexec_dtb_addr - Physical address of a device tree.
+ */
+
+.globl arm64_kexec_dtb_addr
+arm64_kexec_dtb_addr:
+       .quad   0x0
+
+/*
+ * arm64_kexec_kimage_head - Copy of image->head, the list of kimage entries.
+ */
+
+.globl arm64_kexec_kimage_head
+arm64_kexec_kimage_head:
+       .quad   0x0
+
+.Lrelocate_new_kernel_end:
+
+/*
+ * relocate_new_kernel_size - Number of bytes to copy to the control_code_page.
+ */
+
+.globl relocate_new_kernel_size
+relocate_new_kernel_size:
+       .quad .Lrelocate_new_kernel_end - relocate_new_kernel
+
+.org   KEXEC_CONTROL_PAGE_SIZE

include/uapi/linux/kexec.h +1 -1

index 6925f5b..04626b9 100644
@@ -39,6 +39,7 @@
 #define KEXEC_ARCH_SH      (42 << 16)
 #define KEXEC_ARCH_MIPS_LE (10 << 16)
 #define KEXEC_ARCH_MIPS    ( 8 << 16)
+#define KEXEC_ARCH_ARM64   (183 << 16)
 
 /* The artificial cap on the number of segments passed to kexec_load. */
 #define KEXEC_SEGMENT_MAX 16
--

1.9.1

I have issue in flushing with L3 cache on . kexec does not reboot with L3. Is the above logic of flushing (clean + invalidate) after copying the image data is correct? Is there exist the danger of image data being overwritten by invalid cache lines?

I tried with only invalidate(dc ivac); but that also not seems to be working.

Finally I moved the entire flushing logic before copying then things started working. Please see the code change below.

Hi Arun, Dave,

Quoted 5 lines · Arun Chandran

I have issue in flushing with L3 cache on . kexec does not reboot with L3. Is the above logic of flushing (clean + invalidate) after copying the image data is correct? Is there exist the danger of image data being overwritten by invalid cache lines?

Quoted 5 lines

I tried with only invalidate(dc ivac); but that also not seems to be working.

Finally I moved the entire flushing logic before copying then things started working. Please see the code change below.

I added this with some minor changes. Please let me know if it works for you.

Hi Arun, Dave,

Quoted 5 lines · Arun Chandran

I have issue in flushing with L3 cache on . kexec does not reboot with L3. Is the above logic of flushing (clean + invalidate) after copying the image data is correct? Is there exist the danger of image data being overwritten by invalid cache lines?

Quoted 5 lines

I tried with only invalidate(dc ivac); but that also not seems to be working.

Finally I moved the entire flushing logic before copying then things started working. Please see the code change below.

I added this with some minor changes. Please let me know if it works for you.

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/configs/defconfig | 1 +
 1 file changed, 1 insertion(+)

arch/arm64/configs/defconfig +0 -0

index d92ef3c..ebf8b3f 100644

arch/arm64/configs/defconfig +0 -0

 CONFIG_CMDLINE="console=ttyAMA0"
 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
 CONFIG_COMPAT=y

Signature · 8 lines

1.9.1



_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Quoted 5 lines · Geoff Levand

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/configs/defconfig | 1 +
 1 file changed, 1 insertion(+)

arch/arm64/configs/defconfig CONFIG_PREEMPT=y · line 42

Quoted 9 lines

arch/arm64/configs/defconfig +1 -0

index d92ef3c..ebf8b3f 100644
@@ -39,6 +39,7 @@ CONFIG_PREEMPT=y
 CONFIG_KSM=y
 CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_CMA=y
+CONFIG_KEXEC=y

Given this is going to be incompatible with KVM, and KVM is already in defconfig, I don't think we can add this until that incompatibility is fixed (or at the very least detected and handled gracefully).

Quoted 6 lines · Mark Rutland

Quoted 5 lines · Geoff Levand

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/configs/defconfig | 1 +
 1 file changed, 1 insertion(+)

Quoted 13 lines

arch/arm64/configs/defconfig CONFIG_PREEMPT=y · line 42

Quoted 9 lines

arch/arm64/configs/defconfig +1 -0

index d92ef3c..ebf8b3f 100644
@@ -39,6 +39,7 @@ CONFIG_PREEMPT=y
 CONFIG_KSM=y
 CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_CMA=y
+CONFIG_KEXEC=y

Given this is going to be incompatible with KVM, and KVM is already in defconfig, I don't think we can add this until that incompatibility is fixed (or at the very least detected and handled gracefully).

Sure, I can put in a workaround to print out a message and fail the kexec_load sys call if KVM is configured.

Quoted 21 lines · Geoff Levand

Quoted 19 lines · Mark Rutland

Quoted 10 lines · Geoff Levand

Signed-off-by Geoff Levand <[email protected]>

---
 arch/arm64/configs/defconfig | 1 +
 1 file changed, 1 insertion(+)

arch/arm64/configs/defconfig +0 -0

index d92ef3c..ebf8b3f 100644

arch/arm64/configs/defconfig CONFIG_PREEMPT=y · line 42

Quoted 4 lines

arch/arm64/configs/defconfig +1 -0

@@ -39,6 +39,7 @@ CONFIG_PREEMPT=y
 CONFIG_KSM=y
 CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_CMA=y
+CONFIG_KEXEC=y

Given this is going to be incompatible with KVM, and KVM is already in defconfig, I don't think we can add this until that incompatibility is fixed (or at the very least detected and handled gracefully).

Quoted 2 lines

Sure, I can put in a workaround to print out a message and fail the kexec_load sys call if KVM is configured.

What would be the plan for fixing up kexec to work with kvm were we to do that?

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

2. adding "acpi=off" to 1st kernel boot cmdline, kexec load fails with error like below:
machine_apply_elf_rel: ERROR Unknown type: 261

I did below hack then kexec load works fine,

but kexec -e still does not work
there's nothing more than "Disableing non-boot CPUS ...\n Bye!":

Hi Dave,

Thanks for testing.

Quoted 3 lines · Dave Young

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

Quoted 4 lines

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

You can supply a DTB with the kexec --dtb option, then kexec will not need /proc/device-tree. Please try if you can and let me know what happens.

Quoted 6 lines

2. adding "acpi=off" to 1st kernel boot cmdline, kexec load fails with error like below:
machine_apply_elf_rel: ERROR Unknown type: 261

I did below hack then kexec load works fine,

OK, thanks, I'll add support for R_AARCH64_PREL32 in.

Quoted 2 lines

but kexec -e still does not work
there's nothing more than "Disableing non-boot CPUS ...\n Bye!":

It is really tough to say what happened. The 'Bye!' message is printed just before the 1st stage kernel exits and the 2nd stage kernel is entered. If you have a debugger you can put some breakpoints in there to see how far it gets. That is generally how I figure out what went wrong.

You could try the kexec --lite option, it will do a non-purgatory boot.

You could also try my master branch that has more debugging output. Some of it is through the VE's serial port, so you may need to change that to what your platform has.

Hi Dave,

Thanks for testing.

Quoted 3 lines · Dave Young

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

Quoted 4 lines

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

You can supply a DTB with the kexec --dtb option, then kexec will not need /proc/device-tree. Please try if you can and let me know what happens.

In your kexec-tools code, by default it will create dtb1 from procfs, as for external --dtb it will be dtb2 for comparing with dtb1.

Since dtb1 is a must so it will not work without /proc/device-tree.

There had some discussion about this in the thread. But I think we have no conclusion where to go.

There maybe below possibilities:
1) unflattern dtb both for EFI/acpi booting even if acpi boot does not need it.

2) copy the bootloader passed dtb stub to somewhere, and export the stub to userspace ie. in debugfs or procfs
like /proc/saved_dtb

Thoughts to continue the discuss?

Quoted 4 lines · Dave Young

In your kexec-tools code, by default it will create dtb1 from procfs, as for external --dtb it will be dtb2 for comparing with dtb1.

Since dtb1 is a must so it will not work without /proc/device-tree.

I changed that to allow for no /proc/device-tree if --dtb is passed since it will also allow a kernel built with CONFIG_PROC_FS=n to do kexec re-boot.

Please try it and let me know.

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

Kexec should recreate the tree from scratch in that case.

Quoted 30 lines · Grant Likely

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 6 lines

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

We need to UEFI properties [1] from /chosen to access the memory map and system table (both of which are necessary to access any ACPI tables).

Quoted 1 line

Kexec should recreate the tree from scratch in that case.

I'm not sure if the required information is exposed to userspace elsewhere. Ard, Leif?

Quoted 38 lines · Mark Rutland

Quoted 30 lines · Grant Likely

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 6 lines

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

Quoted 8 lines

We need to UEFI properties [1] from /chosen to access the memory map and system table (both of which are necessary to access any ACPI tables).

Quoted 1 line

Kexec should recreate the tree from scratch in that case.

I'm not sure if the required information is exposed to userspace elsewhere. Ard, Leif?

Personally, I think we should not be using /proc/device-tree at all, but instead retain the original blob in some way and expose that.

We already have /sys/firmware/efi/systab which contains the physical addresses of the UEFI configuration tables. We should probably add an entry for the FDT there anyway, but we would still be looking at mmap(/dev/mem) to access it, which is not a practice we want to encourage, I suppose.

Also, we *must* take the secure boot scenario into account. Booting with an arbitrary user generated DTB is nice, but if you are doing kexec without an initrd, for instance, it would also be nice if we could just reuse the existing DTB without bothering the user for it at all, which would be something we could also allow when running securely.

--
Ard.

Quoted 48 lines · Ard Biesheuvel

Quoted 38 lines · Mark Rutland

Quoted 30 lines · Grant Likely

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 6 lines

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

Quoted 8 lines

We need to UEFI properties [1] from /chosen to access the memory map and system table (both of which are necessary to access any ACPI tables).

Quoted 1 line

Kexec should recreate the tree from scratch in that case.

I'm not sure if the required information is exposed to userspace elsewhere. Ard, Leif?

Quoted 9 lines

Personally, I think we should not be using /proc/device-tree at all, but instead retain the original blob in some way and expose that.

We already have /sys/firmware/efi/systab which contains the physical addresses of the UEFI configuration tables. We should probably add an entry for the FDT there anyway, but we would still be looking at mmap(/dev/mem) to access it, which is not a practice we want to encourage, I suppose.

Nah, strike that. The configuration table entry contains the original fdt, so with the device nodes etc still present. The stub makes

changes to the DTB, and /that/ is the version we want to retain so subsequent kexec reboots can use it.

Quoted 41 lines · Mark Rutland

Quoted 30 lines · Grant Likely

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 6 lines

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

We need to UEFI properties [1] from /chosen to access the memory map and system table (both of which are necessary to access any ACPI tables).

Quoted 5 lines

Quoted 1 line

Kexec should recreate the tree from scratch in that case.

I'm not sure if the required information is exposed to userspace elsewhere. Ard, Leif?

Personally, I think we should not be using /proc/device-tree at all, but instead retain the original blob in some way and expose that.

We already have /sys/firmware/efi/systab which contains the physical addresses of the UEFI configuration tables. We should probably add an entry for the FDT there anyway, but we would still be looking at mmap(/dev/mem) to access it, which is not a practice we want to encourage, I suppose.

Also, we *must* take the secure boot scenario into account. Booting with an arbitrary user generated DTB is nice, but if you are doing kexec without an initrd, for instance, it would also be nice if we could just reuse the existing DTB without bothering the user for it at all, which would be something we could also allow when running securely.

Quoted 42 lines · Ard Biesheuvel

Quoted 41 lines · Mark Rutland

Quoted 30 lines · Grant Likely

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 6 lines

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

We need to UEFI properties [1] from /chosen to access the memory map and system table (both of which are necessary to access any ACPI tables).

Quoted 8 lines

Quoted 5 lines

Quoted 1 line

Kexec should recreate the tree from scratch in that case.

I'm not sure if the required information is exposed to userspace elsewhere. Ard, Leif?

Personally, I think we should not be using /proc/device-tree at all, but instead retain the original blob in some way and expose that.

Grant took objection to that approach previously.

Quoted 5 lines

We already have /sys/firmware/efi/systab which contains the physical addresses of the UEFI configuration tables. We should probably add an entry for the FDT there anyway, but we would still be looking at mmap(/dev/mem) to access it, which is not a practice we want to encourage, I suppose.

We should not encourage use of /dev/mem.

Using /sys/firmware/efi/systab doesn't get us the memory map though, unless that's also exposed?

Quoted 6 lines

Also, we *must* take the secure boot scenario into account. Booting with an arbitrary user generated DTB is nice, but if you are doing kexec without an initrd, for instance, it would also be nice if we could just reuse the existing DTB without bothering the user for it at all, which would be something we could also allow when running securely.

Secure boot has to be handled completely differently. That will require new syscall support, in-kernel purgatory, and so on.

That should not affect the non-secureboot cases where a user wants to load their own DTB (or other objects) into memory for the next OS (which might not be Linux).

Quoted 44 lines · Ard Biesheuvel

Quoted 43 lines · Mark Rutland

Quoted 30 lines · Grant Likely

Quoted 29 lines · Mark Rutland

Quoted 20 lines · Dave Young

Quoted 11 lines · Mark Rutland

Quoted 9 lines · Dave Young

Hi Geoff

I tested your patches. The macihne is using spin-table cpu enable method so I tried maxcpus=1 as you suggested.

There's below issues for me, thoughts?

1. For acpi booting there's no /proc/device-tree so kexec can not find dtb to use.

Quoted 7 lines

Are you absolutely certain of this?

To use ACPI, you must have booted via EFI, as the only mechanism for finding the ACPI tables is via EFI. If booted via EFI, the stub will have created a stub DTB if there is no provided DTB, to pass the command line and pointers to EFI data structures. This stub DTB should be present in the usual place.

Quoted 4 lines

Mark, I used kexec-tools from Geoff's git tree, it will create dtb from procfs maybe I can pass external dtb to kexec-tools. What you mentioned should be true though but I have not get idea how to get the dtb which kernel is using for boot since it is not unflattened.

Ah, sorry. I see the problem now. For ACPI you don't unflatten the tree, so there's nothing to expose at in sysfs/procfs.

Quoted 6 lines

Quoted 3 lines

Somehow we need to unflatten the DTB without exposing it to drivers, such that it can be exposed to userspace in the usual place but drivers don't being probing based off of it.

Is that even necessary? If the tree isn't unflattened, then it is just a stub tree. There really isn't anything interesting in it.

We need to UEFI properties [1] from /chosen to access the memory map and system table (both of which are necessary to access any ACPI tables).

Quoted 1 line

Kexec should recreate the tree from scratch in that case.

Quoted 6 lines

Quoted 3 lines

I'm not sure if the required information is exposed to userspace elsewhere. Ard, Leif?

Personally, I think we should not be using /proc/device-tree at all, but instead retain the original blob in some way and expose that.

Grant took objection to that approach previously.

Quoted 6 lines

We already have /sys/firmware/efi/systab which contains the physical addresses of the UEFI configuration tables. We should probably add an entry for the FDT there anyway, but we would still be looking at mmap(/dev/mem) to access it, which is not a practice we want to encourage, I suppose.

We should not encourage use of /dev/mem.

Using /sys/firmware/efi/systab doesn't get us the memory map though, unless that's also exposed?

Quoted 6 lines

Also, we *must* take the secure boot scenario into account. Booting with an arbitrary user generated DTB is nice, but if you are doing kexec without an initrd, for instance, it would also be nice if we could just reuse the existing DTB without bothering the user for it at all, which would be something we could also allow when running securely.

Secure boot has to be handled completely differently. That will require new syscall support, in-kernel purgatory, and so on.

That should not affect the non-secureboot cases where a user wants to load their own DTB (or other objects) into memory for the next OS (which might not be Linux).

Mark.

Hi All,

This series adds the core support for kexec re-boots on arm64. I have tested with the ARM VE fast model, the ARM Base model and the ARM Foundation model with various kernel config options for both the first and second stage kernels.

To load a second stage kernel and execute a kexec re-boot on arm64 my patches to kexec-tools [2], which have not yet been merged upstream, are needed.

Patches 1-4 rework the arm64 hcall mechanism to give the arm64 soft_restart() routine the ability to switch exception levels from EL1 to EL2 for kernels that were entered in EL2.

Patches 5 and 6 convert the use of device tree /memreserve/ to device tree reserved-memory nodes.

Patch 7 moves proc-macros.S from arm64/mm to arm64/include/asm so that the dcache_line_size macro it defines can be uesd by kexec's relocate kernel routine.

Patches 8-10 add the actual kexec support.

Please consider all patches for inclusion. Any comments or suggestions on how to improve are welcome.

[1] https://git.linaro.org/people/geoff.levand/linux-kexec.git
[2] https://git.linaro.org/people/geoff.levand/kexec-tools.git

Several things are known to have problems on kexec re-boot:

spin-table
----------

PROBLEM: The spin-table enable method does not implement all the methods needed for CPU hot-plug, so the first stage kernel cannot be shutdown properly.

WORK-AROUND: Upgrade to system firmware that provides PSCI enable method support, OR build the first stage kernel with CONFIG_SMP=n, OR pass 'maxcpus=1' on the first stage kernel command line.

FIX: Upgrade system firmware to provide PSCI enable method support.

KVM
---

PROBLEM: KVM acquires hypervisor resources on startup, but does not free those resources on shutdown, so the first stage kernel cannot be shutdown properly.

WORK-AROUND: Build the first stage kernel with CONFIG_KVM=n.

Hi Geoff,

I am trying this on my hardware (apm-mustang.dtb)

Quoted 34 lines · Geoff Levand

Hi All,

This series adds the core support for kexec re-boots on arm64. I have tested with the ARM VE fast model, the ARM Base model and the ARM Foundation model with various kernel config options for both the first and second stage kernels.

To load a second stage kernel and execute a kexec re-boot on arm64 my patches to kexec-tools [2], which have not yet been merged upstream, are needed.

Patches 1-4 rework the arm64 hcall mechanism to give the arm64 soft_restart() routine the ability to switch exception levels from EL1 to EL2 for kernels that were entered in EL2.

Patches 5 and 6 convert the use of device tree /memreserve/ to device tree reserved-memory nodes.

Patch 7 moves proc-macros.S from arm64/mm to arm64/include/asm so that the dcache_line_size macro it defines can be uesd by kexec's relocate kernel routine.

Patches 8-10 add the actual kexec support.

Please consider all patches for inclusion. Any comments or suggestions on how to improve are welcome.

[1] https://git.linaro.org/people/geoff.levand/linux-kexec.git
[2] https://git.linaro.org/people/geoff.levand/kexec-tools.git

Several things are known to have problems on kexec re-boot:

spin-table
----------

Quoted 6 lines

PROBLEM: The spin-table enable method does not implement all the methods needed for CPU hot-plug, so the first stage kernel cannot be shutdown properly.

WORK-AROUND: Upgrade to system firmware that provides PSCI enable method support, OR build the first stage kernel with CONFIG_SMP=n, OR pass 'maxcpus=1' on the first stage kernel command line.

I have CONFIG_SMP=n

Quoted 10 lines

FIX: Upgrade system firmware to provide PSCI enable method support.

KVM
---

PROBLEM: KVM acquires hypervisor resources on startup, but does not free those resources on shutdown, so the first stage kernel cannot be shutdown properly.

WORK-AROUND: Build the first stage kernel with CONFIG_KVM=n.

KVM also disabled.

/root@genericarmv8:~# usr/local/sbin/kexec --lite -l vmlinux
--dtb=apm-mustang.dtb --command-line=
"root=/dev/nfs rw
nfsroot=10.162.103.145:/nfs_root/linaro-image-lamp-genericarmv8,nfsvers=3
 ip=:::::eth0:dhcp panic=1 console=ttyS0,115200
earlyprintk=uart8250-32bit,0x1c020000"

kexec version: 14.10.21.16.36-ga38e0a6 arch_process_options:85: command_line: root=/dev/nfs rw nfsroot=10.162.103.145:/nfs_root/linaro-image-lamp-genericarmv8,nfsvers=3 ip=:::::eth0:dhcp panic=1 console=ttyS0,115200 earlyprintk=uart8250-32bit,0x1
c020000
arch_process_options:87: initrd: (null) arch_process_options:88: dtb: apm-mustang.dtb arch_process_options:89: lite: 1
kernel: 0x7f756e7010 kernel_size: 0x488a308
Modified cmdline: root=/dev/nfs
Unable to find /proc/device-tree//chosen/linux,stdout-path, printing from purgatory is diabled
get_memory_ranges_dt:638: node_1516 memory get_memory_ranges_dt:664: RAM: 0000004000000000 - 0000004400000000 get_memory_ranges_dt:659: SKIP: 0000000000000000 - 0000000000000000 get_memory_ranges_dt:659: SKIP: 0000000000000000 - 0000000000000000 get_memory_ranges_dt:659: SKIP: 0000000000000000 - 0000000000000000 get_memory_ranges_dt:678: Success
elf_arm64_load: PE format: yes
p_vaddr: ffffffc000080000
virt_to_phys: ffffffc000080000 -> 0000004000080000
add_segment_phys_virt: 0000007f756f7010 - 0000007f75d3cf70 (00645f60) -> 0000004000080000 - 00000040006fb000 (0067b000)
elf_arm64_load: text_offset: 0000000000080000
elf_arm64_load: image_size: 000000000067f000
elf_arm64_load: page_offset: ffffffc000000000
elf_arm64_load: memstart: 0000004000000000
virt_to_phys: ffffffc000080000 -> 0000004000080000
elf_arm64_load: e_entry: ffffffc000080000 -> 0000004000080000
virt_to_phys: ffffffc000080000 -> 0000004000080000
Modified cmdline:root=/dev/nfs rw nfsroot=10.162.103.145:/nfs_root/linaro-image-lamp-genericarmv8,nfsvers=3 ip=:::::eth0:dhcp panic=1 console=ttyS0,115200 earlyprintk=uart8250-32bit,0x1c020000
Unable to find /proc/device-tree//chosen/linux,stdout-path, printing from purgatory is diabled
read_cpu_info: dtb_1 cpu-0 (/cpus/cpu@000): hwid-0, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-1 (/cpus/cpu@001): hwid-1, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-2 (/cpus/cpu@100): hwid-100, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-3 (/cpus/cpu@101): hwid-101, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-4 (/cpus/cpu@200): hwid-200, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-5 (/cpus/cpu@201): hwid-201, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-6 (/cpus/cpu@300): hwid-300, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_1 cpu-7 (/cpus/cpu@301): hwid-301, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-0 (/cpus/cpu@000): hwid-0, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-1 (/cpus/cpu@001): hwid-1, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-2 (/cpus/cpu@100): hwid-100, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-3 (/cpus/cpu@101): hwid-101, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-4 (/cpus/cpu@200): hwid-200, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-5 (/cpus/cpu@201): hwid-201, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-6 (/cpus/cpu@300): hwid-300, 'spin-table', cpu-release-addr 400000fff8
read_cpu_info: dtb_2 cpu-7 (/cpus/cpu@301): hwid-301, 'spin-table', cpu-release-addr 400000fff8
check_cpu_properties: hwid-0: OK
check_cpu_properties: hwid-1: OK
check_cpu_properties: hwid-100: OK
check_cpu_properties: hwid-101: OK
check_cpu_properties: hwid-200: OK
check_cpu_properties: hwid-201: OK
check_cpu_properties: hwid-300: OK
check_cpu_properties: hwid-301: OK
dtb: base 4000700000, size 221ch (8732)
add_segment_phys_virt: 0000000024c14c90 - 0000000024c16eac (0000221c) -> 0000004000700000 - 0000004000703000 (00003000)
kexec_load: entry = 0x4000080000 flags = 0xb70000
nr_segments = 2
segment[0].buf = 0x7f756f7010
segment[0].bufsz = 0x645f60
segment[0].mem = 0x4000080000
segment[0].memsz = 0x67b000
segment[1].buf = 0x24c14c90
segment[1].bufsz = 0x221c
segment[1].mem = 0x4000700000
segment[1].memsz = 0x3000

root@genericarmv8:~# /usr/local/sbin/kexec --lite -e
kexec version: 14.10.21.16.36-ga38e0a6
arch_process_options:85: command_line: (null)
arch_process_options:87: initrd: (null)
arch_process_options:88: dtb: (null)
arch_process_options:89: lite: 1
sd 0:0:0:0: [sda] Synchronizing SCSI cache
kexec: Starting new kernel
Bye!