kexec for ia64

I yanked Eric's original patch out of a webpage and bashed it into a recent BK tree. You'll need Randy's full kexec patch
(http://developer.osdl.org/rddunlap/kexec/) in addition to this one to have something remotely useful. It still needs a lot of work:

o userspace tools need ia64 support
o need to deal with in-flight DMA (see FIXME in machine_kexec)

I'm also worried about a few things in this patch. Is relocate_kernel.S really necessary in 2.6? Can we copy the kernel to a contiguous 64MB aligned area, drop into phys mode and just jump to it? Also, what about EFI boot services and PROM tables that the kernel frees part way through boot? Should we copy those into a safe place for the new image at boot time? Or just leave them there if CONFIG_KEXEC is enabled?

Comments and suggestions welcome. It would be really nice to have this stuff working since it appears that crash dumps will be collected with a panic->kexec'd kernel rather than polling mode network/disk writing. (Well, that and reboots would be *much* faster :)

Thanks,
Jesse

# This is a BitKeeper generated diff -Nru style patch. #
# ChangeSet
# 2004/07/26 15:16:34-07:00 [email protected]
# kexec
#

# include/asm-ia64/kexec.h
#   2004/07/26 15:16:25-07:00 [email protected] +15 -0
# 
# include/asm-ia64/kexec.h
#   2004/07/26 15:16:25-07:00 [email protected] +0 -0
#   BitKeeper file /home/jbarnes/working/linux-2.5-kexec/include/asm-ia64/kexec.h
# 
# arch/ia64/kernel/relocate_kernel.S
#   2004/07/26 15:16:24-07:00 [email protected] +97 -0
# 
# arch/ia64/kernel/relocate_kernel.S
#   2004/07/26 15:16:24-07:00 [email protected] +0 -0
#   BitKeeper file /home/jbarnes/working/linux-2.5-kexec/arch/ia64/kernel/relocate_kernel.S
# 
# arch/ia64/kernel/machine_kexec.c
#   2004/07/26 15:16:23-07:00 [email protected] +52 -0
# 
# include/asm-ia64/mmu_context.h
#   2004/07/26 15:16:23-07:00 [email protected] +2 -0
#   kexec
# 
# arch/ia64/kernel/machine_kexec.c
#   2004/07/26 15:16:23-07:00 [email protected] +0 -0
#   BitKeeper file /home/jbarnes/working/linux-2.5-kexec/arch/ia64/kernel/machine_kexec.c
# 
# arch/ia64/kernel/entry.S
#   2004/07/26 15:16:23-07:00 [email protected] +1 -1
#   kexec
# 
# arch/ia64/kernel/efi.c
#   2004/07/26 15:16:23-07:00 [email protected] +6 -0
#   kexec
# 
# arch/ia64/kernel/Makefile
#   2004/07/26 15:16:23-07:00 [email protected] +1 -0
#   kexec
# 
# arch/ia64/Kconfig
#   2004/07/26 15:16:23-07:00 [email protected] +17 -0
#   kexec
# 
diff -Nru a/arch/ia64/Kconfig b/arch/ia64/Kconfig
--- a/arch/ia64/Kconfig	2004-07-26 15:21:02 -07:00
+++ b/arch/ia64/Kconfig	2004-07-26 15:21:02 -07:00
@@ -251,6 +251,23 @@
           Say Y here if you are building a kernel for a desktop, embedded
           or real-time system.  Say N if you are unsure.

+config KEXEC
+ bool "kexec system call (EXPERIMENTAL)"
+ depends on EXPERIMENTAL
+ help

+	  kexec is a system call that implements the ability to  shutdown your
+	  current kernel, and to start another kernel.  It is like a reboot
+	  but it is indepedent of the system firmware.   And like a reboot
+	  you can start any kernel with it not just Linux.
+
+	  The name comes from the similiarity to the exec system call.
+
+	  It is on an going process to be certain the hardware in a machine
+	  is properly shutdown, so do not be surprised if this code does not
+	  initially work for you.  It may help to enable device hotplugging
+	  support.  As of this writing the exact hardware interface is
+	  strongly in flux, so no good recommendation can be made.

+
config HAVE_DEC_LOCK
bool
depends on (SMP || PREEMPT)

diff -Nru a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
@@ -17,6 +17,7 @@
 obj-$(CONFIG_SMP)		+= smp.o smpboot.o
 obj-$(CONFIG_PERFMON)		+= perfmon_default_smpl.o
 obj-$(CONFIG_IA64_CYCLONE)	+= cyclone.o
+obj-$(CONFIG_KEXEC)		+= machine_kexec.o relocate_kernel.o
 
 # The gate DSO image is built using a special linker script.
 targets += gate.so gate-syms.o

diff -Nru a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c
@@ -198,6 +198,7 @@
 
 #define id(arg)	arg
 
+#if 0
 STUB_GET_TIME(virt, id)
 STUB_SET_TIME(virt, id)
 STUB_GET_WAKEUP_TIME(virt, id)

@@ -207,6 +208,7 @@
 STUB_SET_VARIABLE(virt, id)
 STUB_GET_NEXT_HIGH_MONO_COUNT(virt, id)
 STUB_RESET_SYSTEM(virt, id)
+#endif
 
 void
 efi_gettimeofday (struct timespec *ts)

@@ -596,9 +598,12 @@
 #endif
 
 	efi_map_pal_code();
+#if 0
 	efi_enter_virtual_mode();
+#endif
 }
 
+#if 0
 void
 efi_enter_virtual_mode (void)
 {

@@ -670,6 +675,7 @@
 	efi.get_next_high_mono_count = virt_get_next_high_mono_count;
 	efi.reset_system = virt_reset_system;
 }
+#endif
 
 /*
  * Walk the EFI memory map looking for the I/O port range.  There can only be one entry of

diff -Nru a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
@@ -1525,7 +1525,7 @@
 	data8 sys_mq_timedreceive		// 1265
 	data8 sys_mq_notify
 	data8 sys_mq_getsetattr
-	data8 sys_ni_syscall			// reserved for kexec_load
+	data8 sys_kexec_load
 	data8 sys_ni_syscall
 	data8 sys_ni_syscall			// 1270
 	data8 sys_ni_syscall

diff -Nru a/include/asm-ia64/kexec.h b/include/asm-ia64/kexec.h
@@ -0,0 +1,15 @@
+#ifndef _ASM_IA64_KEXEC_H
+#define _ASM_IA64_KEXEC_H
+
+
+/* Maximum physical address we can use pages from */
+#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL)
+/* Maximum address we can reach in physical address mode */
+#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL)
+
+/* Zone to allocate memory from */
+#define GFP_KEXEC GFP_KERNEL
+
+#define KEXEC_REBOOT_CODE_SIZE (8192 + 8192 + 4096)
+
+#endif /* _ASM_IA64_KEXEC_H */

diff -Nru a/include/asm-ia64/mmu_context.h b/include/asm-ia64/mmu_context.h
@@ -203,5 +203,7 @@
 
 #define switch_mm(prev_mm,next_mm,next_task)	activate_mm(prev_mm, next_mm)
 
+extern void use_mm(struct mm_struct *mm);
+
 # endif /* ! __ASSEMBLY__ */
 #endif /* _ASM_IA64_MMU_CONTEXT_H */

After looking at it a little more, I suppose device_shutdown() should theoretically deal with this.

Also, it would be nice if there were a Documentation/kexec.txt or something in the full patch that describes all the pieces and what the arch dependent functions are responsible for. Randy, do you have anything like that written up somewhere that you could include in the next spin of the patch?

Thanks,
Jesse

Jesse> I'm also worried about a few things in this patch. Is Jesse> relocate_kernel.S really necessary in 2.6? Can we copy the Jesse> kernel to a contiguous 64MB aligned area, drop into phys mode Jesse> and just jump to it? Also, what about EFI boot services and Jesse> PROM tables that the kernel frees part way through boot? Jesse> Should we copy those into a safe place for the new image at Jesse> boot time? Or just leave them there if CONFIG_KEXEC is Jesse> enabled?

I suspect option one is really feasible. The first one probably just doesn't work and the second one could waste relatively large amounts of memory. Perhaps the third option of not using EFI boot-time services at all during kexec should be considered.

	--david

Oops, the first sentence got garabled: it was meant to say:

David> I suspect [neither] option is really feasible.

	--david

So we just need to avoid EFI boot time services in the kexec path, ok. How about the kernel relocation question?

Thanks,
Jesse

Jesse> So we just need to avoid EFI boot time services in the kexec Jesse> path, ok. How about the kernel relocation question?

No idea. I haven't looked at the ia64 kexec patch or relocate_kernel.S. I don't see any reason why your suggestion shouldn't work, but with these kinds of things, the devil is always in the detail.

	--david

You should be safe with that but you should consider flushing TLB entries of the old kernel before the newly started kernel inserts its own translations.

Christian

Right, probably a good idea.

Jesse

and Christian replied:

Don't just "consider" it ... this is essential or you will take MCA when you try to install overlapping TLB entries as the new kernel boots. Luckily all the code you need to do this already exists in the MCA-TLB recovery code in mca_asm.S (so long as you think that you can trust the 'ia64_mca_tlb_list' has not been corrupted).

-Tony

On Mon, 26 Jul 2004 15:36:05 -0700 Jesse Barnes wrote:

| On Monday, July 26, 2004 3:24 pm, Jesse Barnes wrote:
| >   o userspace tools need ia64 support
| >   o need to deal with in-flight DMA (see FIXME in machine_kexec)
| 
| After looking at it a little more, I suppose device_shutdown() should 
| theoretically deal with this.
| 
| Also, it would be nice if there were a Documentation/kexec.txt or something in 
| the full patch that describes all the pieces and what the arch dependent 
| functions are responsible for.  Randy, do you have anything like that written 
| up somewhere that you could include in the next spin of the patch?

Nope, sorry, I don't have anything like that.

Eric, do you have anything like Jesse asked about (arch-dependent requirements)?

--
~Randy

Correct. But all they need are the ia64 bits of the ELF loader, plus ia64 specific goo. The generic part of the ELF loader is already written.

Sort of fundamentally they are arch dependent.

I believe that DMA FIXME is a red hearing. Initially that patch was targeted for a kernel without device_shutdown(), so I was likely considering the old trick of running through all of the PCI devices and disabling their bus master bit.

In general there are two arch specific pieces of information here.

1) What is the kernel's argument passing format, what arguments does the kernel need, and how do you derive those arguments from a running kernel.

Usually this is at least the kernels memory map. But the binary arguments a kernel accepts/requires vary widely from architecture to architecture.

(This is user space only)

2) The code itself in machine_kexec.c and relocate_kernel.S needs to place the machine in a state where virtual and physical addresses are identity mapped. And the arch specific registers are in some well defined state. Usually the least setup you can guarantee to make it work the better.

(This is the kernel side)

We should probably start capturing these pieces of information in a kexec.3 man page. Volunteers?

For ia64 in particular I believe the binary arguments are the FPSWA and EFI memory map, and the firmware entry points (PAL and SAL and EFI).

As for the physical mode transition state. I believe that is largely defined by the current set of kernel bootloaders.

Eric

I think Khalid might already have these bits done.

Yeah, I added that bit to remind me to think about it.

Right, and that should be pretty straightforward.

With the addition of some ACPI tables and such. I don't think those are freed by the kernel right now though, so it should be pretty easy to point at the originals from the newly kexec'd kernel, or make copies.

Jesse

Blindly disabling all PCI bus master bits will also kill VGA/serial console and any USB keyboard attached to the system.

I'll comment more on the "DMA is a Red Herring" when I can read more what it is about.

grant

VGA/serial console devices rarely need to do be bus masters so they should be fine.

Most of those cases don't matter as the driver should always be calling pci_set_master() on startup. Disabling all the bus master bits on ioxapics in pci space would likely cripple the system. As they are architectural hardware and rarely have pci drivers that can enable them.

In the general case it appears to be overkill, incorrect and insufficient to disable bus mastering on all PCI devices. Which is why device_shutdown() calls device specific code.

Eric

yeah - you are right. I wasn't thinking. Can anyone comment on UGA or other console devices?

Is anyone else considering using kexec() to recover from a oops/panic? What is the risk calling multiple device_shutdown() will expose another panic?

While calling a device specific cleanup is best, I worry about how much code/data gets touched in this path. I was hoping something simple like twiddling bus master bit would be sufficient. If it's not, oh well.

thanks,
grant

Yes. That is what most of the recent discussion was about. Considering this was one of the subjects brought up at the kernel summit I'm surprised a lot of people have been thinking that way.

It has been agreed that device_shutdown() will not be called in the panic path. What gets called on panic or other fatal case is going to be a streamlined code path, that is little more than a jump to the previously loaded kernel.

The kernel on the other side of the kexec gets to do this. It will run out of memory reserved for it in the kernel that panic'd since boot time.

That is not perfect protection but it simple and quite good. Especially with the addition of verifying a hash of the new kernel before it messes with the hardware. (But that code gets to live in /sbin/kexec and added as a prefix to the recovery kernel)

I don't expect that is enough to give a full recovery but it should be sufficient to take a core dump of the system or do any number of other interesting things. But before running a full kernel it is expected that the entire system will be reset, to get everything back into a sane state.

And of course all of this is largely architecture independent so that the basic code should work on any architecture.

Eric

If you have the driver in your new kernel this should happen as it initializes. So really this only applies to devices whose drivers you are in the kernel invoked by the panic.

I agree with stopping the DMA. I guess I keep thinking there are cases you can potentially recover from. What if you only have a bad address because a transient bus error?

Right. And since we know the RX won't stomp us and know no more DMA is triggered. This is why we are essentially safe.

It is not avoiding shooting it down in general. It is only not shooting it down until we get into a known good kernel that we know is working properly. It's drivers need to be ``hardened'' so the initialization code works in the perverse circumstances.

And we don't currently do device shutdown in the event of a panic in any event. All we do is call that could possibly do anything is the panic notifier.

In the normal kexec case we will shutdown all of the devices cleanly. But if you are already hosed...

In the cluster case unless you modify your minimal user space to respond to the cluster watchdog, you machine will be fenced.

So I don't see that we are really introducing any new cases into the system.

Which is quite likely a good thing as it allows fencing of DMA accesses from malfunctioning devices, or devices controlled by malfunctioning drivers.

The question is how do we recover from a malfunction.... Note the code we run does not have to be a linux kernel. That is just the primary target.

Eric

yes - sorry. I'll add linux-ia64 back again too.

Not true. The new kernel will attempt to reprogram the IOMMU and either cause the system to crash fatally or redirect DMA to random regions of memory. HP platforms will crash (as they should) if we get and IOMMU lookup failure because of previous active DMA.

(well, not really random - page zero is most likely to get clobbered)

Firmware (a) knows platform/chipset specific bits and (b) is read-only. ie it's not suspectible to corruption like code is.
I agree being able to audit and update the code is a good thing.

Ok - that's reasonable.

If no IOMMU were involved, I agree the reserved mem would work fine.

*nod*.

thanks,
grant

Interesting.. One of the things we identified is that the kernel that comes up in this scenario will need truly paranoid device initialization code, so it can get the devices it chooses to use functioning from
any state. For the IOMMU things don't look differently. The code
will need to be tweaked so that it is sufficiently paranoid.

I'm not certain how receiving an unmapped DMA request should be handled but there should be methods that are less drastic than crashing the kernel. Crashing the kernel only seems sane during driver debugging.

One suggestion and I believe that still applies is to have a delay to allow existing in-flight DMA transfers to flush themselves.

It may also make sense to reserve a small portion of the IOMMU for the recovery kernel and not use that chunk of the IOMMU for the normal kernel. That would allow valid DMA transactions the recovery kernel initiated to be recognized.

Given that firmware is quite frequently compressed in flash the read-only bit is not especially true. On given platforms especially on the high-end I can see that being the case.

;)

Anyway kexec on panic is a new thing to the world so we shall have to see how it progresses. So far things look good.

Ok. It looks like the IOMMU case needs some more looking into. But I think we are on the right track.

Would a reserved chunk of the IOMMU address space work? I know things are scarce but we could probably deal with as little as 1M.

Welcome.

Now I just need to time to pull all of the patches together...

Eric

Ok - but killing DMA would make this a NOP and prevents the offending IO card from spewing potentially corrupt data to remote targets.

It's sane *any time*. Or would you rather have the IO device scribbling garbage on your root disk?
I'd rather have the box go down with a higher chance that no corrupt data made it to media.

Maybe. But that's also non-deterministic depending on the type of IO device and how independent it is. Eg. RX rings on a NIC may only slowly fill - harmless if we don't ever handle the interrupts, look at the incoming data, or touch the IOMMU. TX Rings are more likely to be bounded to fairly short times before they are drained.

...

That's an interesting idea. I'm skepitical it's feasible though. I need to think about the trade offs here.

And I'm still really very nervous about not shooting down inflight DMA. For clusters, this is especially important (prevent on-disk shared data from getting clobbered).

Scarcity of IOMMU resource is the lesser of my worries. We no longer depend as much on IOMMU for IA64. parisc still fully depends on it as do some other less common arches.

thanks,
grant

The "trim_bottom" and "trim_top" functions currently modify the memory map in place. But this would only make a difference if you tried to kexec a kernel with a smaller granule size than the originally running kernel, and even then would only result in missing seeing some memory that you might have been able to use.

-Tony

Hmm. Your mailer did not add any references lines.

On x86 and x86-64 we can recover the memory map from /proc/iomem.

Does that work on ia64? Can that be fixed to work on ia64?

All of that information needs to get exported to user space so /sbin/kexec can pass it to the new kernel.

Eric

No, it does not work on ia64. Once I have basic code in place to get somewhat working kexec on ia64, I am considering looking into fixing /proc/iomem.

Khalid

==================================
Khalid Aziz                                Linux and Open Source Lab
(970)898-9214                                        Hewlett-Packard
[email protected]                                  Fort Collins, CO

"The Linux kernel is subject to relentless development" 
				- Alessandro Rubini