purgatory: Add basic support for IPMI command execution

This patch adds an interface to BMC via KCS I/F and a functionality to start/stop BMC watchdog timer in purgatory. Starting the watchdog timer is useful to automatically reboot the server when we fail to boot the second kernel. Stopping the watchdog timer is useful to prevent the second kernel from being stopped by the watchdog timer enabled while the first kernel is running.

If you specify --ipmi-wdt-start or --ipmi-wdt-stop option to kexec command, BMC's watchdog timer will start or stop respectively while executing purgatory. You can't specify the both options at the same time. The start operation doesn't change the parameters of the watchdog timer such as initial counter and action, you need to set those parameters in the first OS. On the other hand, the stop operation changes the parameters. You need to reset them when you want to reuse the watchdog timer.

Signed-off-by Hidehiro Kawai <[email protected]>

---
  kexec/ipmi.h                  |    9 ++
  kexec/kexec.c                 |   18 +++
  kexec/kexec.h                 |    6 +
  purgatory/Makefile            |    1
  purgatory/include/purgatory.h |    3 +
  purgatory/ipmi.c              |  232 +++++++++++++++++++++++++++++++++++++++++
  purgatory/purgatory.c         |    4 +
  7 files changed, 272 insertions(+), 1 deletion(-)

create mode 100644 kexec/ipmi.h
create mode 100644 purgatory/ipmi.c

kexec/ipmi.h new file +9 -0

new file mode 100644
index 0000000..395a2c7
@@ -0,0 +1,9 @@
+#ifndef IPMI_H
+#define IPMI_H
+
+/* Options for IPMI code excuted in purgatory */
+#define IPMI_WDT_DO_NOTHING	0
+#define IPMI_WDT_START		(1 << 0)
+#define IPMI_WDT_STOP		(1 << 1)
+
+#endif /* IPMI_H */

kexec/kexec.c +18 -0

index f0bd527..8a8f268 100644
@@ -49,6 +49,7 @@
  #include "kexec-sha256.h"
  #include "kexec-zlib.h"
  #include "kexec-lzma.h"
+#include "ipmi.h"
  #include <arch/options.h>
 
  unsigned long long mem_min = 0;

@@ -57,6 +58,7 @@ static unsigned long kexec_flags = 0;
  /* Flags for kexec file (fd) based syscall */
  static unsigned long kexec_file_flags = 0;
  int kexec_debug = 0;
+int opt_ipmi_wdt = IPMI_WDT_DO_NOTHING;
 
  void dbgprint_mem_range(const char *prefix, struct memory_range *mr, int nr_mr)
  {

@@ -643,6 +645,10 @@ static void update_purgatory(struct kexec_info *info)
  	if (!info->rhdr.e_shdr) {
  		return;
  	}
+
+	elf_rel_set_symbol(&info->rhdr, "ipmi_wdt", &opt_ipmi_wdt,
+			   sizeof(opt_ipmi_wdt));
+
  	arch_update_purgatory(info);
  	memset(region, 0, sizeof(region));
  	sha256_starts(&ctx);

@@ -1345,6 +1351,12 @@ int main(int argc, char *argv[])
  		case OPT_KEXEC_FILE_SYSCALL:
  			/* We already parsed it. Nothing to do. */
  			break;
+		case OPT_IPMI_WDT_START:
+			opt_ipmi_wdt |= IPMI_WDT_START;
+			break;
+		case OPT_IPMI_WDT_STOP:
+			opt_ipmi_wdt |= IPMI_WDT_STOP;
+			break;
  		default:
  			break;
  		}

@@ -1370,6 +1382,12 @@ int main(int argc, char *argv[])
  		    "\"--mem-max\" parameter\n");
  	}
 
+	if ((opt_ipmi_wdt & IPMI_WDT_START) &&
+	    (opt_ipmi_wdt & IPMI_WDT_STOP)) {
+		die("You can't specify both --ipmi-wdt-start and "
+		    "--ipmi-wdt-stop\n");
+	}
+
  	fileind = optind;
  	/* Reset getopt for the next pass; called in other source modules */
  	opterr = 1;

kexec/kexec.h +5 -1

index c02ac8f..4638866 100644
@@ -224,7 +224,9 @@ extern int file_types;
  #define OPT_LOAD_PRESERVE_CONTEXT 259
  #define OPT_LOAD_JUMP_BACK_HELPER 260
  #define OPT_ENTRY		261
-#define OPT_MAX			262
+#define OPT_IPMI_WDT_START	262
+#define OPT_IPMI_WDT_STOP	263
+#define OPT_MAX			264
  #define KEXEC_OPTIONS \
  	{ "help",		0, 0, OPT_HELP }, \
  	{ "version",		0, 0, OPT_VERSION }, \

@@ -244,6 +246,8 @@ extern int file_types;
  	{ "reuseinitrd",	0, 0, OPT_REUSE_INITRD }, \
  	{ "kexec-file-syscall",	0, 0, OPT_KEXEC_FILE_SYSCALL }, \
  	{ "debug",		0, 0, OPT_DEBUG }, \
+	{ "ipmi-wdt-start",	0, 0, OPT_IPMI_WDT_START }, \
+	{ "ipmi-wdt-stop",	0, 0, OPT_IPMI_WDT_STOP }, \
 
  #define KEXEC_OPT_STR "h?vdfxyluet:ps"
 

purgatory/Makefile +1 -0

index 2b5c061..ae1d2bd 100644
@@ -9,6 +9,7 @@
 
  PURGATORY = purgatory/purgatory.ro
  PURGATORY_SRCS =
+PURGATORY_SRCS += purgatory/ipmi.c
  PURGATORY_SRCS += purgatory/purgatory.c
  PURGATORY_SRCS += purgatory/printf.c
  PURGATORY_SRCS += purgatory/string.c

purgatory/include/purgatory.h +3 -0

index 788ce49..9b9ffbe 100644
@@ -1,11 +1,14 @@
  #ifndef PURGATORY_H
  #define PURGATORY_H
 
+extern int ipmi_wdt;
+
  void putchar(int ch);
  void sprintf(char *buffer, const char *fmt, ...)
  	__attribute__ ((format (printf, 2, 3)));
  void printf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
  void setup_arch(void);
  void post_verification_setup_arch(void);
+void ipmi_wdt_start_stop(void);
 
  #endif /* PURGATORY_H */

purgatory/ipmi.c new file +232 -0

new file mode 100644
index 0000000..0acb853
@@ -0,0 +1,232 @@
+#include <sys/io.h>
+#include <purgatory.h>
+#include "../kexec/ipmi.h"
+
+#define KCS_PORT_DEFAULT	0xca2
+#define KCS_PORT_DATA		(kcs_port + 0)
+#define KCS_PORT_STATUS		(kcs_port + 1)
+#define KCS_PORT_CMD		(kcs_port + 1)
+
+#define KCS_STATUS_OBF		0x1
+#define KCS_STATUS_IBF		0x2
+
+#define KCS_CMD_WRITE_START	0x61
+#define KCS_CMD_WRITE_END	0x62
+#define KCS_CMD_READ		0x68
+
+#define GET_STATUS_STATE(status) (((status) >> 6) & 0x03)
+#define KCS_IDLE_STATE	0x0
+#define KCS_READ_STATE	0x1
+#define KCS_WRITE_STATE	0x2
+#define KCS_ERROR_STATE	0x3
+
+int kcs_port = KCS_PORT_DEFAULT;
+int ipmi_wdt = IPMI_WDT_DO_NOTHING;
+
+/* IPMI command to start BMC watchdog timer */
+const unsigned char cmd_start_wdt[] = {
+	0x06 << 2,	/* App */
+	0x22,		/* Reset Watchdog Timer */
+};
+
+/* IPMI command to stop BMC watchdog timer */
+const unsigned char cmd_stop_wdt[] = {
+	0x06 << 2,	/* App */
+	0x24,		/* Set Watchdog Timer Command */
+	0x84,		/* Timer Use: don't log, and SMS/OS use */
+	0x00,		/* Timer Actions: no action */
+	0x00,		/* Pre-timeout interval: 0 */
+	0x10,		/* Timer Use Expiration flag clear: SMS/OS */
+	0xff,		/* Initial countdown value: 0xffff */
+	0xff,
+};
+
+static inline unsigned char read_status(void)
+{
+	return inb(KCS_PORT_STATUS);
+}
+
+unsigned char wait_out(void)
+{
+	unsigned char status;
+
+	do {
+		status = read_status();
+	} while ((status & KCS_STATUS_OBF) == 0);
+
+	return status;
+}
+
+unsigned char wait_in(void)
+{
+	unsigned char status;
+
+	do {
+		status = read_status();
+	} while (status & KCS_STATUS_IBF);
+
+	return status;
+}
+
+unsigned char read_data(void)
+{
+	wait_out();
+	return inb(KCS_PORT_DATA);
+}
+
+void clear_obf(void)
+{
+	if (inb(KCS_PORT_STATUS) & KCS_STATUS_OBF)
+		read_data();
+}
+
+unsigned char write_data(unsigned char byte)
+{
+	clear_obf();
+	outb(byte, KCS_PORT_DATA);
+	return wait_in();
+}
+
+unsigned char write_cmd(unsigned char byte)
+{
+	clear_obf();
+	outb(byte, KCS_PORT_CMD);
+	return wait_in();
+}
+
+/*
+ * Issue a given IPMI command via KCS I/F.
+ *
+ * Return 0 on success, otherwise -1.
+ */
+int write_ipmi_cmd(const unsigned char *cmd, int size)
+{
+	unsigned char status;
+	int i;
+
+	wait_in();
+	status = write_cmd(KCS_CMD_WRITE_START);
+	if (GET_STATUS_STATE(status) != KCS_WRITE_STATE)
+		return -1;
+
+	for (i = 0; i < size - 1; i++) {
+		status = write_data(cmd[i]);
+
+		if (GET_STATUS_STATE(status) != KCS_WRITE_STATE)
+			return -1;
+	}
+
+	/* last write */
+	status = write_cmd(KCS_CMD_WRITE_END);
+	if (GET_STATUS_STATE(status) != KCS_WRITE_STATE)
+		return -1;
+
+	write_data(cmd[i]);
+
+	return 0;
+}
+
+/*
+ * Read result bytes for the previously issued IPMI command.
+ *
+ * Return the completion code, which is 0 on success.  Otherwise, return
+ * non-zero value.
+ */
+unsigned char read_result(void)
+{
+	unsigned char state;
+	unsigned char data[4] = { 0 };
+	int count = 0;
+
+	while (1) {
+		state = GET_STATUS_STATE(wait_in());
+		if (state == KCS_READ_STATE) {
+			data[count] = read_data();
+			outb(KCS_CMD_READ, KCS_PORT_DATA);
+		} else if (state == KCS_IDLE_STATE) {
+			data[count] = read_data();
+			break;
+		} else {
+			/*
+			 * Error! Set 0xff (unspecified error) as the
+			 * completion code.
+			 */
+			data[2] = 0xff;
+			break;
+		}
+
+		/*
+		 * We are interested only in the completion code in the
+		 * 3rd byte. Skip the following bytes.
+		 */
+		if (count < 3)
+			count++;
+	}
+
+	return data[2]; /* Return completion code */
+}
+
+/*
+ * Issue one IPMI command and check the result.
+ *
+ * Return 0 on success, otherwise -1.
+ */
+int issue_ipmi_cmd(const unsigned char *cmd, int size)
+{
+	int i, ret;
+	unsigned char comp_code;
+
+	/*
+	 * Retry 3 times at most on error.
+	 *
+	 * write_ipmi_cmd() issues WRITE_START KCS command at the beginning,
+	 * which aborts the ongoing IPMI command and starts new one.  So
+	 * basically, write_ipmi_cmd() will succeed anytime.
+	 *
+	 * This command abort can fail on some BMC depending on its KCS I/F
+	 * state, and  we get ERROR_STATE from the status register in that
+	 * case.  However, there is no problem.  We can recover it by
+	 * simply retrying from writing WRITE_START KCS command.
+	 */
+	for (i = 0; i < 3; i++) {
+		ret = write_ipmi_cmd(cmd, size);
+		if (ret < 0)
+			continue;
+
+		comp_code = read_result();
+		if (comp_code == 0)
+			break; /* succeeded */
+	}
+
+	return (i < 3) ? 0 : -1;
+}
+
+int do_start_wdt(void)
+{
+	int ret;
+
+	printf("IPMI: starting watchdog timer...");
+	ret = issue_ipmi_cmd(cmd_start_wdt, sizeof(cmd_start_wdt));
+	printf("done\n");
+
+	return ret;
+}
+
+int do_stop_wdt(void)
+{
+	int ret;
+
+	printf("IPMI: stopping watchdog timer...");
+	ret = issue_ipmi_cmd(cmd_stop_wdt, sizeof(cmd_stop_wdt));
+	printf("done\n");
+
+	return ret;
+}
+
+void ipmi_wdt_start_stop(void)
+{
+	if (ipmi_wdt & IPMI_WDT_START)
+		do_start_wdt();
+	else if (ipmi_wdt & IPMI_WDT_STOP)
+		do_stop_wdt();
+}

purgatory/purgatory.c +0 -0

index 3bbcc09..f60879d 100644

A general note here. It does not appear that you implement the error recovery states in your state machine. If the system fails in the middle of doing an IPMI operation, it is likely to fail.

If you do this you will need to detect and abort any running operation. Implementing the full state machine is probably the best approach, it should handle this, though it is rather complex.

-corey

If the second kernel for crash dumping hangs up while booting, no information related to the first kernel will be saved. This makes crash cause analysis difficult. So, some enterprise users want to save minimal information befor booting the second kernel.

One of the approaches is to use panic notifier call or pstore feature. For example, a panic notifier callback registered by IPMI driver saves the panic message to BMC's SEL before booting the second kernel. Similarly, pstore saves kernel logs to a non-volatile memory on the server. However, since these functionalities run with crashed kernel, they may fail to complete their work and boot the second kernel.

So, another approach; saving minimal information to BMC's SEL in the purgatory. Since the purgatory code doesn't rely on the crashed kernel, we can run it safely after verifying the hash of the code.

This patch set is the first step to the final goal; it provides a basic support for IPMI command execution in purgatory. IPMI specification defines multiple interfaces to BMC, and this patch set uses one of them, KCS I/F, which talks with BMC via I/O port like keyboard controllers. As a use case for that, options to start/stop BMC's watchdog timer before booting the second kernel are also provided. These options are useful for the cases where:

- you want to automatically reboot the server when the second kernel hangs up while booting
- you want to prevent the second kernel from being stopped by the watchdog timer enabled while the first kernel is running

If the BMC doesn't work well, the IPMI command execution can take indefinite time and fail to boot the second kernel. To avoid this, timeout logic based on RTC polling is also implemented.

NOTE: This is an RFC version, so some parts are incomplete; these codes are unconditionally built into the kexec binary, and I/O ports for KCS I/F and timeout (5 seconds) are hard-coded, and etc.

Future plan:
Add an option to save the panic message and instruction pointers to BMC's SEL in purgatory. To realize this, we first need to pass the panic message to the purgatory. Instruction pointers are already passed to the second kernel through ELF notes, so just read them.

---

Hidehiro Kawai (4):
purgatory/ipmi: Support BMC watchdog timer start/stop in purgatory
purgatory: Introduce timeout API
purgatory/x86: Support CMOS RTC
purgatory/ipmi: Add timeout logic to IPMI command processing

  kexec/ipmi.h                   |    9 +
  kexec/kexec.c                  |   18 ++
  kexec/kexec.h                  |    6 +
  purgatory/Makefile             |    5 +
  purgatory/arch/i386/Makefile   |    1
  purgatory/arch/i386/rtc_cmos.c |  104 ++++++++++++++
  purgatory/arch/x86_64/Makefile |    1
  purgatory/include/purgatory.h  |    3
  purgatory/include/time.h       |   33 +++++
  purgatory/ipmi.c               |  293 ++++++++++++++++++++++++++++++++++++++++
  purgatory/purgatory.c          |    4 +
  purgatory/time.c               |   58 ++++++++
  12 files changed, 533 insertions(+), 2 deletions(-)

create mode 100644 kexec/ipmi.h
create mode 100644 purgatory/arch/i386/rtc_cmos.c create mode 100644 purgatory/include/time.h create mode 100644 purgatory/ipmi.c create mode 100644 purgatory/time.c

I understand what you are trying to accomplish here, but I'm not sure of the wisdom of this approach. I'll give some more information and the kexec maintainers can decide, I suppose.

The KCS interface given here probably covers ~70% of the systems out there right now.

Other systems have:
* KCS interfaces at a different port or in a different place like
memory, PCI,
and with different register sizes and spacing.

* Other standard interfaces. SMIC (probably not relevant), BT
(faster, it does
block transfers) and SSIF (which is IPMI over I2C).
* Those other standard interfaces can be in different places, just
like KCS.
Hundreds of I2C interfaces exist.
* Non-standard interfaces. Power systems have their own IPMI interfaces, for instance. Some systems have IPMI over serial ports, though
hopefully
that has pretty much gone away.

I'd guess that over half of the IPMI SI driver is discovering and handling all the
various interface types, locations from all the sources it can come from.

As time goes on that 70% number is decreasing in favour of other faster and more convenient interfaces. I expect that SSIF will become much more popular over time because it has block transfer capability and all the hardware
is already there on systems.

This is no different, of course, than any other common hardware interface out
there. USB, ATA, etc. But it makes it hard to cover all the possibilities in something
like purgatory.

I know how valuable this information can be. It has saved my butt on occasions,
which is why I go through the inconvenience of handling it in the IPMI driver.
But it seems to me that the failure rate of doing this in the crashing kernel should
be pretty low. Not zero of course. But I have no idea what it is.