Skip to contentSkip to conversation

ath10k: save firmware stacks upon firmware crash

PATCH In review [email protected] 2 participants 3 messages Sep 5, 2014 – Sep 23, 2014 Qualcomm Atheros ATH10K wireless driver

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 ath10k: save firmware stacks upon firmware crash

From: Ben Greear <[email protected]>

Should help debug firmware crashes, and give users a way to provide some useful debug reports to firmware developers.

---

This is a merge/rework of previously posted stack-dump patches. This is on top of the debug-log dumping patch and the firmware crash-by-assert patch. It has been tested.

 drivers/net/wireless/ath/ath10k/core.h  |  4 +++
 drivers/net/wireless/ath/ath10k/debug.c | 29 +++++++++++++++++-
 drivers/net/wireless/ath/ath10k/hw.h    |  1 +
 drivers/net/wireless/ath/ath10k/pci.c   | 54 ++++++++++++++++++++++++++++++++-
 4 files changed, 86 insertions(+), 2 deletions(-)
drivers/net/wireless/ath/ath10k/core.h +4 -0
index fc39995..0ca52e0 100644
@@ -303,6 +303,10 @@ struct ath10k_fw_crash_data {
 	uuid_le uuid;
 	struct timespec timestamp;
 	__le32 registers[REG_DUMP_COUNT_QCA988X];
+	__le32 stack_buf[ATH10K_FW_STACK_SIZE / sizeof(__le32)];
+	__le32 exc_stack_buf[ATH10K_FW_STACK_SIZE / sizeof(__le32)];
+	__le32 stack_addr;
+	__le32 exc_stack_addr;
 };
 
 struct ath10k_debug {
drivers/net/wireless/ath/ath10k/debug.c +28 -1
index 229573e..5c11966 100644
@@ -33,10 +33,15 @@
  * enum ath10k_fw_crash_dump_type - types of data in the dump file
  * @ATH10K_FW_CRASH_DUMP_REGDUMP: Register crash dump in binary format
  * @ATH10K_FW_ERROR_DUMP_DBGLOG:  Recent firmware debug log entries
+ * @ATH10K_FW_CRASH_DUMP_STACK:   Stack memory contents.
+ * @ATH10K_FW_CRASH_DUMP_EXC_STACK:   Exception stack memory contents.
  */
 enum ath10k_fw_crash_dump_type {
 	ATH10K_FW_CRASH_DUMP_REGISTERS = 0,
 	ATH10K_FW_CRASH_DUMP_DBGLOG = 1,
+	ATH10K_FW_CRASH_DUMP_STACK = 2,
+	ATH10K_FW_CRASH_DUMP_EXC_STACK = 3,
+
 	ATH10K_FW_CRASH_DUMP_MAX,
 };
 
@@ -100,8 +105,11 @@ struct ath10k_dump_file_data {
 	/* VERMAGIC_STRING */
 	char kernel_ver[64];
 
+	__le32 stack_addr;
+	__le32 exc_stack_addr;
+
 	/* room for growth w/out changing binary format */
-	u8 unused[128];
+	u8 unused[120];
 
 	/* struct ath10k_tlv_dump_data + more */
 	u8 data[0];
@@ -774,6 +782,8 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar)
 	len = hdr_len;
 	len += sizeof(*dump_tlv) + sizeof(crash_data->registers);
 	len += sizeof(*dump_tlv) + sizeof(ar->debug.dbglog_entry_data);
+	len += sizeof(*dump_tlv) + sizeof(crash_data->stack_buf);
+	len += sizeof(*dump_tlv) + sizeof(crash_data->exc_stack_buf);
 
 	sofar += hdr_len;
 
@@ -813,6 +823,8 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar)
 	dump_data->ht_cap_info = cpu_to_le32(ar->ht_cap_info);
 	dump_data->vht_cap_info = cpu_to_le32(ar->vht_cap_info);
 	dump_data->num_rf_chains = cpu_to_le32(ar->num_rf_chains);
+	dump_data->stack_addr = cpu_to_le32(crash_data->stack_addr);
+	dump_data->exc_stack_addr = cpu_to_le32(crash_data->exc_stack_addr);
 
 	strlcpy(dump_data->fw_ver, ar->hw->wiphy->fw_version,
 		sizeof(dump_data->fw_ver));
@@ -845,7 +857,22 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar)
 		cpu_to_le32(ar->debug.dbglog_entry_data.head_idx);
 	dbglog_storage->tail_idx =
 		cpu_to_le32(ar->debug.dbglog_entry_data.tail_idx);
+	sofar += sizeof(*dump_tlv) + tmp;
 
+	/* Gather firmware stack dump */
+	tmp = sizeof(crash_data->stack_buf);
+	dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+	dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_STACK);
+	dump_tlv->tlv_len = cpu_to_le32(tmp);
+	memcpy(dump_tlv->tlv_data, crash_data->stack_buf, tmp);
+	sofar += sizeof(*dump_tlv) + tmp;
+
+	/* Gather firmware exception stack dump */
+	tmp = sizeof(crash_data->exc_stack_buf);
+	dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+	dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_EXC_STACK);
+	dump_tlv->tlv_len = cpu_to_le32(tmp);
+	memcpy(dump_tlv->tlv_data, crash_data->exc_stack_buf, tmp);
 	sofar += sizeof(*dump_tlv) + tmp;
 
 	ar->debug.fw_crash_data->crashed_since_read = false;
drivers/net/wireless/ath/ath10k/hw.h +1 -0
index 28fedba..c7ec412 100644
@@ -40,6 +40,7 @@
 #define ATH10K_FIRMWARE_MAGIC               "QCA-ATH10K"
 
 #define REG_DUMP_COUNT_QCA988X 60
+#define ATH10K_FW_STACK_SIZE 4096
 
 struct ath10k_fw_ie {
 	__le32 id;
drivers/net/wireless/ath/ath10k/pci.c +53 -1

78 lines not on this page · raw message

Signature · 1 line
1.7.11.7
#1 lore raw

PATCH 2/2 ath10k: save firmware RAM and ROM BSS sections on crash

From: Ben Greear <[email protected]>

This can be used to get a useful back trace out of a firmware crash that involves an interrupt handler. For instance, a null-pointer-exception would be this kind of trace. A user-space tool can read the debugfs file and decode things as wished.

This requires a packaged firmware with a new IE to describe the BSS section starts and length.

---

This is a re-work and consolidation of previously posted patches. It uses a single IE with 4 members: My CT firmware already uses this, and no other firmware has any support at all for this IE, so I would like to be compatible. I use a well-defined struct now, so hopefully that is more acceptable than my first patches.

Using one IE decreases code and makes code easier to maintain.

Fix several bugs in earlier versions (copy-paste error in IE handling, inversion of ROM v/s RAM bss sizes)

This has been tested.

 drivers/net/wireless/ath/ath10k/core.c  | 41 ++++++++++++++++++++++++++
 drivers/net/wireless/ath/ath10k/core.h  | 16 ++++++++++
 drivers/net/wireless/ath/ath10k/debug.c | 34 ++++++++++++++++++++-
 drivers/net/wireless/ath/ath10k/hw.h    |  1 +
 drivers/net/wireless/ath/ath10k/pci.c   | 52 +++++++++++++++++++++++++++++++++
 5 files changed, 143 insertions(+), 1 deletion(-)
drivers/net/wireless/ath/ath10k/core.c +41 -0
index 651a6da..9828a79 100644
@@ -360,6 +360,13 @@ err:
 	return ret;
 }
 
+struct ath10k_bss_rom_ie {
+	__le32 ram_addr;
+	__le32 ram_len;
+	__le32 rom_addr;
+	__le32 rom_len;
+} __packed;
+
 static int ath10k_core_fetch_firmware_api_n(struct ath10k *ar, const char *name)
 {
 	size_t magic_len, len, ie_len;
@@ -367,6 +374,7 @@ static int ath10k_core_fetch_firmware_api_n(struct ath10k *ar, const char *name)
 	struct ath10k_fw_ie *hdr;
 	const u8 *data;
 	__le32 *timestamp;
+	struct ath10k_bss_rom_ie *bss;
 
 	/* first fetch the firmware file (firmware-*.bin) */
 	ar->firmware = ath10k_fetch_fw_file(ar, ar->hw_params.fw.dir, name);
@@ -481,6 +489,39 @@ static int ath10k_core_fetch_firmware_api_n(struct ath10k *ar, const char *name)
 			ar->otp_len = ie_len;
 
 			break;
+		case ATH10K_FW_IE_BSS_INFO:
+			if (ie_len < sizeof(*bss)) {
+				ath10k_warn(ar, "invalid ie len for bss-info (%zd)\n",
+					    ie_len);
+				break;
+			}
+			bss = (struct ath10k_bss_rom_ie *)(data);
+
+			ar->fw.ram_bss_addr = le32_to_cpu(bss->ram_addr);
+			ar->fw.ram_bss_len = le32_to_cpu(bss->ram_len);
+			ath10k_dbg(ar, ATH10K_DBG_BOOT,
+				   "found RAM BSS addr 0x%x length %d\n",
+				   ar->fw.ram_bss_addr, ar->fw.ram_bss_len);
+
+			if (ar->fw.ram_bss_len > ATH10K_RAM_BSS_BUF_LEN) {
+				ath10k_warn(ar, "too long firmware RAM BSS length: %d\n",
+					    ar->fw.ram_bss_len);
+				ar->fw.ram_bss_len = 0;
+			}
+
+			ar->fw.rom_bss_addr = le32_to_cpu(bss->rom_addr);
+			ar->fw.rom_bss_len = le32_to_cpu(bss->rom_len);
+			ath10k_dbg(ar, ATH10K_DBG_BOOT,
+				   "found ROM BSS addr 0x%x length %d\n",
+				   ar->fw.rom_bss_addr, ar->fw.rom_bss_len);
+
+			if (ar->fw.rom_bss_len > ATH10K_ROM_BSS_BUF_LEN) {
+				ath10k_warn(ar, "too long firmware ROM BSS length: %d\n",
+					    ar->fw.rom_bss_len);
+				ar->fw.rom_bss_len = 0;
+			}
+
+			break;
 		default:
 			ath10k_warn(ar, "Unknown FW IE: %u\n",
 				    le32_to_cpu(hdr->id));
drivers/net/wireless/ath/ath10k/core.h +16 -0
index 0ca52e0..b16c0c5 100644
@@ -296,6 +296,10 @@ struct ath10k_dbglog_entry_storage {
 #define DBGLOG_NUM_ARGS_MASK             0xFC000000 /* Bit 26-31 */
 #define DBGLOG_NUM_ARGS_MAX              5 /* firmware tool chain limit */
 
+/* estimated values, hopefully these are enough */
+#define ATH10K_ROM_BSS_BUF_LEN 30000
+#define ATH10K_RAM_BSS_BUF_LEN 10000
+
 /* used for crash-dump storage, protected by data-lock */
 struct ath10k_fw_crash_data {
 	bool crashed_since_read;
@@ -307,6 +311,8 @@ struct ath10k_fw_crash_data {
 	__le32 exc_stack_buf[ATH10K_FW_STACK_SIZE / sizeof(__le32)];
 	__le32 stack_addr;
 	__le32 exc_stack_addr;
+	__le32 rom_bss_buf[ATH10K_ROM_BSS_BUF_LEN / sizeof(__le32)];
+	__le32 ram_bss_buf[ATH10K_RAM_BSS_BUF_LEN / sizeof(__le32)];
 };
 
 struct ath10k_debug {
@@ -455,6 +461,16 @@ struct ath10k {
 		} fw;
 	} hw_params;
 
+	/* These are written to only during first firmware load from user
+	 * space so no need for any locking.
+	 */
+	struct {
+		u32 ram_bss_addr;
+		u32 ram_bss_len;
+		u32 rom_bss_addr;
+		u32 rom_bss_len;
+	} fw;
+
 	const struct firmware *board;
 	const void *board_data;
 	size_t board_len;
drivers/net/wireless/ath/ath10k/debug.c +33 -1
index 5c11966..4a5e2e1 100644
@@ -35,12 +35,16 @@
  * @ATH10K_FW_ERROR_DUMP_DBGLOG:  Recent firmware debug log entries
  * @ATH10K_FW_CRASH_DUMP_STACK:   Stack memory contents.
  * @ATH10K_FW_CRASH_DUMP_EXC_STACK:   Exception stack memory contents.
+ * @ATH10K_FW_CRASH_DUMP_RAM_BSS:  BSS area for RAM code
+ * @ATH10K_FW_CRASH_DUMP_ROM_BSS:  BSS area for ROM code
  */
 enum ath10k_fw_crash_dump_type {
 	ATH10K_FW_CRASH_DUMP_REGISTERS = 0,
 	ATH10K_FW_CRASH_DUMP_DBGLOG = 1,
 	ATH10K_FW_CRASH_DUMP_STACK = 2,
 	ATH10K_FW_CRASH_DUMP_EXC_STACK = 3,
+	ATH10K_FW_CRASH_DUMP_RAM_BSS = 4,
+	ATH10K_FW_CRASH_DUMP_ROM_BSS = 5,
 
 	ATH10K_FW_CRASH_DUMP_MAX,
 };
@@ -107,9 +111,11 @@ struct ath10k_dump_file_data {
 
 	__le32 stack_addr;
 	__le32 exc_stack_addr;
+	__le32 rom_bss_addr;
+	__le32 ram_bss_addr;
 
 	/* room for growth w/out changing binary format */
-	u8 unused[120];
+	u8 unused[112];
 
 	/* struct ath10k_tlv_dump_data + more */
 	u8 data[0];
@@ -785,6 +791,12 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar)
 	len += sizeof(*dump_tlv) + sizeof(crash_data->stack_buf);
 	len += sizeof(*dump_tlv) + sizeof(crash_data->exc_stack_buf);
 
+	if (ar->fw.ram_bss_addr && ar->fw.ram_bss_len)
+		len += sizeof(*dump_tlv) + ar->fw.ram_bss_len;
+
+	if (ar->fw.rom_bss_addr && ar->fw.rom_bss_len)
+		len += sizeof(*dump_tlv) + ar->fw.rom_bss_len;
+
 	sofar += hdr_len;
 
 	/* This is going to get big when we start dumping FW RAM and such,
@@ -825,6 +837,8 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar)
 	dump_data->num_rf_chains = cpu_to_le32(ar->num_rf_chains);
 	dump_data->stack_addr = cpu_to_le32(crash_data->stack_addr);
 	dump_data->exc_stack_addr = cpu_to_le32(crash_data->exc_stack_addr);
+	dump_data->rom_bss_addr = cpu_to_le32(ar->fw.rom_bss_addr);
+	dump_data->ram_bss_addr = cpu_to_le32(ar->fw.ram_bss_addr);
 
 	strlcpy(dump_data->fw_ver, ar->hw->wiphy->fw_version,
 		sizeof(dump_data->fw_ver));
@@ -875,6 +889,24 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar)
 	memcpy(dump_tlv->tlv_data, crash_data->exc_stack_buf, tmp);
 	sofar += sizeof(*dump_tlv) + tmp;
 
+	if (ar->fw.ram_bss_addr && ar->fw.ram_bss_len) {
+		tmp = ar->fw.ram_bss_len;
+		dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+		dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_RAM_BSS);
+		dump_tlv->tlv_len = cpu_to_le32(tmp);
+		memcpy(dump_tlv->tlv_data, crash_data->ram_bss_buf, tmp);
+		sofar += sizeof(*dump_tlv) + tmp;
+	}
+
+	if (ar->fw.rom_bss_addr && ar->fw.rom_bss_len) {
+		tmp = ar->fw.rom_bss_len;
+		dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+		dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_ROM_BSS);
+		dump_tlv->tlv_len = cpu_to_le32(tmp);
+		memcpy(dump_tlv->tlv_data, crash_data->rom_bss_buf, tmp);
+		sofar += sizeof(*dump_tlv) + tmp;
+	}
+
 	ar->debug.fw_crash_data->crashed_since_read = false;
 
 	WARN_ON(sofar != len);
drivers/net/wireless/ath/ath10k/hw.h +1 -0
index c7ec412..eb1a14b 100644
@@ -54,6 +54,7 @@ enum ath10k_fw_ie_type {
 	ATH10K_FW_IE_FEATURES = 2,
 	ATH10K_FW_IE_FW_IMAGE = 3,
 	ATH10K_FW_IE_OTP_IMAGE = 4,
+	ATH10K_FW_IE_BSS_INFO = 5,
 };
 
 /* Known pecularities:
drivers/net/wireless/ath/ath10k/pci.c +52 -0

64 lines not on this page · raw message

Signature · 1 line
1.7.11.7
#2 lore raw
In reply to [email protected]
Quoted 7 lines · [email protected] writes:

From: Ben Greear <[email protected]>

Should help debug firmware crashes, and give users a way to provide some useful debug reports to firmware developers.

Like we talked on IRC, I think it's better to take a dump of the whole DRAM and find all the info during postprocessing in user space. I just consider it too much work to investigate firmware internals from ath10k as the internals can change between firmware versions. In user space that's a lot easier to manage.

Of course downside is the increased memory usage, but if that becomes a problem there are ways to solve that. For example, adding a module parameter with bitmask for individually enabling enum ath10k_fw_crash_dump_type is first which comes to my mind.

I have patches ready for the DRAM thing, but I have some issues which I found during testing. I'll post the patches soon as I have solved the issues, but don't know how long that will take.

Signature · 1 line
Kalle Valo