Skip to contentSkip to conversation

ASoC: Intel: catpt: Error handling and debug improvements

PATCH Superseded Cezary Rojewski 2 participants 6 messages May 27

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Cezary Rojewski

author maintainer
#0 lore raw

cover ASoC: Intel: catpt: Error handling and debug improvements

Outcome of a long debug to solve one, long-standing bug ocurring very rarely on Haswell/Broadwell machines during the boot procedure of the AudioDSP firmware. Clever/unfortunate user can increase the reproduction rare to 100%.

The bug: an exception occurring early during FW boot (firmware side, not the software one) leaves the firmware hanging and the existing software code is incappable of recognizing such problem. The only solution a user currently has is: rmmod and then modprobe the driver.

Recently, together with Krzysztof from the firmware team decided to take it up and clear the dashboard.

The exception handling takes just a few lines of code (all part of the first patch), everything else that this patchset is composed of improves the debugability and logging. If anything similar pops up, the developers can see what's going on.

Cezary Rojewski (3):
ASoC: Intel: catpt: Complete coredump handling
ASoC: Intel: catpt: Add pretty-trace for large IPC payloads
ASoC: Intel: catpt: Print error code if board-registration fails

 sound/soc/intel/catpt/Makefile    |  3 ++-
 sound/soc/intel/catpt/device.c    |  5 +----
 sound/soc/intel/catpt/dsp.c       |  5 +++++
 sound/soc/intel/catpt/ipc.c       |  3 +++
 sound/soc/intel/catpt/loader.c    |  3 +++
 sound/soc/intel/catpt/registers.h | 12 ++++++++++++
 sound/soc/intel/catpt/trace.c     | 27 +++++++++++++++++++++++++++
 sound/soc/intel/catpt/trace.h     | 22 +++++++++++++++-------
 8 files changed, 68 insertions(+), 12 deletions(-)
 create mode 100644 sound/soc/intel/catpt/trace.c
Signature · 1 line
2.34.1
author maintainer
#1 lore raw

PATCH 1/3 ASoC: Intel: catpt: Complete coredump handling

An exception may occur during the firmware booting procedure. In such case the firmware sends COREDYMP_REQUESTS and expects the driver to dump relevant information and finish with the COREDUMP_RELEASE write.

To distinguish such situation from generic timeout, always signal fw_ready completion when a coredump request is received and translate it to -EREMOTEIO in catpt_boot_firmware().

The "FW READY" print makes the success clearly visible even when the event-traces are not enabled.

---
 sound/soc/intel/catpt/dsp.c       |  5 +++++
 sound/soc/intel/catpt/ipc.c       |  3 +++
 sound/soc/intel/catpt/loader.c    |  3 +++
 sound/soc/intel/catpt/registers.h | 12 ++++++++++++
 4 files changed, 23 insertions(+)
sound/soc/intel/catpt/dsp.c +5 -0
index 677f348909c8..e7c577510865 100644
@@ -536,6 +536,11 @@ int catpt_coredump(struct catpt_dev *cdev)
 		pos += CATPT_DMA_REGS_SIZE;
 	}
 
+	if (catpt_readl_dram(cdev, COREDUMP) == CATPT_COREDUMP_REQUEST) {
+		dev_dbg(cdev->dev, "releasing firmware from the coredump state\n");
+		catpt_writel_dram(cdev, COREDUMP, CATPT_COREDUMP_RELEASE);
+	}
+
 	dev_coredumpv(cdev->dev, dump, dump_size, GFP_KERNEL);
 
 	return 0;
sound/soc/intel/catpt/ipc.c +3 -0
index 2e3b7a5cbb9b..21ccec55923e 100644
@@ -210,6 +210,7 @@ static void catpt_dsp_process_response(struct catpt_dev *cdev, u32 header)
 		memcpy_fromio(&config, cdev->lpe_ba + off, sizeof(config));
 		trace_catpt_ipc_payload((u8 *)&config, sizeof(config));
 
+		dev_dbg(cdev->dev, "FW READY 0x%08x\n", header);
 		catpt_ipc_arm(ipc, &config);
 		complete(&cdev->fw_ready);
 		return;
@@ -220,6 +221,8 @@ static void catpt_dsp_process_response(struct catpt_dev *cdev, u32 header)
 		dev_err(cdev->dev, "ADSP device coredump received\n");
 		ipc->ready = false;
 		catpt_coredump(cdev);
+
+		complete(&cdev->fw_ready);
 		/* TODO: attempt recovery */
 		break;
 
sound/soc/intel/catpt/loader.c +3 -0
index 432cb1f0ab4e..75457187b614 100644
@@ -624,6 +624,9 @@ int catpt_boot_firmware(struct catpt_dev *cdev, bool restore)
 	if (!ret) {
 		dev_err(cdev->dev, "firmware ready timeout\n");
 		return -ETIMEDOUT;
+	/* Wake up does not mean FW is ready, an exception could occur. */
+	} else if (!cdev->ipc.ready) {
+		return -EREMOTEIO;
 	}
 
 	/* update sram pg & clock once done booting */
sound/soc/intel/catpt/registers.h +12 -0
index 6c1ad28c6d69..64bd534a76ff 100644
@@ -124,6 +124,11 @@
 #define CATPT_SSCR2_DEFAULT		0x0
 #define CATPT_SSPSP2_DEFAULT		0x0
 
+/* Coredump register and its states */
+#define CATPT_DRAM_COREDUMP		0x1F4
+#define CATPT_COREDUMP_REQUEST		UINT_MAX
+#define CATPT_COREDUMP_RELEASE		0
+
 /* Physically the same block, access address differs between host and dsp */
 #define CATPT_DSP_DRAM_OFFSET		0x400000
 #define catpt_to_host_offset(offset)	((offset) & ~(CATPT_DSP_DRAM_OFFSET))
@@ -137,6 +142,8 @@
 
 /* registry I/O helpers */
 
+#define catpt_dram_addr(cdev) \
+	((cdev)->lpe_ba + (cdev)->spec->host_dram_offset)
 #define catpt_shim_addr(cdev) \
 	((cdev)->lpe_ba + (cdev)->spec->host_shim_offset)
 #define catpt_dma_addr(cdev, dma) \
@@ -151,6 +158,11 @@
 #define catpt_writel_ssp(cdev, ssp, reg, val) \
 	writel(val, catpt_ssp_addr(cdev, ssp) + (reg))
 
+#define catpt_readl_dram(cdev, reg) \
+	readl(catpt_dram_addr(cdev) + CATPT_DRAM_##reg)
+#define catpt_writel_dram(cdev, reg, val) \
+	writel(val, catpt_dram_addr(cdev) + CATPT_DRAM_##reg)
+
 #define catpt_readl_shim(cdev, reg) \
 	readl(catpt_shim_addr(cdev) + CATPT_SHIM_##reg)
 #define catpt_writel_shim(cdev, reg, val) \
Signature · 1 line
2.34.1
maintainer
#2 lore raw

Re: [PATCH 1/3] ASoC: Intel: catpt: Complete coredump handling

Quoted 12 lines · Cezary Rojewski

An exception may occur during the firmware booting procedure. In such case the firmware sends COREDYMP_REQUESTS and expects the driver to dump relevant information and finish with the COREDUMP_RELEASE write.

To distinguish such situation from generic timeout, always signal fw_ready completion when a coredump request is received and translate it to -EREMOTEIO in catpt_boot_firmware().

The "FW READY" print makes the success clearly visible even when the event-traces are not enabled.

sound/soc/intel/catpt/dsp.c int catpt_coredump(struct catpt_dev *cdev) · line 543

Quoted 8 lines
sound/soc/intel/catpt/dsp.c +5 -0
@@ -536,6 +536,11 @@ int catpt_coredump(struct catpt_dev *cdev)
 		pos += CATPT_DMA_REGS_SIZE;
 	}
 
+	if (catpt_readl_dram(cdev, COREDUMP) == CATPT_COREDUMP_REQUEST) {
+		dev_dbg(cdev->dev, "releasing firmware from the coredump state\n");
+		catpt_writel_dram(cdev, COREDUMP, CATPT_COREDUMP_RELEASE);
+	}
+

Do we also need to do this if we fail to allocate the buffer for the core dump?

author maintainer
#3 lore raw

Re: [PATCH 1/3] ASoC: Intel: catpt: Complete coredump handling

Quoted 16 lines · Mark Brown
Quoted 15 lines · Cezary Rojewski

An exception may occur during the firmware booting procedure. In such case the firmware sends COREDYMP_REQUESTS and expects the driver to dump relevant information and finish with the COREDUMP_RELEASE write.

To distinguish such situation from generic timeout, always signal fw_ready completion when a coredump request is received and translate it to -EREMOTEIO in catpt_boot_firmware().

The "FW READY" print makes the success clearly visible even when the event-traces are not enabled.

sound/soc/intel/catpt/dsp.c +0 -0
@@ -536,6 +536,11 @@ int catpt_coredump(struct catpt_dev *cdev)
  		pos += CATPT_DMA_REGS_SIZE;
  	}
 
Quoted 8 lines

sound/soc/intel/catpt/dsp.c int catpt_coredump(struct catpt_dev *cdev) · line 543

Quoted 5 lines
sound/soc/intel/catpt/dsp.c +5 -0
+	if (catpt_readl_dram(cdev, COREDUMP) == CATPT_COREDUMP_REQUEST) {
+		dev_dbg(cdev->dev, "releasing firmware from the coredump state\n");
+		catpt_writel_dram(cdev, COREDUMP, CATPT_COREDUMP_RELEASE);
+	}
+

Do we also need to do this if we fail to allocate the buffer for the core dump?

Yes, the driver must ACK (COREDUMP_RELEASE) the request. Otherwise rmmod->modprobe remains as the only option.

Thank you for spotting this out. Will update in v2.

author maintainer
#4 lore raw

PATCH 2/3 ASoC: Intel: catpt: Add pretty-trace for large IPC payloads

In reply to Cezary Rojewski

Mimic mechanism found in the Intel's avs-driver and update the existing IPC payload tracing to allow for pretty printing even large payloads.

---
 sound/soc/intel/catpt/Makefile |  3 ++-
 sound/soc/intel/catpt/device.c |  3 ---
 sound/soc/intel/catpt/trace.c  | 27 +++++++++++++++++++++++++++
 sound/soc/intel/catpt/trace.h  | 22 +++++++++++++++-------
 4 files changed, 44 insertions(+), 11 deletions(-)
 create mode 100644 sound/soc/intel/catpt/trace.c
sound/soc/intel/catpt/Makefile +2 -1
index e8316e33b820..8005fc677f28 100644
@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 snd-soc-catpt-y := device.o dsp.o loader.o ipc.o messages.o pcm.o sysfs.o
 
+snd-soc-catpt-y += trace.o
 # tell define_trace.h where to find the trace header
-CFLAGS_device.o := -I$(src)
+CFLAGS_trace.o := -I$(src)
 
 obj-$(CONFIG_SND_SOC_INTEL_CATPT) += snd-soc-catpt.o
sound/soc/intel/catpt/device.c +0 -3
index ca4fd18b6a6e..6fe212f498c6 100644
@@ -25,9 +25,6 @@
 #include "core.h"
 #include "registers.h"
 
-#define CREATE_TRACE_POINTS
-#include "trace.h"
-
 static int catpt_do_suspend(struct device *dev)
 {
 	struct catpt_dev *cdev = dev_get_drvdata(dev);
sound/soc/intel/catpt/trace.c new file +27 -0
new file mode 100644
index 000000000000..e97c372cc2af
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/types.h>
+
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+
+#define BYTES_PER_LINE 16
+#define MAX_CHUNK_SIZE ((PAGE_SIZE - 150) /* Place for trace header */	\
+			/ (2 * BYTES_PER_LINE + 4) /* chars per line */	\
+			* BYTES_PER_LINE)
+
+void trace_catpt_ipc_payload(const void *data, size_t size)
+{
+	size_t remaining = size;
+	size_t offset = 0;
+
+	while (remaining > 0) {
+		u32 chunk;
+
+		chunk = min_t(size_t, remaining, MAX_CHUNK_SIZE);
+		trace_catpt_ipc_payload_chunk(data, chunk, offset, size);
+
+		remaining -= chunk;
+		offset += chunk;
+	}
+}
sound/soc/intel/catpt/trace.h +15 -7
index 010f57b6a7a8..6b528d933734 100644
@@ -51,29 +51,37 @@ DEFINE_EVENT(catpt_ipc_msg, catpt_ipc_notify,
 	TP_ARGS(header)
 );
 
-TRACE_EVENT_CONDITION(catpt_ipc_payload,
+TRACE_EVENT_CONDITION(catpt_ipc_payload_chunk,
 
-	TP_PROTO(const u8 *data, size_t size),
+	TP_PROTO(const u8 *data, size_t size, size_t offset, size_t total),
 
-	TP_ARGS(data, size),
+	TP_ARGS(data, size, offset, total),
 
 	TP_CONDITION(data && size),
 
 	TP_STRUCT__entry(
-		__dynamic_array(u8, buf, size)
+		__dynamic_array(u8,	buf,	size	)
+		__field(size_t,		offset		)
+		__field(size_t,		pos		)
+		__field(size_t,		total		)
 	),
 
 	TP_fast_assign(
-		memcpy(__get_dynamic_array(buf), data, size);
+		memcpy(__get_dynamic_array(buf), data + offset, size);
+		__entry->offset = offset;
+		__entry->pos = offset + size;
+		__entry->total = total;
 	),
 
-	TP_printk("%u byte(s)%s",
-		  __get_dynamic_array_len(buf),
+	TP_printk("range %zu-%zu out of %zu bytes%s",
+		  __entry->offset, __entry->pos, __entry->total,
 		  __print_hex_dump("", DUMP_PREFIX_NONE, 16, 4,
 				   __get_dynamic_array(buf),
 				   __get_dynamic_array_len(buf), false))
 );
 
+void trace_catpt_ipc_payload(const void *data, size_t size);
+
 #endif /* __SND_SOC_INTEL_CATPT_TRACE_H */
 
 /* This part must be outside protection */
Signature · 1 line
2.34.1
author maintainer
#5 lore raw

PATCH 3/3 ASoC: Intel: catpt: Print error code if board-registration fails

In reply to Cezary Rojewski

Message alone without the code does not tell us much.

---
 sound/soc/intel/catpt/device.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
sound/soc/intel/catpt/device.c +1 -1
index 6fe212f498c6..b0a926db483c 100644
@@ -154,7 +154,7 @@ static int catpt_register_board(struct catpt_dev *cdev)
 					PLATFORM_DEVID_NONE,
 					(const void *)mach, sizeof(*mach));
 	if (IS_ERR(board)) {
-		dev_err(cdev->dev, "board register failed\n");
+		dev_err(cdev->dev, "register board failed: %ld\n", PTR_ERR(board));
 		return PTR_ERR(board);
 	}
 
Signature · 1 line
2.34.1