Skip to contentSkip to conversation

drm/msm: GPU crash state

RFC Superseded Jordan Crouse 3 participants 8 messages Jan 26, 2018 – Feb 2, 2018 1 message not in the archive

Subsystems (4)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover drm/msm: GPU crash state

This is an update for my previous stack of GPU state code
(https://patchwork.freedesktop.org/series/36097/).

The goal is to store and provide enough information to debug software and hardware issues on the Adreno hardware in a semi human-readable format that can also be parsed by scripts.

Based on previous suggestions, I re-organized the output into a mostly YAML compatible format that should be extensible for future changes. Once I get basic consensus on the format I'll write up official format documentation. I also removed some of the "cleanup" code for the show function that just ended up being more trouble than it is worth. Finally, I added code to dump the ringbuffers in ascii85 format and started work on a5xx specific code.

Future additions will include tweaking the format, adding more 5xx specific components and dumping active buffers that contributed to the crash.

You can see an example of the output for a simple invalid opcode error on the db820c here: https://hastebin.com/olaruyakaz.bash

Jordan Crouse (6):
drm/msm: gpu: Capture the state of the GPU
drm/msm: gpu: Convert the GPU show function to use the GPU state drm/msm: gpu: Capture the GPU state on a GPU hang drm/msm/adreno: Convert the show/crash file format drm/msm/adreno: Add ringbuffer contexts to the GPU state drm/msm/adreno: Add a5xx specific registers for the GPU state

 drivers/gpu/drm/msm/adreno/a3xx_gpu.c   |  28 ++--
 drivers/gpu/drm/msm/adreno/a4xx_gpu.c   |  20 ++-
 drivers/gpu/drm/msm/adreno/a5xx_gpu.c   | 238 ++++++++++++++++++++++++++++++--
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 148 +++++++++++++++++---
 drivers/gpu/drm/msm/adreno/adreno_gpu.h |   7 +-
 drivers/gpu/drm/msm/msm_debugfs.c       |  84 ++++++++++-
 drivers/gpu/drm/msm/msm_gpu.c           |  47 +++++--
 drivers/gpu/drm/msm/msm_gpu.h           |  58 +++++++-
 8 files changed, 567 insertions(+), 63 deletions(-)
Signature · 6 lines
1.9.1

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

A message that is not in the archive (sent off-list or not yet received): 1517000930-1893-1-git-send-email-jcrouse-sgV2jX0FEOL9JmXXK+q4OQ@public.gmane.org

#2 lore raw

PATCH 1/6 drm/msm: gpu: Capture the state of the GPU

Add the infrastructure to capture the state current state of the GPU and store it in memory. This is useful for storing the state of a hung GPU so it can be dumped later.

For now grab the same basic ringbuffer information and registers that are provided by the debugfs 'gpu' node but obviously this can be extended to capture a much larger set of GPU information.

---
 drivers/gpu/drm/msm/adreno/a3xx_gpu.c   | 15 +++++++++
 drivers/gpu/drm/msm/adreno/a4xx_gpu.c   | 14 +++++++++
 drivers/gpu/drm/msm/adreno/a5xx_gpu.c   | 22 ++++++++++++++
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 54 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/msm/adreno/adreno_gpu.h |  3 ++
 drivers/gpu/drm/msm/msm_gpu.h           | 19 ++++++++++++
 6 files changed, 127 insertions(+)
drivers/gpu/drm/msm/adreno/a3xx_gpu.c +15 -0
index 4baef27..012e997 100644
@@ -426,6 +426,19 @@ static void a3xx_dump(struct msm_gpu *gpu)
 			gpu_read(gpu, REG_A3XX_RBBM_STATUS));
 	adreno_dump(gpu);
 }
+
+static struct msm_gpu_state *a3xx_gpu_state_get(struct msm_gpu *gpu)
+{
+	struct msm_gpu_state *state = adreno_gpu_state_get(gpu);
+
+	if (IS_ERR(state))
+		return state;
+
+	state->rbbm_status = gpu_read(gpu, REG_A3XX_RBBM_STATUS);
+
+	return state;
+}
+
 /* Register offset defines for A3XX */
 static const unsigned int a3xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_AXXX_CP_RB_BASE),
@@ -452,6 +465,8 @@ static void a3xx_dump(struct msm_gpu *gpu)
 #ifdef CONFIG_DEBUG_FS
 		.show = a3xx_show,
 #endif
+		.gpu_state_get = a3xx_gpu_state_get,
+		.gpu_state_put = adreno_gpu_state_put,
 	},
 };
 
drivers/gpu/drm/msm/adreno/a4xx_gpu.c +14 -0
index 8199a4b..c875dccc 100644
@@ -464,6 +464,18 @@ static void a4xx_show(struct msm_gpu *gpu, struct seq_file *m)
 }
 #endif
 
+static struct msm_gpu_state *a4xx_gpu_state_get(struct msm_gpu *gpu)
+{
+	struct msm_gpu_state *state = adreno_gpu_state_get(gpu);
+
+	if (IS_ERR(state))
+		return state;
+
+	state->rbbm_status = gpu_read(gpu, REG_A4XX_RBBM_STATUS);
+
+	return state;
+}
+
 /* Register offset defines for A4XX, in order of enum adreno_regs */
 static const unsigned int a4xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A4XX_CP_RB_BASE),
@@ -540,6 +552,8 @@ static int a4xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 #ifdef CONFIG_DEBUG_FS
 		.show = a4xx_show,
 #endif
+		.gpu_state_get = a4xx_gpu_state_get,
+		.gpu_state_put = adreno_gpu_state_put,
 	},
 	.get_timestamp = a4xx_get_timestamp,
 };
drivers/gpu/drm/msm/adreno/a5xx_gpu.c +22 -0
index 7e09d44..6199a67 100644
@@ -1149,6 +1149,26 @@ static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 	return 0;
 }
 
+static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
+{
+	struct msm_gpu_state *state;
+
+	/*
+	 * Temporarily disable hardware clock gating before going into
+	 * adreno_show to avoid issues while reading the registers
+	 */
+	a5xx_set_hwcg(gpu, false);
+
+	state = adreno_gpu_state_get(gpu);
+
+	if (!IS_ERR(state))
+		state->rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
+
+	a5xx_set_hwcg(gpu, true);
+
+	return state;
+}
+
 #ifdef CONFIG_DEBUG_FS
 static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
 {
@@ -1197,6 +1217,8 @@ static int a5xx_gpu_busy(struct msm_gpu *gpu, uint64_t *value)
 		.show = a5xx_show,
 #endif
 		.gpu_busy = a5xx_gpu_busy,
+		.gpu_state_get = a5xx_gpu_state_get,
+		.gpu_state_put = adreno_gpu_state_put,
 	},
 	.get_timestamp = a5xx_get_timestamp,
 };
drivers/gpu/drm/msm/adreno/adreno_gpu.c +54 -0

60 lines not on this page · raw message

drivers/gpu/drm/msm/adreno/adreno_gpu.h +3 -0
index 8d3d0a9..1c32d18 100644
@@ -220,6 +220,9 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 void adreno_gpu_cleanup(struct adreno_gpu *gpu);
 
 
+struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu);
+void adreno_gpu_state_put(struct msm_gpu_state *state);
+
 /* ringbuffer helpers (the parts that are adreno specific) */
 
 static inline void
drivers/gpu/drm/msm/msm_gpu.h +19 -0
index fccfccd..96e8b6f 100644
@@ -27,6 +27,7 @@
 
 struct msm_gem_submit;
 struct msm_gpu_perfcntr;
+struct msm_gpu_state;
 
 struct msm_gpu_config {
 	const char *ioname;
@@ -67,6 +68,8 @@ struct msm_gpu_funcs {
 	void (*show)(struct msm_gpu *gpu, struct seq_file *m);
 #endif
 	int (*gpu_busy)(struct msm_gpu *gpu, uint64_t *value);
+	struct msm_gpu_state *(*gpu_state_get)(struct msm_gpu *gpu);
+	void (*gpu_state_put)(struct msm_gpu_state *state);
 };
 
 struct msm_gpu {
@@ -173,6 +176,22 @@ struct msm_gpu_submitqueue {
 	struct kref ref;
 };
 
+struct msm_gpu_state {
+	struct timeval time;
+
+	struct {
+		u32 fence;
+		u32 seqno;
+		u32 rptr;
+		u32 wptr;
+	} ring[MSM_GPU_MAX_RINGS];
+
+	int nr_registers;
+	u32 *registers;
+
+	u32 rbbm_status;
+};
+
 static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
 {
 	msm_writel(data, gpu->mmio + (reg << 2));
Signature · 6 lines
1.9.1

_______________________________________________
Freedreno mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/freedreno
#3 lore raw

PATCH 6/6 drm/msm/adreno: Add a5xx specific registers for the GPU state

In reply to a message not in the archive

HLSQ, SP and TP registers are only accessible from a special aperture and to make matters worsex, the aperture is blocked from the CPU on targets that can support secure rendering. Luckily the GPU hardware has its own purpose built register dumper that can access the registers from the aperture. Add a5xx specific code to program the crashdumper and retrieve the wayward registers and dump them for the crash state.

Also, remove a block of registers the regular CPU accessible list that aren't useful for debug which helps reduce the size of the crash state file by a goodly amount.

---
 drivers/gpu/drm/msm/adreno/a3xx_gpu.c   |   8 +-
 drivers/gpu/drm/msm/adreno/a4xx_gpu.c   |   8 +-
 drivers/gpu/drm/msm/adreno/a5xx_gpu.c   | 234 ++++++++++++++++++++++++++++++--
 drivers/gpu/drm/msm/adreno/adreno_gpu.c |  23 ++--
 drivers/gpu/drm/msm/adreno/adreno_gpu.h |   4 +-
 5 files changed, 246 insertions(+), 31 deletions(-)
drivers/gpu/drm/msm/adreno/a3xx_gpu.c +5 -3
index 72e523f1..14340e3 100644
@@ -420,10 +420,12 @@ static void a3xx_dump(struct msm_gpu *gpu)
 
 static struct msm_gpu_state *a3xx_gpu_state_get(struct msm_gpu *gpu)
 {
-	struct msm_gpu_state *state = adreno_gpu_state_get(gpu);
+	struct msm_gpu_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
 
-	if (IS_ERR(state))
-		return state;
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	adreno_gpu_state_get(gpu, state);
 
 	state->rbbm_status = gpu_read(gpu, REG_A3XX_RBBM_STATUS);
 
drivers/gpu/drm/msm/adreno/a4xx_gpu.c +5 -3
index e363e65..4dde681 100644
@@ -456,10 +456,12 @@ static irqreturn_t a4xx_irq(struct msm_gpu *gpu)
 
 static struct msm_gpu_state *a4xx_gpu_state_get(struct msm_gpu *gpu)
 {
-	struct msm_gpu_state *state = adreno_gpu_state_get(gpu);
+	struct msm_gpu_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
 
-	if (IS_ERR(state))
-		return state;
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	adreno_gpu_state_get(gpu, state);
 
 	state->rbbm_status = gpu_read(gpu, REG_A4XX_RBBM_STATUS);
 
drivers/gpu/drm/msm/adreno/a5xx_gpu.c +221 -13

270 lines not on this page · raw message

drivers/gpu/drm/msm/adreno/adreno_gpu.c +12 -11
index 79df82b..fd39b3c 100644
@@ -363,16 +363,11 @@ bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 	return false;
 }
 
-struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu)
+int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-	struct msm_gpu_state *state;
 	int i, count = 0;
 
-	state = kzalloc(sizeof(*state), GFP_KERNEL);
-	if (!state)
-		return ERR_PTR(-ENOMEM);
-
 	kref_init(&state->ref);
 
 	do_gettimeofday(&state->time);
@@ -426,14 +421,12 @@ struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu)
 		state->nr_registers = count;
 	}
 
-	return state;
+	return 0;
 }
 
-static void adreno_gpu_state_destroy(struct kref *kref)
+void adreno_gpu_state_destroy(struct msm_gpu_state *state)
 {
 	int i;
-	struct msm_gpu_state *state = container_of(kref,
-		struct msm_gpu_state, ref);
 
 	for(i = 0; i < ARRAY_SIZE(state->ring); i++)
 		kfree(state->ring[i].data);
@@ -441,6 +434,14 @@ static void adreno_gpu_state_destroy(struct kref *kref)
 	kfree(state->comm);
 	kfree(state->cmd);
 	kfree(state->registers);
+}
+
+static void adreno_gpu_state_kref_destroy(struct kref *kref)
+{
+	struct msm_gpu_state *state = container_of(kref,
+		struct msm_gpu_state, ref);
+
+	adreno_gpu_state_destroy(state);
 	kfree(state);
 }
 
@@ -449,7 +450,7 @@ int adreno_gpu_state_put(struct msm_gpu_state *state)
 	if (IS_ERR_OR_NULL(state))
 		return 1;
 
-	return kref_put(&state->ref, adreno_gpu_state_destroy);
+	return kref_put(&state->ref, adreno_gpu_state_kref_destroy);
 }
 
 #ifdef CONFIG_DEBUG_FS
drivers/gpu/drm/msm/adreno/adreno_gpu.h +3 -1
index bcf755e..313e66a 100644
@@ -220,7 +220,9 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		int nr_rings);
 void adreno_gpu_cleanup(struct adreno_gpu *gpu);
 
-struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu);
+void adreno_gpu_state_destroy(struct msm_gpu_state *state);
+
+int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state);
 int adreno_gpu_state_put(struct msm_gpu_state *state);
 
 /* ringbuffer helpers (the parts that are adreno specific) */
Signature · 6 lines
1.9.1

_______________________________________________
Freedreno mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/freedreno
#4 lore raw

PATCH v2 2/6 drm/msm: gpu: Convert the GPU show function to use the GPU state

In reply to Jordan Crouse

Convert the existing GPU show function to use the GPU state to dump the information rather than reading it directly from the hardware. This will require an additional step to capture the state before dumping it for the existing nodes but it will greatly facilitate reusing the same code for dumping a previously captured state from a GPU hang.

---
 drivers/gpu/drm/msm/adreno/a3xx_gpu.c   | 11 +----------
 drivers/gpu/drm/msm/adreno/a4xx_gpu.c   | 12 +-----------
 drivers/gpu/drm/msm/adreno/a5xx_gpu.c   | 18 +-----------------
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 30 +++++++++++++-----------------
 drivers/gpu/drm/msm/adreno/adreno_gpu.h |  4 ++--
 drivers/gpu/drm/msm/msm_debugfs.c       | 21 +++++++++++++++------
 drivers/gpu/drm/msm/msm_gpu.h           |  3 ++-
 7 files changed, 35 insertions(+), 64 deletions(-)
drivers/gpu/drm/msm/adreno/a3xx_gpu.c +1 -10
index 012e997..72e523f1 100644
@@ -410,15 +410,6 @@ static irqreturn_t a3xx_irq(struct msm_gpu *gpu)
 	~0   /* sentinel */
 };
 
-#ifdef CONFIG_DEBUG_FS
-static void a3xx_show(struct msm_gpu *gpu, struct seq_file *m)
-{
-	seq_printf(m, "status:   %08x\n",
-			gpu_read(gpu, REG_A3XX_RBBM_STATUS));
-	adreno_show(gpu, m);
-}
-#endif
-
 /* would be nice to not have to duplicate the _show() stuff with printk(): */
 static void a3xx_dump(struct msm_gpu *gpu)
 {
@@ -463,7 +454,7 @@ static struct msm_gpu_state *a3xx_gpu_state_get(struct msm_gpu *gpu)
 		.irq = a3xx_irq,
 		.destroy = a3xx_destroy,
 #ifdef CONFIG_DEBUG_FS
-		.show = a3xx_show,
+		.show = adreno_show,
 #endif
 		.gpu_state_get = a3xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
drivers/gpu/drm/msm/adreno/a4xx_gpu.c +1 -11
index c875dccc..e363e65 100644
@@ -454,16 +454,6 @@ static irqreturn_t a4xx_irq(struct msm_gpu *gpu)
 	~0 /* sentinel */
 };
 
-#ifdef CONFIG_DEBUG_FS
-static void a4xx_show(struct msm_gpu *gpu, struct seq_file *m)
-{
-	seq_printf(m, "status:   %08x\n",
-			gpu_read(gpu, REG_A4XX_RBBM_STATUS));
-	adreno_show(gpu, m);
-
-}
-#endif
-
 static struct msm_gpu_state *a4xx_gpu_state_get(struct msm_gpu *gpu)
 {
 	struct msm_gpu_state *state = adreno_gpu_state_get(gpu);
@@ -550,7 +540,7 @@ static int a4xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 		.irq = a4xx_irq,
 		.destroy = a4xx_destroy,
 #ifdef CONFIG_DEBUG_FS
-		.show = a4xx_show,
+		.show = adreno_show,
 #endif
 		.gpu_state_get = a4xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
drivers/gpu/drm/msm/adreno/a5xx_gpu.c +1 -17
index 6199a67..971aaee 100644
@@ -1169,22 +1169,6 @@ static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
 	return state;
 }
 
-#ifdef CONFIG_DEBUG_FS
-static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
-{
-	seq_printf(m, "status:   %08x\n",
-			gpu_read(gpu, REG_A5XX_RBBM_STATUS));
-
-	/*
-	 * Temporarily disable hardware clock gating before going into
-	 * adreno_show to avoid issues while reading the registers
-	 */
-	a5xx_set_hwcg(gpu, false);
-	adreno_show(gpu, m);
-	a5xx_set_hwcg(gpu, true);
-}
-#endif
-
 static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -1214,7 +1198,7 @@ static int a5xx_gpu_busy(struct msm_gpu *gpu, uint64_t *value)
 		.irq = a5xx_irq,
 		.destroy = a5xx_destroy,
 #ifdef CONFIG_DEBUG_FS
-		.show = a5xx_show,
+		.show = adreno_show,
 #endif
 		.gpu_busy = a5xx_gpu_busy,
 		.gpu_state_get = a5xx_gpu_state_get,
drivers/gpu/drm/msm/adreno/adreno_gpu.c +13 -17
index 754b88b..81da214 100644
@@ -417,38 +417,34 @@ void adreno_gpu_state_put(struct msm_gpu_state *state)
 }
 
 #ifdef CONFIG_DEBUG_FS
-void adreno_show(struct msm_gpu *gpu, struct seq_file *m)
+void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct seq_file *m)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	int i;
 
+	if (IS_ERR_OR_NULL(state))
+		return;
+
+	seq_printf(m, "status:   %08x\n", state->rbbm_status);
 	seq_printf(m, "revision: %d (%d.%d.%d.%d)\n",
 			adreno_gpu->info->revn, adreno_gpu->rev.core,
 			adreno_gpu->rev.major, adreno_gpu->rev.minor,
 			adreno_gpu->rev.patchid);
 
 	for (i = 0; i < gpu->nr_rings; i++) {
-		struct msm_ringbuffer *ring = gpu->rb[i];
-
 		seq_printf(m, "rb %d: fence:    %d/%d\n", i,
-			ring->memptrs->fence, ring->seqno);
+			state->ring[i].fence, state->ring[i].seqno);
 
-		seq_printf(m, "      rptr:     %d\n",
-			get_rptr(adreno_gpu, ring));
-		seq_printf(m, "rb wptr:  %d\n", get_wptr(ring));
+		seq_printf(m, "      rptr:     %d\n", state->ring[i].rptr);
+		seq_printf(m, "rb wptr:  %d\n", state->ring[i].wptr);
 	}
 
-	/* dump these out in a form that can be parsed by demsm: */
 	seq_printf(m, "IO:region %s 00000000 00020000\n", gpu->name);
-	for (i = 0; adreno_gpu->registers[i] != ~0; i += 2) {
-		uint32_t start = adreno_gpu->registers[i];
-		uint32_t end   = adreno_gpu->registers[i+1];
-		uint32_t addr;
-
-		for (addr = start; addr <= end; addr++) {
-			uint32_t val = gpu_read(gpu, addr);
-			seq_printf(m, "IO:R %08x %08x\n", addr<<2, val);
-		}
+	for (i = 0; i < state->nr_registers; i++) {
+		seq_printf(m, "IO:R %08x %08x\n",
+			state->registers[i * 2] << 2,
+			state->registers[(i * 2) + 1]);
 	}
 }
 #endif
drivers/gpu/drm/msm/adreno/adreno_gpu.h +2 -2
index 1c32d18..b44e0b9 100644
@@ -207,7 +207,8 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 void adreno_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 #ifdef CONFIG_DEBUG_FS
-void adreno_show(struct msm_gpu *gpu, struct seq_file *m);
+void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct seq_file *m);
 #endif
 void adreno_dump_info(struct msm_gpu *gpu);
 void adreno_dump(struct msm_gpu *gpu);
@@ -219,7 +220,6 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		int nr_rings);
 void adreno_gpu_cleanup(struct adreno_gpu *gpu);
 
-
 struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu);
 void adreno_gpu_state_put(struct msm_gpu_state *state);
 
drivers/gpu/drm/msm/msm_debugfs.c +15 -6
index 1855182..89ee74b 100644
@@ -25,13 +25,22 @@ static int msm_gpu_show(struct drm_device *dev, struct seq_file *m)
 {
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_gpu *gpu = priv->gpu;
+	struct msm_gpu_state *state;
 
-	if (gpu) {
-		seq_printf(m, "%s Status:\n", gpu->name);
-		pm_runtime_get_sync(&gpu->pdev->dev);
-		gpu->funcs->show(gpu, m);
-		pm_runtime_put_sync(&gpu->pdev->dev);
-	}
+	if (!gpu)
+		return 0;
+
+	pm_runtime_get_sync(&gpu->pdev->dev);
+	state = gpu->funcs->gpu_state_get(gpu);
+	pm_runtime_put_sync(&gpu->pdev->dev);
+
+	if (IS_ERR(state))
+		return PTR_ERR(state);
+
+	seq_printf(m, "%s Status:\n", gpu->name);
+	gpu->funcs->show(gpu, state, m);
+
+	gpu->funcs->gpu_state_put(state);
 
 	return 0;
 }
drivers/gpu/drm/msm/msm_gpu.h +2 -1
index 96e8b6f..42853e9 100644
@@ -65,7 +65,8 @@ struct msm_gpu_funcs {
 	void (*destroy)(struct msm_gpu *gpu);
 #ifdef CONFIG_DEBUG_FS
 	/* show GPU status in debugfs: */
-	void (*show)(struct msm_gpu *gpu, struct seq_file *m);
+	void (*show)(struct msm_gpu *gpu, struct msm_gpu_state *state,
+			struct seq_file *m);
 #endif
 	int (*gpu_busy)(struct msm_gpu *gpu, uint64_t *value);
 	struct msm_gpu_state *(*gpu_state_get)(struct msm_gpu *gpu);
Signature · 1 line
1.9.1
#5 lore raw

PATCH v2 3/6 drm/msm: gpu: Capture the GPU state on a GPU hang

In reply to Jordan Crouse

Capture the GPU state on a GPU hang and store it for later playback using the 'crash' node in the debugfs directory. Only one crash state is stored at a time on the assumption that the first hang is usually the most interesting. The existing crash state can be cleared by writing to the debugfs node and then a new one will be captured on the next hang.

---
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 18 ++++++++--
 drivers/gpu/drm/msm/adreno/adreno_gpu.h |  2 +-
 drivers/gpu/drm/msm/msm_debugfs.c       | 61 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/msm/msm_gpu.c           | 47 ++++++++++++++++++++-----
 drivers/gpu/drm/msm/msm_gpu.h           | 36 ++++++++++++++++++-
 5 files changed, 151 insertions(+), 13 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_gpu.c +15 -3
index 81da214..963fce3 100644
@@ -372,6 +372,8 @@ struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu)
 	if (!state)
 		return ERR_PTR(-ENOMEM);
 
+	kref_init(&state->ref);
+
 	do_gettimeofday(&state->time);
 
 	for (i = 0; i < gpu->nr_rings; i++) {
@@ -407,15 +409,25 @@ struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu)
 	return state;
 }
 
-void adreno_gpu_state_put(struct msm_gpu_state *state)
+static void adreno_gpu_state_destroy(struct kref *kref)
 {
-	if (IS_ERR_OR_NULL(state))
-		return;
+	struct msm_gpu_state *state = container_of(kref,
+		struct msm_gpu_state, ref);
 
+	kfree(state->comm);
+	kfree(state->cmd);
 	kfree(state->registers);
 	kfree(state);
 }
 
+int adreno_gpu_state_put(struct msm_gpu_state *state)
+{
+	if (IS_ERR_OR_NULL(state))
+		return 1;
+
+	return kref_put(&state->ref, adreno_gpu_state_destroy);
+}
+
 #ifdef CONFIG_DEBUG_FS
 void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
 		struct seq_file *m)
drivers/gpu/drm/msm/adreno/adreno_gpu.h +1 -1
index b44e0b9..bcf755e 100644
@@ -221,7 +221,7 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 void adreno_gpu_cleanup(struct adreno_gpu *gpu);
 
 struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu);
-void adreno_gpu_state_put(struct msm_gpu_state *state);
+int adreno_gpu_state_put(struct msm_gpu_state *state);
 
 /* ringbuffer helpers (the parts that are adreno specific) */
 
drivers/gpu/drm/msm/msm_debugfs.c +61 -0

78 lines not on this page · raw message

drivers/gpu/drm/msm/msm_gpu.c +39 -8

79 lines not on this page · raw message

drivers/gpu/drm/msm/msm_gpu.h +35 -1
index 42853e9..23e3b06 100644
@@ -70,7 +70,7 @@ struct msm_gpu_funcs {
 #endif
 	int (*gpu_busy)(struct msm_gpu *gpu, uint64_t *value);
 	struct msm_gpu_state *(*gpu_state_get)(struct msm_gpu *gpu);
-	void (*gpu_state_put)(struct msm_gpu_state *state);
+	int (*gpu_state_put)(struct msm_gpu_state *state);
 };
 
 struct msm_gpu {
@@ -131,6 +131,8 @@ struct msm_gpu {
 		u64 busy_cycles;
 		ktime_t time;
 	} devfreq;
+
+	struct msm_gpu_state *crashstate;
 };
 
 /* It turns out that all targets use the same ringbuffer size */
@@ -178,6 +180,7 @@ struct msm_gpu_submitqueue {
 };
 
 struct msm_gpu_state {
+	struct kref ref;
 	struct timeval time;
 
 	struct {
@@ -191,6 +194,9 @@ struct msm_gpu_state {
 	u32 *registers;
 
 	u32 rbbm_status;
+
+	char *comm;
+	char *cmd;
 };
 
 static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
@@ -272,4 +278,32 @@ static inline void msm_submitqueue_put(struct msm_gpu_submitqueue *queue)
 		kref_put(&queue->ref, msm_submitqueue_destroy);
 }
 
+static inline struct msm_gpu_state *msm_gpu_crashstate_get(struct msm_gpu *gpu)
+{
+	struct msm_gpu_state *state = NULL;
+
+	mutex_lock(&gpu->dev->struct_mutex);
+
+	if (gpu->crashstate) {
+		kref_get(&gpu->crashstate->ref);
+		state = gpu->crashstate;
+	}
+
+	mutex_unlock(&gpu->dev->struct_mutex);
+
+	return state;
+}
+
+static inline void msm_gpu_crashstate_put(struct msm_gpu *gpu)
+{
+	mutex_lock(&gpu->dev->struct_mutex);
+
+	if (gpu->crashstate) {
+		if (gpu->funcs->gpu_state_put(gpu->crashstate))
+			gpu->crashstate = NULL;
+	}
+
+	mutex_unlock(&gpu->dev->struct_mutex);
+}
+
 #endif /* __MSM_GPU_H__ */
Signature · 1 line
1.9.1
#6 lore raw

PATCH v2 4/6 drm/msm/adreno: Convert the show/crash file format

In reply to Jordan Crouse

Convert the format of the 'show' and 'crash' debugfs files to mostly standard YAML. This should be easier to parse and be more flexible for future changes and expansions.

---
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 23 ++++++++++++++---------
 drivers/gpu/drm/msm/msm_debugfs.c       |  8 +++++---
 2 files changed, 19 insertions(+), 12 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_gpu.c +14 -9
index 963fce3..9e83c70 100644
@@ -438,23 +438,28 @@ void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
 	if (IS_ERR_OR_NULL(state))
 		return;
 
-	seq_printf(m, "status:   %08x\n", state->rbbm_status);
 	seq_printf(m, "revision: %d (%d.%d.%d.%d)\n",
 			adreno_gpu->info->revn, adreno_gpu->rev.core,
 			adreno_gpu->rev.major, adreno_gpu->rev.minor,
 			adreno_gpu->rev.patchid);
 
-	for (i = 0; i < gpu->nr_rings; i++) {
-		seq_printf(m, "rb %d: fence:    %d/%d\n", i,
-			state->ring[i].fence, state->ring[i].seqno);
+	seq_printf(m, "rbbm-status: 0x%08x\n", state->rbbm_status);
+
+	seq_printf(m, "ringbuffer:\n");
 
-		seq_printf(m, "      rptr:     %d\n", state->ring[i].rptr);
-		seq_printf(m, "rb wptr:  %d\n", state->ring[i].wptr);
+	for (i = 0; i < gpu->nr_rings; i++) {
+		seq_printf(m, "  - id: %d\n", i);
+		seq_printf(m, "    last-fence: %d\n", state->ring[i].seqno);
+		seq_printf(m, "    retired-fence: %d\n", state->ring[i].fence);
+		seq_printf(m, "    rptr: %d\n", state->ring[i].rptr);
+		seq_printf(m, "    wptr: %d\n", state->ring[i].wptr);
 	}
 
-	seq_printf(m, "IO:region %s 00000000 00020000\n", gpu->name);
-	for (i = 0; i < state->nr_registers; i++) {
-		seq_printf(m, "IO:R %08x %08x\n",
+	seq_printf(m, "registers:\n");
+	seq_printf(m, "  - [offset, value]\n");
+
+	for(i = 0; i < state->nr_registers; i++) {
+		seq_printf(m, "  - [0x%04x, 0x%08x]\n",
 			state->registers[i * 2] << 2,
 			state->registers[(i * 2) + 1]);
 	}
drivers/gpu/drm/msm/msm_debugfs.c +5 -3
index 50e049c..e6d41d9 100644
@@ -33,9 +33,11 @@ static int msm_gpu_crash_show(struct seq_file *m, void *data)
 	if (!state)
 		return 0;
 
-	seq_printf(m, "%s Crash Status:\n", gpu->name);
-	seq_puts(m, "Kernel: " UTS_RELEASE "\n");
-	seq_printf(m, "Time: %ld s %ld us\n",
+	/* FIXME: add tags? */
+	seq_puts(m, "---\n");
+	seq_puts(m, "kernel: " UTS_RELEASE "\n");
+	seq_printf(m, "module: " KBUILD_MODNAME "\n");
+	seq_printf(m, "time: %ld.%ld\n",
 		state->time.tv_sec, state->time.tv_usec);
 	if (state->comm)
 		seq_printf(m, "comm: %s\n", state->comm);
Signature · 6 lines
1.9.1

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
#7 lore raw

PATCH v2 5/6 drm/msm/adreno: Add ringbuffer contexts to the GPU state

In reply to Jordan Crouse

Add the contents of each ringbuffer to the GPU state and dump the data in the crash file encoded with ascii85. To save space only the used portions of the ringbuffer are saved.

---
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 44 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/msm/msm_gpu.h           |  2 ++
 2 files changed, 46 insertions(+)
drivers/gpu/drm/msm/adreno/adreno_gpu.c +44 -0
index 9e83c70..79df82b 100644
@@ -17,6 +17,7 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/ascii85.h>
 #include <linux/pm_opp.h>
 #include "adreno_gpu.h"
 #include "msm_gem.h"
@@ -377,10 +378,29 @@ struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu)
 	do_gettimeofday(&state->time);
 
 	for (i = 0; i < gpu->nr_rings; i++) {
+		int size = 0, j;
+
 		state->ring[i].fence = gpu->rb[i]->memptrs->fence;
 		state->ring[i].seqno = gpu->rb[i]->seqno;
 		state->ring[i].rptr = get_rptr(adreno_gpu, gpu->rb[i]);
 		state->ring[i].wptr = get_wptr(gpu->rb[i]);
+
+		/*
+		 * Only copy used parts of the ring buffers (this should save
+		 * data size for lightly used rings)
+		 */
+		for(j = 0; j < MSM_GPU_RINGBUFFER_SZ >> 2; j++)
+			if (gpu->rb[i]->start[j])
+				size = j;
+
+		if (size) {
+			state->ring[i].data = kmalloc((size + 1) << 2, GFP_KERNEL);
+			if (state->ring[i].data) {
+				memcpy(state->ring[i].data, gpu->rb[i]->start,
+				(size + 1) << 2);
+				state->ring[i].data_size = (size + 1) << 2;
+			}
+		}
 	}
 
 	/* Count the number of registers */
@@ -411,9 +431,13 @@ struct msm_gpu_state *adreno_gpu_state_get(struct msm_gpu *gpu)
 
 static void adreno_gpu_state_destroy(struct kref *kref)
 {
+	int i;
 	struct msm_gpu_state *state = container_of(kref,
 		struct msm_gpu_state, ref);
 
+	for(i = 0; i < ARRAY_SIZE(state->ring); i++)
+		kfree(state->ring[i].data);
+
 	kfree(state->comm);
 	kfree(state->cmd);
 	kfree(state->registers);
@@ -453,6 +477,26 @@ void adreno_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
 		seq_printf(m, "    retired-fence: %d\n", state->ring[i].fence);
 		seq_printf(m, "    rptr: %d\n", state->ring[i].rptr);
 		seq_printf(m, "    wptr: %d\n", state->ring[i].wptr);
+		seq_printf(m, "    size: %d\n", MSM_GPU_RINGBUFFER_SZ);
+
+		if (state->ring[i].data && state->ring[i].data_size) {
+			u32 *ptr = (u32 *) state->ring[i].data;
+			char out[6];
+			int len = ascii85_encode_len(state->ring[i].data_size);
+			int j;
+
+			seq_printf(m, "    data: !!ascii85 |\n");
+			seq_printf(m, "     ");
+
+			for(j = 0; j < len; j++) {
+				if (ascii85_encode(ptr[j], out))
+					seq_printf(m, out);
+				else
+					seq_printf(m, "z");
+			}
+
+			seq_printf(m, "\n");
+		}
 	}
 
 	seq_printf(m, "registers:\n");
drivers/gpu/drm/msm/msm_gpu.h +2 -0
index 23e3b06..e13b23b 100644
@@ -188,6 +188,8 @@ struct msm_gpu_state {
 		u32 seqno;
 		u32 rptr;
 		u32 wptr;
+		void *data;
+		int data_size;
 	} ring[MSM_GPU_MAX_RINGS];
 
 	int nr_registers;
Signature · 6 lines
1.9.1

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
#8 lore raw

Re: [PATCH 5/6] drm/msm/adreno: Add ringbuffer contexts to the GPU state

Thank you for the patch! Yet something to improve:

Hi Jordan,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on robclark/msm-next]
[cannot apply to v4.15 next-20180201]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Jordan-Crouse/drm-msm-GPU-crash-state/20180129-033415
base:   git://people.freedesktop.org/~robclark/linux msm-next
config: arm-allmodconfig (attached as .config)
compiler: arm-linux-gnueabi-gcc (Debian 7.2.0-11) 7.2.0

reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=arm

All errors (new ones prefixed by >>):

>> drivers/gpu/drm/msm/adreno/adreno_gpu.c:20:10: fatal error: linux/ascii85.h: No such file or directory
    #include <linux/ascii85.h>
             ^~~~~~~~~~~~~~~~~

compilation terminated.

vim +20 drivers/gpu/drm/msm/adreno/adreno_gpu.c

  > 20	#include <linux/ascii85.h>
    21	#include <linux/pm_opp.h>
    22	#include "adreno_gpu.h"
    23	#include "msm_gem.h"
    24	#include "msm_mmu.h"
    25	

---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation