Skip to contentSkip to conversation

drm/i915: Print dmesg warn on unintended hangs

RFC PATCH In review Mika Kuoppala 4 participants 9 messages Nov 10, 2017

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 drm/i915: Print dmesg warn on unintended hangs

We have a problem of distinguishing intended hangs submitted by igt during CI/bat and hangs that are nonintended happening in close proximity.

As we know how igt constructs a batch intended to hang the gpu, we can use this in our advantage when error state is constructed. The signature of a intended hang can be parsed out in this stage.

Batches that are expected to hang can be watermarked by:

batch[i++] = MI_BATCH_BUFFER_END;
batch[i++] = MI_NOOP;
batch[i++] = IGT_HANG_SIGNATURE; /* 0xc5c5c5c5 */

Note that we do the parsing only with gem debug builds.

Later when we spew out notification about the hang into the dmesg, we can use DRM_INFO for intended hangs and DRM_WARN for nonintended. This way auxiliary tools can make better judgement on what to consider important for reporting.

Cc Chris Wilson <[email protected]>
Cc Daniel Vetter <[email protected]>
Cc Joonas Lahtinen <[email protected]>
---
 drivers/gpu/drm/i915/i915_debugfs.c              |   2 +-
 drivers/gpu/drm/i915/i915_drv.h                  |   6 +-
 drivers/gpu/drm/i915/i915_gpu_error.c            | 100 +++++++++++++++++++++--
 drivers/gpu/drm/i915/i915_irq.c                  |   4 +-
 drivers/gpu/drm/i915/intel_hangcheck.c           |   6 +-
 drivers/gpu/drm/i915/selftests/intel_hangcheck.c |   2 +-
 6 files changed, 104 insertions(+), 16 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
index 39883cd915db..678f472a8956 100644
@@ -4092,7 +4092,7 @@ i915_wedged_set(void *data, u64 val)
 		engine->hangcheck.stalled = true;
 	}
 
-	i915_handle_error(i915, val, "Manually setting wedged to %llu", val);
+	i915_handle_error(i915, true, val, "Manually setting wedged to %llu", val);
 
 	wait_on_bit(&i915->gpu_error.flags,
 		    I915_RESET_HANDOFF,
drivers/gpu/drm/i915/i915_drv.h +5 -1
index fe93115c4caa..91e0ab8509d2 100644
@@ -1004,6 +1004,8 @@ struct i915_gpu_state {
 			u32 *pages[0];
 		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
 
+		bool batch_had_hang_signature;
+
 		struct drm_i915_error_object **user_bo;
 		long user_bo_count;
 
@@ -3372,8 +3374,9 @@ static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv)
 			   &dev_priv->gpu_error.hangcheck_work, delay);
 }
 
-__printf(3, 4)
+__printf(4, 5)
 void i915_handle_error(struct drm_i915_private *dev_priv,
+		       bool intentional,
 		       u32 engine_mask,
 		       const char *fmt, ...);
 
@@ -3960,6 +3963,7 @@ static inline void i915_error_state_buf_release(
 
 struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
 void i915_capture_error_state(struct drm_i915_private *dev_priv,
+			      bool intentional,
 			      u32 engine_mask,
 			      const char *error_msg);
 
drivers/gpu/drm/i915/i915_gpu_error.c +91 -9

175 lines not on this page · raw message

drivers/gpu/drm/i915/i915_irq.c +3 -1
index ff00e462697a..75f519f910cc 100644
@@ -2828,6 +2828,7 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
 /**
  * i915_handle_error - handle a gpu error
  * @dev_priv: i915 device private
+ * @intentional: error was intentionally injected
  * @engine_mask: mask representing engines that are hung
  * @fmt: Error message format string
  *
@@ -2838,6 +2839,7 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
  * of a ring dump etc.).
  */
 void i915_handle_error(struct drm_i915_private *dev_priv,
+		       bool intentional,
 		       u32 engine_mask,
 		       const char *fmt, ...)
 {
@@ -2859,7 +2861,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 	 */
 	intel_runtime_pm_get(dev_priv);
 
-	i915_capture_error_state(dev_priv, engine_mask, error_msg);
+	i915_capture_error_state(dev_priv, intentional, engine_mask, error_msg);
 	i915_clear_error_registers(dev_priv);
 
 	/*
drivers/gpu/drm/i915/intel_hangcheck.c +3 -3
index 12ac270a5f93..7b9e8dec7d65 100644
@@ -266,7 +266,7 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 	 */
 	tmp = I915_READ_CTL(engine);
 	if (tmp & RING_WAIT) {
-		i915_handle_error(dev_priv, 0,
+		i915_handle_error(dev_priv, false, 0,
 				  "Kicking stuck wait on %s",
 				  engine->name);
 		I915_WRITE_CTL(engine, tmp);
@@ -278,7 +278,7 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 		default:
 			return ENGINE_DEAD;
 		case 1:
-			i915_handle_error(dev_priv, 0,
+			i915_handle_error(dev_priv, false, 0,
 					  "Kicking stuck semaphore on %s",
 					  engine->name);
 			I915_WRITE_CTL(engine, tmp);
@@ -407,7 +407,7 @@ static void hangcheck_declare_hang(struct drm_i915_private *i915,
 				 "%s, ", engine->name);
 	msg[len-2] = '\0';
 
-	return i915_handle_error(i915, hung, "%s", msg);
+	return i915_handle_error(i915, false, hung, "%s", msg);
 }
 
 /*
drivers/gpu/drm/i915/selftests/intel_hangcheck.c +1 -1
index 71ce06680d66..7900bba9fe16 100644
@@ -846,7 +846,7 @@ static int igt_handle_error(void *arg)
 	engine->hangcheck.stalled = true;
 	engine->hangcheck.seqno = intel_engine_get_seqno(engine);
 
-	i915_handle_error(i915, intel_engine_flag(engine), "%s", __func__);
+	i915_handle_error(i915, true, intel_engine_flag(engine), "%s", __func__);
 
 	xchg(&i915->gpu_error.first_error, error);
 
Signature · 6 lines
2.11.0

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#2 lore raw
Quoted 6 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 11:53:47)

Quoted 3 lines

We have a problem of distinguishing intended hangs submitted by igt during CI/bat and hangs that are nonintended happening in close proximity.

Do we? I haven't had that problem in distinguishing them.

Piglit can't tell them apart afaik. Due to info level.

-Mika

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#3 lore raw

Quoting Mika Kuoppala (2017-11-10 12:20:55)

Quoted 10 lines
Quoted 6 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 11:53:47)

Quoted 3 lines

We have a problem of distinguishing intended hangs submitted by igt during CI/bat and hangs that are nonintended happening in close proximity.

Do we? I haven't had that problem in distinguishing them.

Piglit can't tell them apart afaik. Due to info level.

Piglit? If the test passes, it doesn't matter how the kernel got there, the user behaviour is as expected. If the test wants to assert that it didn't hang, it can do that.

I am very opposed to interpreting magic contents within the kernel, more so when it is already known by userspace.
-Chris
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#4 lore raw
Quoted 8 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 12:20:55)

Quoted 7 lines
Quoted 5 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 11:53:47)

Quoted 3 lines

We have a problem of distinguishing intended hangs submitted by igt during CI/bat and hangs that are nonintended happening in close proximity.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

Do we? I haven't had that problem in distinguishing them.

Piglit can't tell them apart afaik. Due to info level.

Piglit? If the test passes, it doesn't matter how the kernel got there, the user behaviour is as expected. If the test wants to assert that it didn't hang, it can do that.

Through reset counts? At starters we could assert in framework that all tests that do not call igt_hang() expect reset count to stay the same between entry/exit.

I see the logic behind that user behaviour is as expected.

Would be good that CI folks chime in here and detail how they want things to work.

-Mika
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#5 lore raw
Quoted 18 lines · Mika Kuoppala
Quoted 8 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 12:20:55)

Quoted 7 lines
Quoted 5 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 11:53:47)

Quoted 3 lines

We have a problem of distinguishing intended hangs submitted by igt during CI/bat and hangs that are nonintended happening in close proximity.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

Do we? I haven't had that problem in distinguishing them.

Piglit can't tell them apart afaik. Due to info level.

Piglit? If the test passes, it doesn't matter how the kernel got there, the user behaviour is as expected. If the test wants to assert that it didn't hang, it can do that.

Quoted 8 lines

Through reset counts? At starters we could assert in framework that all tests that do not call igt_hang() expect reset count to stay the same between entry/exit.

I see the logic behind that user behaviour is as expected.

Would be good that CI folks chime in here and detail how they want things to work.

I'm very vary of having to sprinkle that all over CI tbh, but if it's in the framework I guess it can work too. Will be fun to figure out how to catch unintended hangs in the tests that do provoke hangs, but should be doable.

But for adding it to the framework I think we're already putting way too much random quiescent stuff in there, and for generic kms tests there's kinda no need for that. So not entirely sold that this is the best approach we can do.

A semi-middleground would be if we have new functions that open a gem fd for rendering, and we have some sanity-checks to make sure that only when you ask for rendering do the igt ioctl wrappers allow you to. Then we could stuff all these checks in there.

But that still leaves the issue that a gpu hang on e.g. a s/r test or module reload won't be caught, and we really want to catch these.

Module reload btw is also one case where just checking the reset counter will just not work. And module reload is exactly one of these cases where we do want to make sure we don't misprogram the gpu so it dies. -Daniel

Signature · 7 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#6 lore raw
In reply to Chris Wilson

Quoting Chris Wilson (2017-11-10 12:30:21)

Quoted 10 lines

Quoting Mika Kuoppala (2017-11-10 12:20:55)

Quoted 9 lines
Quoted 6 lines · Chris Wilson

Quoting Mika Kuoppala (2017-11-10 11:53:47)

Quoted 3 lines

We have a problem of distinguishing intended hangs submitted by igt during CI/bat and hangs that are nonintended happening in close proximity.

Do we? I haven't had that problem in distinguishing them.

Quoted 8 lines
Quoted 1 line

Piglit can't tell them apart afaik. Due to info level.

Piglit? If the test passes, it doesn't matter how the kernel got there, the user behaviour is as expected. If the test wants to assert that it didn't hang, it can do that.

I am very opposed to interpreting magic contents within the kernel, more so when it is already known by userspace.

A clearer form of identification would be either a context-param saying that userspace is going to use this context for injecting hang, which we already have and use, or for fine grained precision having an execbuf flag (copying onto the request) saying that this is the hanging batch.

But that only makes sense (to me) in the context of gem_reset_stats, drv_hangman, gem_exec_capture or whichever, where you want such fine grained checking. And as some point, we acknowledge we are checking kernel internals (not user behaviour) and move it to the selftests. -Chris
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#7 lore raw

✓ Fi.CI.BAT: success for drm/i915: Print dmesg warn on unintended hangs

In reply to Mika Kuoppala
== Series Details ==

== Series Details ==

Series: drm/i915: Print dmesg warn on unintended hangs
URL : https://patchwork.freedesktop.org/series/33597/
State : success

== Summary ==

Series 33597v1 drm/i915: Print dmesg warn on unintended hangs
https://patchwork.freedesktop.org/api/1.0/series/33597/revisions/1/mbox/

fi-bdw-5557u     total:289  pass:268  dwarn:0   dfail:0   fail:0   skip:21  time:444s
fi-bdw-gvtdvm    total:289  pass:265  dwarn:0   dfail:0   fail:0   skip:24  time:454s
fi-blb-e6850     total:289  pass:223  dwarn:1   dfail:0   fail:0   skip:65  time:382s
fi-bsw-n3050     total:289  pass:243  dwarn:0   dfail:0   fail:0   skip:46  time:540s
fi-bwr-2160      total:289  pass:183  dwarn:0   dfail:0   fail:0   skip:106 time:274s
fi-bxt-dsi       total:289  pass:259  dwarn:0   dfail:0   fail:0   skip:30  time:498s
fi-bxt-j4205     total:289  pass:260  dwarn:0   dfail:0   fail:0   skip:29  time:505s
fi-byt-j1900     total:289  pass:254  dwarn:0   dfail:0   fail:0   skip:35  time:498s
fi-byt-n2820     total:289  pass:250  dwarn:0   dfail:0   fail:0   skip:39  time:499s
fi-elk-e7500     total:289  pass:229  dwarn:0   dfail:0   fail:0   skip:60  time:427s
fi-gdg-551       total:289  pass:178  dwarn:1   dfail:0   fail:1   skip:109 time:259s
fi-glk-1         total:289  pass:261  dwarn:0   dfail:0   fail:0   skip:28  time:545s
fi-hsw-4770      total:289  pass:262  dwarn:0   dfail:0   fail:0   skip:27  time:430s
fi-hsw-4770r     total:289  pass:262  dwarn:0   dfail:0   fail:0   skip:27  time:440s
fi-ilk-650       total:289  pass:228  dwarn:0   dfail:0   fail:0   skip:61  time:427s
fi-ivb-3520m     total:289  pass:260  dwarn:0   dfail:0   fail:0   skip:29  time:482s
fi-ivb-3770      total:289  pass:260  dwarn:0   dfail:0   fail:0   skip:29  time:460s
fi-kbl-7500u     total:289  pass:264  dwarn:1   dfail:0   fail:0   skip:24  time:481s
fi-kbl-7560u     total:289  pass:270  dwarn:0   dfail:0   fail:0   skip:19  time:521s
fi-kbl-7567u     total:289  pass:269  dwarn:0   dfail:0   fail:0   skip:20  time:473s
fi-kbl-r         total:289  pass:262  dwarn:0   dfail:0   fail:0   skip:27  time:530s
fi-pnv-d510      total:289  pass:222  dwarn:1   dfail:0   fail:0   skip:66  time:574s
fi-skl-6260u     total:289  pass:269  dwarn:0   dfail:0   fail:0   skip:20  time:466s
fi-skl-6600u     total:289  pass:262  dwarn:0   dfail:0   fail:0   skip:27  time:546s
fi-skl-6700hq    total:289  pass:263  dwarn:0   dfail:0   fail:0   skip:26  time:565s
fi-skl-6700k     total:289  pass:265  dwarn:0   dfail:0   fail:0   skip:24  time:522s
fi-skl-6770hq    total:289  pass:269  dwarn:0   dfail:0   fail:0   skip:20  time:499s
fi-skl-gvtdvm    total:289  pass:266  dwarn:0   dfail:0   fail:0   skip:23  time:462s
fi-snb-2520m     total:289  pass:250  dwarn:0   dfail:0   fail:0   skip:39  time:559s
fi-snb-2600      total:289  pass:249  dwarn:0   dfail:0   fail:0   skip:40  time:420s
Blacklisted hosts:
fi-cfl-s         total:289  pass:254  dwarn:3   dfail:0   fail:0   skip:32  time:529s
fi-glk-dsi       total:289  pass:259  dwarn:0   dfail:0   fail:0   skip:30  time:489s

fi-cnl-y failed to connect after reboot

c36994b067f0771eee4d3136f40c0c6040a8fa77 drm-tip: 2017y-11m-10d-11h-48m-19s UTC integration manifest b4aa111288af drm/i915: Print dmesg warn on unintended hangs

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_7057/
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#8 lore raw

✓ Fi.CI.IGT: success for drm/i915: Print dmesg warn on unintended hangs

In reply to Mika Kuoppala
== Series Details ==

== Series Details ==

Series: drm/i915: Print dmesg warn on unintended hangs
URL : https://patchwork.freedesktop.org/series/33597/
State : success

== Summary ==

Test kms_flip:

        Subgroup plain-flip-ts-check:
                pass       -> FAIL       (shard-hsw) fdo#100368
        Subgroup dpms-vs-vblank-race-interruptible:
                pass       -> FAIL       (shard-hsw) fdo#103060 +1
Test kms_setmode:
        Subgroup basic:
                fail       -> PASS       (shard-hsw) fdo#99912
Test kms_busy:
        Subgroup extended-modeset-hang-oldfb-with-reset-render-c:
                dmesg-warn -> PASS       (shard-hsw) fdo#102249

fdo#100368 https://bugs.freedesktop.org/show_bug.cgi?id=100368
fdo#103060 https://bugs.freedesktop.org/show_bug.cgi?id=103060
fdo#99912 https://bugs.freedesktop.org/show_bug.cgi?id=99912
fdo#102249 https://bugs.freedesktop.org/show_bug.cgi?id=102249

shard-hsw total:2584 pass:1451 dwarn:1 dfail:2 fail:12 skip:1118 time:9354s
Blacklisted hosts:

shard-apl        total:2511 pass:1553 dwarn:3   dfail:2   fail:19  skip:934 time:12317s
shard-kbl        total:2534 pass:1654 dwarn:20  dfail:4   fail:24  skip:830 time:9681s
shard-snb        total:2584 pass:1191 dwarn:1   dfail:2   fail:12  skip:1378 time:7764s

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_7057/shards.html
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx