Skip to contentSkip to conversation

drm/xe: Wedged state PoC

RFC PATCH New Matthew Brost 2 participants 2 messages Apr 5, 2024

Subsystems (2)
lore
Share & tools
lore

Matthew Brost

author maintainer
#0 lore raw

RFC PATCH v2 1/1 drm/xe: Wedged state PoC

Simple PoC for wedged state based on [1].

Expands on [2] by taking a ref to all exec queues when a jobs times out and stopping all interaction with the GuC. Ref to exec queues dropped on module unload. The idea being if we have ref to the exec queue the state associated exec queue / GuC state can be examined via debugfs, memory debug tools, etc... Jobs are currently canceled and cleanup natrually as they do not hold much state aside from batch address and seqno. This can be revisited if needed.

v2:
- Fix unload after Rodrigo shared script to remove devcore dump

[1] https://patchwork.freedesktop.org/series/131998/
[2] https://patchwork.freedesktop.org/patch/586968/?series=131998&rev=1

---
 drivers/gpu/drm/xe/xe_devcoredump.c  |   2 +-
 drivers/gpu/drm/xe/xe_device.c       |  38 ++++++++++
 drivers/gpu/drm/xe/xe_device.h       |   7 ++
 drivers/gpu/drm/xe/xe_device_types.h |   3 +
 drivers/gpu/drm/xe/xe_exec_queue.h   |   9 +++
 drivers/gpu/drm/xe/xe_gt.c           |   5 +-
 drivers/gpu/drm/xe/xe_guc.c          |  41 +++++------
 drivers/gpu/drm/xe/xe_guc_ads.c      |   9 ++-
 drivers/gpu/drm/xe/xe_guc_ct.c       |   2 +-
 drivers/gpu/drm/xe/xe_guc_pc.c       |   3 +
 drivers/gpu/drm/xe/xe_guc_submit.c   | 105 +++++++++++++++++----------
 drivers/gpu/drm/xe/xe_guc_submit.h   |   4 +-
 drivers/gpu/drm/xe/xe_module.c       |   5 ++
 drivers/gpu/drm/xe/xe_module.h       |   1 +
 14 files changed, 167 insertions(+), 67 deletions(-)
drivers/gpu/drm/xe/xe_devcoredump.c +1 -1
index a951043b2943..283ca7518aff 100644
@@ -188,7 +188,7 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
 
 	coredump->snapshot.ct = xe_guc_ct_snapshot_capture(&guc->ct, true);
-	coredump->snapshot.ge = xe_guc_exec_queue_snapshot_capture(job);
+	coredump->snapshot.ge = xe_guc_exec_queue_snapshot_capture(q);
 	coredump->snapshot.job = xe_sched_job_snapshot_capture(job);
 	coredump->snapshot.vm = xe_vm_snapshot_capture(q->vm);
 
drivers/gpu/drm/xe/xe_device.c +38 -0
index 9083f5e02dd9..7928a5470cee 100644
@@ -142,6 +142,9 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
 	long ret;
 
+	if (xe_device_wedged(xe))
+		return -ECANCELED;
+
 	ret = xe_pm_runtime_get_ioctl(xe);
 	if (ret >= 0)
 		ret = drm_ioctl(file, cmd, arg);
@@ -157,6 +160,9 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo
 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
 	long ret;
 
+	if (xe_device_wedged(xe))
+		return -ECANCELED;
+
 	ret = xe_pm_runtime_get_ioctl(xe);
 	if (ret >= 0)
 		ret = drm_compat_ioctl(file, cmd, arg);
@@ -779,3 +785,35 @@ u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
 {
 	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
 }
+
+/**
+ * xe_device_declare_wedged - Declare device wedged
+ * @xe: xe device instance
+ *
+ * This is a final state that can only be cleared with a module
+ * re-probe (unbind + bind).
+ * In this state every IOCTL will be blocked so the GT cannot be used.
+ * In general it will be called upon any critical error such as gt reset
+ * failure or guc loading failure.
+ * If xe.wedged module parameter is set to 2, this function will be called
+ * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
+ * snapshot capture. In this mode, GT reset won't be attempted so the state of
+ * the issue is preserved for further debugging.
+ */
+void xe_device_declare_wedged(struct xe_device *xe)
+{
+	if (xe_modparam.wedged_mode == 0)
+		return;
+
+	if (!atomic_xchg(&xe->wedged, 1)) {
+		xe->needs_flr_on_fini = true;
+		drm_err(&xe->drm,
+			"CRITICAL: Xe has declared device %s as wedged.\n"
+			"IOCTLs and executions are blocked until device is probed again with unbind and bind operations:\n"
+			"echo '%s' | sudo tee /sys/bus/pci/drivers/xe/unbind\n"
+			"echo '%s' | sudo tee /sys/bus/pci/drivers/xe/bind\n"
+			"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
+			dev_name(xe->drm.dev), dev_name(xe->drm.dev),
+			dev_name(xe->drm.dev));
+	}
+}
drivers/gpu/drm/xe/xe_device.h +7 -0
index d413bc2c6be5..0fea5c18f76d 100644
@@ -176,4 +176,11 @@ void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p);
 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address);
 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);
 
+static inline bool xe_device_wedged(struct xe_device *xe)
+{
+	return atomic_read(&xe->wedged);
+}
+
+void xe_device_declare_wedged(struct xe_device *xe);
+
 #endif
drivers/gpu/drm/xe/xe_device_types.h +3 -0
index c710cec835a7..290fdebaffe1 100644
@@ -458,6 +458,9 @@ struct xe_device {
 	/** @needs_flr_on_fini: requests function-reset on fini */
 	bool needs_flr_on_fini;
 
+	/** @wedged: Xe device faced a critical error and is now blocked. */
+	atomic_t wedged;
+
 	/* private: */
 
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
drivers/gpu/drm/xe/xe_exec_queue.h +9 -0
index 02ce8d204622..c6e7f0b9037b 100644
@@ -28,6 +28,15 @@ void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance);
 
 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id);
 
+static inline struct xe_exec_queue *
+xe_exec_queue_get_unless_zero(struct xe_exec_queue *q)
+{
+	if (kref_get_unless_zero(&q->refcount))
+		return q;
+
+	return NULL;
+}
+
 static inline struct xe_exec_queue *xe_exec_queue_get(struct xe_exec_queue *q)
 {
 	kref_get(&q->refcount);
drivers/gpu/drm/xe/xe_gt.c +4 -1
index cfa5da900461..0844081b88ef 100644
@@ -633,6 +633,9 @@ static int gt_reset(struct xe_gt *gt)
 {
 	int err;
 
+	if (xe_device_wedged(gt_to_xe(gt)))
+		return -ECANCELED;
+
 	/* We only support GT resets with GuC submission */
 	if (!xe_device_uc_enabled(gt_to_xe(gt)))
 		return -ENODEV;
@@ -685,7 +688,7 @@ static int gt_reset(struct xe_gt *gt)
 err_fail:
 	xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
 
-	gt_to_xe(gt)->needs_flr_on_fini = true;
+	xe_device_declare_wedged(gt_to_xe(gt));
 
 	return err;
 }
drivers/gpu/drm/xe/xe_guc.c +18 -23

72 lines not on this page · raw message

drivers/gpu/drm/xe/xe_guc_ads.c +8 -1
index e025f3e10c9b..37f30c333c93 100644
@@ -18,6 +18,7 @@
 #include "xe_lrc.h"
 #include "xe_map.h"
 #include "xe_mmio.h"
+#include "xe_module.h"
 #include "xe_platform_types.h"
 
 /* Slack of a few additional entries per engine */
@@ -313,11 +314,17 @@ int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads)
 
 static void guc_policies_init(struct xe_guc_ads *ads)
 {
+	u32 global_flags = 0;
+
 	ads_blob_write(ads, policies.dpc_promote_time,
 		       GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
 	ads_blob_write(ads, policies.max_num_work_items,
 		       GLOBAL_POLICY_MAX_NUM_WI);
-	ads_blob_write(ads, policies.global_flags, 0);
+
+	if (xe_modparam.wedged_mode == 2)
+		global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
+
+	ads_blob_write(ads, policies.global_flags, global_flags);
 	ads_blob_write(ads, policies.is_valid, 1);
 }
 
drivers/gpu/drm/xe/xe_guc_ct.c +1 -1
index 6c37f4f9bddd..0aa3abaca66d 100644
@@ -1403,7 +1403,7 @@ struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct,
 		return NULL;
 	}
 
-	if (xe_guc_ct_enabled(ct)) {
+	if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
 		snapshot->ct_enabled = true;
 		snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
 		guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g,
drivers/gpu/drm/xe/xe_guc_pc.c +3 -0
index 521ae24f2314..f4663f1b0a80 100644
@@ -902,6 +902,9 @@ static void xe_guc_pc_fini(struct drm_device *drm, void *arg)
 		return;
 	}
 
+	if (xe_device_wedged(xe))
+		return;
+
 	XE_WARN_ON(xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL));
 	XE_WARN_ON(xe_guc_pc_gucrc_disable(pc));
 	XE_WARN_ON(xe_guc_pc_stop(pc));
drivers/gpu/drm/xe/xe_guc_submit.c +67 -38

255 lines not on this page · raw message

drivers/gpu/drm/xe/xe_guc_submit.h +2 -2
index 2f14dfd04722..fad0421ead36 100644
@@ -9,8 +9,8 @@
 #include <linux/types.h>
 
 struct drm_printer;
+struct xe_exec_queue;
 struct xe_guc;
-struct xe_sched_job;
 
 int xe_guc_submit_init(struct xe_guc *guc);
 
@@ -27,7 +27,7 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len);
 
 struct xe_guc_submit_exec_queue_snapshot *
-xe_guc_exec_queue_snapshot_capture(struct xe_sched_job *job);
+xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q);
 void
 xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot);
 void
drivers/gpu/drm/xe/xe_module.c +5 -0
index 110b69864656..5e023df0bea9 100644
@@ -17,6 +17,7 @@ struct xe_modparam xe_modparam = {
 	.enable_display = true,
 	.guc_log_level = 5,
 	.force_probe = CONFIG_DRM_XE_FORCE_PROBE,
+	.wedged_mode = 1,
 	/* the rest are 0 by default */
 };
 
@@ -48,6 +49,10 @@ module_param_named_unsafe(force_probe, xe_modparam.force_probe, charp, 0400);
 MODULE_PARM_DESC(force_probe,
 		 "Force probe options for specified devices. See CONFIG_DRM_XE_FORCE_PROBE for details.");
 
+module_param_named_unsafe(wedged_mode, xe_modparam.wedged_mode, int, 0600);
+MODULE_PARM_DESC(wedged_mode,
+		 "Module's default policy for the wedged mode - 0=never, 1=upon-critical-errors[default], 2=upon-any-hang");
+
 struct init_funcs {
 	int (*init)(void);
 	void (*exit)(void);
drivers/gpu/drm/xe/xe_module.h +1 -0
index 88ef0e8b2bfd..bc6f370c9a8e 100644
@@ -18,6 +18,7 @@ struct xe_modparam {
 	char *huc_firmware_path;
 	char *gsc_firmware_path;
 	char *force_probe;
+	int wedged_mode;
 };
 
 extern struct xe_modparam xe_modparam;
Signature · 1 line
2.34.1
#1 lore raw

✗ CI.Patch_applied: failure for drm/xe: Wedged state PoC (rev2)

== Series Details ==

== Series Details ==

Series: drm/xe: Wedged state PoC (rev2)
URL : https://patchwork.freedesktop.org/series/132099/
State : failure

== Summary ==

=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: 11155f150a41 drm-tip: 2024y-04m-05d-18h-03m-49s UTC integration manifest === git am output follows ===
error: patch failed: drivers/gpu/drm/xe/xe_guc.c:452
error: drivers/gpu/drm/xe/xe_guc.c: patch does not apply
hint: Use 'git am --show-current-patch=diff' to see the failed patch
Applying: drm/xe: Wedged state PoC
Patch failed at 0001 drm/xe: Wedged state PoC
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".