Skip to contentSkip to conversation

drm/xe: Use RAS logging for GT fault paths

RFC In review Arvind Yadav 2 participants 3 messages Jul 24

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover drm/xe: Use RAS logging for GT fault paths

This patch is a small follow-up on top of the structured RAS error logging infrastructure series:

https://patchwork.freedesktop.org/series/168333/

That series adds SIG_ID based error reporting and the xe_ras_log_*() helpers. This patch converts a few existing driver-detected GT fault logs to the structured RAS logging path.

Arvind Yadav (1):
drm/xe: Use RAS logging for GT fault paths

 drivers/gpu/drm/xe/xe_guc_submit.c | 35 ++++++++++++++++++------------
 drivers/gpu/drm/xe/xe_pagefault.c  | 10 +++++----
 drivers/gpu/drm/xe/xe_ras_log.h    | 14 ++++++++++++
 drivers/gpu/drm/xe/xe_vm.c         |  4 +++-
 4 files changed, 44 insertions(+), 19 deletions(-)
Signature · 1 line
2.43.0
#1 lore raw

RFC 1/1 drm/xe: Use RAS logging for GT fault paths

Route selected GT fault logs through the structured RAS logging helpers.

Use the GT TDR category for GuC scheduling timeouts, uncorrectable local errors, and engine reset failures, as these paths lead to job invalidation, TDR handling, or GT reset. Use the memory fault category for terminal preempt rebind worker failures, unknown-context GuC Memory CAT errors, and page fault queue overflow.

Add a ratelimited memory fault wrapper for the noisy fault paths.

This keeps these fault reports on the SIG_ID-based RAS logging path while leaving informational and notice-only paths unchanged.

Cc Mallesh Koujalagi <[email protected]>
Cc Badal Nilawar <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_guc_submit.c | 35 ++++++++++++++++++------------
 drivers/gpu/drm/xe/xe_pagefault.c  | 10 +++++----
 drivers/gpu/drm/xe/xe_ras_log.h    | 14 ++++++++++++
 drivers/gpu/drm/xe/xe_vm.c         |  4 +++-
 4 files changed, 44 insertions(+), 19 deletions(-)
drivers/gpu/drm/xe/xe_guc_submit.c +21 -14
index 8aaed4fd13ea..a4467601e931 100644
@@ -40,6 +40,7 @@
 #include "xe_mocs.h"
 #include "xe_module.h"
 #include "xe_pm.h"
+#include "xe_ras_log.h"
 #include "xe_ring_ops_types.h"
 #include "xe_sched_job.h"
 #include "xe_sleep.h"
@@ -1287,7 +1288,8 @@ static void disable_scheduling_deregister(struct xe_guc *guc,
 	if (!ret && !vf_recovery(guc)) {
 		struct xe_gpu_scheduler *sched = &q->guc->sched;
 
-		xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n");
+		xe_ras_log_gt_tdr(guc_to_xe(guc), q->gt, -ETIMEDOUT,
+				  "Pending enable/disable failed to respond");
 		xe_sched_submission_start(sched);
 		xe_gt_reset_async(q->gt);
 		xe_sched_tdr_queue_imm(sched);
@@ -1375,9 +1377,10 @@ static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job)
 	u64 running_time_ms;
 
 	if (!xe_sched_job_started(job)) {
-		xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started",
-			   xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
-			   q->guc->id);
+		xe_ras_log_gt_tdr(gt_to_xe(gt), gt, -ETIMEDOUT,
+				  "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started",
+				  xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
+				  q->guc->id);
 
 		/* GuC never scheduled this job - let the caller trigger a GT reset. */
 		return true;
@@ -1390,9 +1393,10 @@ static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job)
 				     xe_sched_job_seqno(job),
 				     xe_sched_job_lrc_seqno(job), q->guc->id);
 		else
-			xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck",
-				   xe_sched_job_seqno(job),
-				   xe_sched_job_lrc_seqno(job), q->guc->id);
+			xe_ras_log_gt_tdr(gt_to_xe(gt), gt, -ETIMEDOUT,
+					  "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck",
+					  xe_sched_job_seqno(job),
+					  xe_sched_job_lrc_seqno(job), q->guc->id);
 
 		return xe_sched_invalidate_job(job, 0);
 	}
@@ -3278,7 +3282,8 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 		 * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF
 		 * context. In such case only PF will be notified about that fault.
 		 */
-		xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n");
+		xe_ras_log_mem_fault_ratelimited(gt_to_xe(gt), gt, -EIO,
+						 "Memory CAT error reported by GuC!");
 		return 0;
 	}
 
@@ -3320,7 +3325,8 @@ int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len)
 	guc_id = msg[0];
 
 	if (guc_id == GUC_ID_UNKNOWN) {
-		xe_gt_err(gt, "GuC: Uncorrectable local error with unknown GuC id\n");
+		xe_ras_log_gt_tdr(gt_to_xe(gt), gt, -EIO,
+				  "GuC: Uncorrectable local error with unknown GuC id");
 		return 0;
 	}
 
@@ -3328,9 +3334,9 @@ int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len)
 	if (unlikely(!q))
 		return -EPROTO;
 
-	xe_gt_err(gt,
-		  "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x",
-		  guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask);
+	xe_ras_log_gt_tdr(gt_to_xe(gt), gt, -EIO,
+			  "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x",
+			  guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask);
 
 	trace_xe_guc_uncorrectable_error(q);
 
@@ -3354,8 +3360,9 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 le
 	reason = msg[2];
 
 	/* Unexpected failure of a hardware feature, log an actual error */
-	xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X",
-		  guc_class, instance, reason);
+	xe_ras_log_gt_tdr(guc_to_xe(guc), gt, -EIO,
+			  "GuC engine reset request failed on %d:%d because 0x%08X",
+			  guc_class, instance, reason);
 
 	xe_gt_reset_async(gt);
 
drivers/gpu/drm/xe/xe_pagefault.c +6 -4
index dd3c068e1a39..b869d5f679ea 100644
@@ -16,6 +16,7 @@
 #include "xe_hw_engine.h"
 #include "xe_pagefault.h"
 #include "xe_pagefault_types.h"
+#include "xe_ras_log.h"
 #include "xe_svm.h"
 #include "xe_trace_bo.h"
 #include "xe_vm.h"
@@ -486,12 +487,13 @@ int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf)
 		pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) %
 			pf_queue->size;
 		queue_work(xe->usm.pf_wq, &pf_queue->worker);
-	} else {
-		drm_warn(&xe->drm,
-			 "PageFault Queue (%d) full, shouldn't be possible\n",
-			 pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT);
 	}
 	spin_unlock_irqrestore(&pf_queue->lock, flags);
 
+	if (full)
+		xe_ras_log_mem_fault_ratelimited(xe, pf->gt, -ENOSPC,
+						 "PageFault Queue (%d) full, shouldn't be possible",
+						 pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT);
+
 	return full ? -ENOSPC : 0;
 }
drivers/gpu/drm/xe/xe_ras_log.h +14 -0
index 3f94e6747e86..b62478f04bfd 100644
@@ -7,6 +7,7 @@
 #define _XE_RAS_LOG_H_
 
 #include <linux/cper.h>
+#include <linux/ratelimit.h>
 
 #include "xe_sig_ids.h"
 
@@ -25,6 +26,15 @@ void __xe_ras_log(struct xe_device *xe, struct xe_gt *gt,
  * Driver error reporting macros
  */
 
+#define xe_ras_log_ratelimited(_xe, _gt, _sig_id, _cper_sev, _errno, _fmt, ...) \
+	do { \
+		static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL, \
+					      DEFAULT_RATELIMIT_BURST); \
+		if (__ratelimit(&_rs)) \
+			__xe_ras_log((_xe), (_gt), (_sig_id), (_cper_sev), \
+				     (_errno), _fmt, ##__VA_ARGS__); \
+	} while (0)
+
 /* FATAL */
 #define xe_ras_log_probe(xe, errno, fmt, ...) \
 	__xe_ras_log((xe), NULL, XE_SIG_PROBE, CPER_SEV_FATAL, \
@@ -55,6 +65,10 @@ void __xe_ras_log(struct xe_device *xe, struct xe_gt *gt,
 	__xe_ras_log((xe), (gt), XE_SIG_MEM_FAULT, CPER_SEV_RECOVERABLE, \
 		     (errno), fmt, ##__VA_ARGS__)
 
+#define xe_ras_log_mem_fault_ratelimited(_xe, _gt, _errno, _fmt, ...) \
+	xe_ras_log_ratelimited((_xe), (_gt), XE_SIG_MEM_FAULT, \
+			       CPER_SEV_RECOVERABLE, (_errno), _fmt, ##__VA_ARGS__)
+
 #define xe_ras_log_io_bus(xe, errno, fmt, ...) \
 	__xe_ras_log((xe), NULL, XE_SIG_IO_BUS, CPER_SEV_RECOVERABLE, \
 		     (errno), fmt, ##__VA_ARGS__)
drivers/gpu/drm/xe/xe_vm.c +3 -1
index 9e0176861cb6..3f4c015c8d61 100644
@@ -34,6 +34,7 @@
 #include "xe_preempt_fence.h"
 #include "xe_pt.h"
 #include "xe_pxp.h"
+#include "xe_ras_log.h"
 #include "xe_sriov_vf.h"
 #include "xe_svm.h"
 #include "xe_sync.h"
@@ -591,7 +592,8 @@ static void preempt_rebind_work_func(struct work_struct *w)
 	}
 
 	if (err) {
-		drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
+		xe_ras_log_mem_fault(vm->xe, NULL, err,
+				      "Preempt rebind worker failed: err=%d", err);
 		xe_vm_kill(vm, true);
 	}
 	up_write(&vm->lock);
Signature · 1 line
2.43.0
#2 lore raw
In reply to Arvind Yadav

Hi Arvind,

Please use new approach helper function from the patch series: https://patchwork.freedesktop.org/series/171022/

Thanks,

-/Mallesh

Quoted 18 lines · Arvind Yadav

This patch is a small follow-up on top of the structured RAS error logging infrastructure series:

https://patchwork.freedesktop.org/series/168333/

That series adds SIG_ID based error reporting and the xe_ras_log_*() helpers. This patch converts a few existing driver-detected GT fault logs to the structured RAS logging path.

Arvind Yadav (1):
drm/xe: Use RAS logging for GT fault paths

  drivers/gpu/drm/xe/xe_guc_submit.c | 35 ++++++++++++++++++------------
  drivers/gpu/drm/xe/xe_pagefault.c  | 10 +++++----
  drivers/gpu/drm/xe/xe_ras_log.h    | 14 ++++++++++++
  drivers/gpu/drm/xe/xe_vm.c         |  4 +++-
  4 files changed, 44 insertions(+), 19 deletions(-)