Skip to contentSkip to conversation

drm/amdgpu: Implement unrecoverable error message handling for VFs

PATCH In review Ellen Pan 2 participants 2 messages Apr 29, 2025 – May 5, 2025

Subsystems (2)
lore
Share & tools
lore

Ellen Pan

author
#0 lore raw

PATCH 4/4 drm/amdgpu: Implement unrecoverable error message handling for VFs

This notification may arrive in VF mailbox while polling for response from another event.

This patches covers the following scenarios:

- If VF is already in RMA state, then do not attempt to contact the host. Host will ignore the VF after sending the notification.

- If the notification is detected during polling, then set the RMA status, and return error to caller.

- If the notification arrives by interrupt, then set the RMA status and queue a reset. This reset will fail and VF will stop runtime services.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  5 ++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c      | 17 +++++++++--
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h      |  1 +
 drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c      | 34 ++++++++++++++++++++--
 drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h      |  1 +
 5 files changed, 52 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -0
index fe68ba9997ae..0e9ff74d6b6f 100644
@@ -6152,6 +6152,11 @@ static int amdgpu_device_asic_reset(struct amdgpu_device *adev,
 	/* Actual ASIC resets if needed.*/
 	/* Host driver will handle XGMI hive reset for SRIOV */
 	if (amdgpu_sriov_vf(adev)) {
+
+		/* Bail out of reset early */
+		if (amdgpu_ras_is_rma(adev))
+			return -ENODEV;
+
 		if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
 			dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
 			amdgpu_ras_set_fed(adev, true);
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +14 -3
index f2a74aa76b56..48101a34e049 100644
@@ -324,6 +324,7 @@ static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
 				   struct amdgpu_iv_entry *entry)
 {
 	enum idh_event event = xgpu_ai_mailbox_peek_msg(adev);
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
 	switch (event) {
 	case IDH_RAS_BAD_PAGES_NOTIFICATION:
@@ -331,12 +332,22 @@ static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
 		if (amdgpu_sriov_runtime(adev))
 			schedule_work(&adev->virt.bad_pages_work);
 		break;
+	case IDH_UNRECOV_ERR_NOTIFICATION:
+		xgpu_ai_mailbox_send_ack(adev);
+		ras->is_rma = true;
+		dev_err(adev->dev, "VF is in an unrecoverable state. Runtime Services are halted.\n");
+		if (amdgpu_sriov_runtime(adev))
+			WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
+					&adev->virt.flr_work),
+					"Failed to queue work! at %s",
+					__func__);
+		break;
 	case IDH_FLR_NOTIFICATION:
 		if (amdgpu_sriov_runtime(adev))
 			WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
-								&adev->virt.flr_work),
-				  "Failed to queue work! at %s",
-				  __func__);
+						&adev->virt.flr_work),
+					"Failed to queue work! at %s",
+					__func__);
 		break;
 	case IDH_QUERY_ALIVE:
 		xgpu_ai_mailbox_send_ack(adev);
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h +1 -0
index efb452ad1700..874b9f8f9804 100644
@@ -57,6 +57,7 @@ enum idh_event {
 	IDH_RAS_ERROR_DETECTED,
 	IDH_RAS_BAD_PAGES_READY = 15,
 	IDH_RAS_BAD_PAGES_NOTIFICATION = 16,
+	IDH_UNRECOV_ERR_NOTIFICATION = 17,
 	IDH_TEXT_MESSAGE = 255,
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +31 -3
index 74a50c0036ef..32ccb625025b 100644
@@ -67,6 +67,8 @@ static int xgpu_nv_mailbox_rcv_msg(struct amdgpu_device *adev,
 	reg = RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW0);
 	if (reg == IDH_FAIL)
 		r = -EINVAL;
+	if (reg == IDH_UNRECOV_ERR_NOTIFICATION)
+		r = -ENODEV;
 	else if (reg != event)
 		return -ENOENT;
 
@@ -103,6 +105,7 @@ static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event)
 {
 	int r;
 	uint64_t timeout, now;
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
 	now = (uint64_t)ktime_to_ms(ktime_get());
 	timeout = now + NV_MAILBOX_POLL_MSG_TIMEDOUT;
@@ -110,8 +113,16 @@ static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event)
 	do {
 		r = xgpu_nv_mailbox_rcv_msg(adev, event);
 		if (!r) {
-			dev_dbg(adev->dev, "rcv_msg 0x%x after %llu ms\n", event, NV_MAILBOX_POLL_MSG_TIMEDOUT - timeout + now);
+			dev_dbg(adev->dev, "rcv_msg 0x%x after %llu ms\n",
+					event, NV_MAILBOX_POLL_MSG_TIMEDOUT - timeout + now);
 			return 0;
+		} else if (r == -ENODEV) {
+			if (!amdgpu_ras_is_rma(adev)) {
+				ras->is_rma = true;
+				dev_err(adev->dev, "VF is in an unrecoverable state. "
+						"Runtime Services are halted.\n");
+			}
+			return r;
 		}
 
 		msleep(10);
@@ -166,6 +177,10 @@ static int xgpu_nv_send_access_requests_with_param(struct amdgpu_device *adev,
 	enum idh_event event = -1;
 
 send_request:
+
+	if (amdgpu_ras_is_rma(adev))
+		return -ENODEV;
+
 	xgpu_nv_mailbox_trans_msg(adev, req, data1, data2, data3);
 
 	switch (req) {
@@ -323,6 +338,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
 {
 	struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
 	struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
+	struct amdgpu_reset_context reset_context = { 0 };
 
 	amdgpu_virt_fini_data_exchange(adev);
 
@@ -333,8 +349,6 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
 		adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
 		adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
 		adev->video_timeout == MAX_SCHEDULE_TIMEOUT)) {
-		struct amdgpu_reset_context reset_context;
-		memset(&reset_context, 0, sizeof(reset_context));
 
 		reset_context.method = AMD_RESET_METHOD_NONE;
 		reset_context.reset_req_dev = adev;
@@ -380,6 +394,7 @@ static int xgpu_nv_mailbox_rcv_irq(struct amdgpu_device *adev,
 				   struct amdgpu_iv_entry *entry)
 {
 	enum idh_event event = xgpu_nv_mailbox_peek_msg(adev);
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
 	switch (event) {
 	case IDH_RAS_BAD_PAGES_NOTIFICATION:
@@ -387,6 +402,19 @@ static int xgpu_nv_mailbox_rcv_irq(struct amdgpu_device *adev,
 		if (amdgpu_sriov_runtime(adev))
 			schedule_work(&adev->virt.bad_pages_work);
 		break;
+	case IDH_UNRECOV_ERR_NOTIFICATION:
+		xgpu_nv_mailbox_send_ack(adev);
+		if (!amdgpu_ras_is_rma(adev)) {
+			ras->is_rma = true;
+			dev_err(adev->dev, "VF is in an unrecoverable state. Runtime Services are halted.\n");
+		}
+
+		if (amdgpu_sriov_runtime(adev))
+			WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
+						&adev->virt.flr_work),
+					"Failed to queue work! at %s",
+					__func__);
+		break;
 	case IDH_FLR_NOTIFICATION:
 		if (amdgpu_sriov_runtime(adev))
 			WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h +1 -0
index 6d292a537c1b..5808689562cc 100644
@@ -61,6 +61,7 @@ enum idh_event {
 	IDH_RAS_CPER_DUMP_READY = 14,
 	IDH_RAS_BAD_PAGES_READY = 15,
 	IDH_RAS_BAD_PAGES_NOTIFICATION = 16,
+	IDH_UNRECOV_ERR_NOTIFICATION = 17,
 
 	IDH_TEXT_MESSAGE = 255,
 };
Signature · 1 line
2.34.1
Reviewed-by on v1 4/4
#1 lore raw

[AMD Official Use Only - AMD Internal Distribution Only]

Looks good.

Thanks,
Shravan

-----Original Message-----
From: Pan, Ellen <[email protected]>
Sent: Tuesday, April 29, 2025 5:41 PM
To: [email protected]
Cc: Skvortsov, Victor <[email protected]>; Rehman, Ahmad <[email protected]>; Chan, Hing Pong <[email protected]>; Gande, Shravan kumar <[email protected]>; Luo, Zhigang <[email protected]>; Pan, Ellen <[email protected]>; Skvortsov, Victor <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: Implement unrecoverable error message handling for VFs

This notification may arrive in VF mailbox while polling for response from another event.

This patches covers the following scenarios:

- If VF is already in RMA state, then do not attempt to contact the host. Host will ignore the VF after sending the notification.

- If the notification is detected during polling, then set the RMA status, and return error to caller.

- If the notification arrives by interrupt, then set the RMA status and queue a reset. This reset will fail and VF will stop runtime services.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  5 ++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c      | 17 +++++++++--
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h      |  1 +
 drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c      | 34 ++++++++++++++++++++--
 drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h      |  1 +
 5 files changed, 52 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -0
index fe68ba9997ae..0e9ff74d6b6f 100644
@@ -6152,6 +6152,11 @@ static int amdgpu_device_asic_reset(struct amdgpu_device *adev,
        /* Actual ASIC resets if needed.*/
        /* Host driver will handle XGMI hive reset for SRIOV */
        if (amdgpu_sriov_vf(adev)) {
+
+               /* Bail out of reset early */
+               if (amdgpu_ras_is_rma(adev))
+                       return -ENODEV;
+
                if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
                        dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
                        amdgpu_ras_set_fed(adev, true);
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +14 -3
index f2a74aa76b56..48101a34e049 100644
@@ -324,6 +324,7 @@ static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
                                   struct amdgpu_iv_entry *entry)
 {
        enum idh_event event = xgpu_ai_mailbox_peek_msg(adev);
+       struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
        switch (event) {
        case IDH_RAS_BAD_PAGES_NOTIFICATION:
@@ -331,12 +332,22 @@ static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
                if (amdgpu_sriov_runtime(adev))
                        schedule_work(&adev->virt.bad_pages_work);
                break;
+       case IDH_UNRECOV_ERR_NOTIFICATION:
+               xgpu_ai_mailbox_send_ack(adev);
+               ras->is_rma = true;
+               dev_err(adev->dev, "VF is in an unrecoverable state. Runtime Services are halted.\n");
+               if (amdgpu_sriov_runtime(adev))
+                       WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
+                                       &adev->virt.flr_work),
+                                       "Failed to queue work! at %s",
+                                       __func__);
+               break;
        case IDH_FLR_NOTIFICATION:
                if (amdgpu_sriov_runtime(adev))
                        WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
-                                                               &adev->virt.flr_work),
-                                 "Failed to queue work! at %s",
-                                 __func__);
+                                               &adev->virt.flr_work),
+                                       "Failed to queue work! at %s",
+                                       __func__);
                break;
        case IDH_QUERY_ALIVE:
                xgpu_ai_mailbox_send_ack(adev);
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h +1 -0
index efb452ad1700..874b9f8f9804 100644
@@ -57,6 +57,7 @@ enum idh_event {
        IDH_RAS_ERROR_DETECTED,
        IDH_RAS_BAD_PAGES_READY = 15,
        IDH_RAS_BAD_PAGES_NOTIFICATION = 16,
+       IDH_UNRECOV_ERR_NOTIFICATION = 17,
        IDH_TEXT_MESSAGE = 255,
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +3 -0
index 74a50c0036ef..32ccb625025b 100644
@@ -67,6 +67,8 @@ static int xgpu_nv_mailbox_rcv_msg(struct amdgpu_device *adev,
        reg = RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW0);
        if (reg == IDH_FAIL)
                r = -EINVAL;
+       if (reg == IDH_UNRECOV_ERR_NOTIFICATION)
+               r = -ENODEV;
        else if (reg != event)
                return -ENOENT;
 
@@ -103,6 +105,7 @@ static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event)  {
        int r;
        uint64_t timeout, now;
+       struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
        now = (uint64_t)ktime_to_ms(ktime_get());
        timeout = now + NV_MAILBOX_POLL_MSG_TIMEDOUT; @@ -110,8 +113,16 @@ static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event)
        do {
                r = xgpu_nv_mailbox_rcv_msg(adev, event);
                if (!r) {
-                       dev_dbg(adev->dev, "rcv_msg 0x%x after %llu ms\n", event, NV_MAILBOX_POLL_MSG_TIMEDOUT - timeout + now);
+                       dev_dbg(adev->dev, "rcv_msg 0x%x after %llu ms\n",
+                                       event, NV_MAILBOX_POLL_MSG_TIMEDOUT - timeout + now);
                        return 0;

+ } else if (r == -ENODEV) {
+ if (!amdgpu_ras_is_rma(adev)) {
+ ras->is_rma = true;
+ dev_err(adev->dev, "VF is in an unrecoverable state. "
+ "Runtime Services are halted.\n");
+ }

+                       return r;
                }

                msleep(10);
@@ -166,6 +177,10 @@ static int xgpu_nv_send_access_requests_with_param(struct amdgpu_device *adev,
        enum idh_event event = -1;

send_request:
+
+ if (amdgpu_ras_is_rma(adev))
+ return -ENODEV;
+

        xgpu_nv_mailbox_trans_msg(adev, req, data1, data2, data3);

        switch (req) {
@@ -323,6 +338,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)  {
        struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
        struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);

+ struct amdgpu_reset_context reset_context = { 0 };

        amdgpu_virt_fini_data_exchange(adev);

@@ -333,8 +349,6 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)

                adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
                adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
                adev->video_timeout == MAX_SCHEDULE_TIMEOUT)) {

- struct amdgpu_reset_context reset_context;
- memset(&reset_context, 0, sizeof(reset_context));

                reset_context.method = AMD_RESET_METHOD_NONE;
                reset_context.reset_req_dev = adev;
@@ -380,6 +394,7 @@ static int xgpu_nv_mailbox_rcv_irq(struct amdgpu_device *adev,
                                   struct amdgpu_iv_entry *entry)
 {
        enum idh_event event = xgpu_nv_mailbox_peek_msg(adev);

+ struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);

        switch (event) {
        case IDH_RAS_BAD_PAGES_NOTIFICATION:
@@ -387,6 +402,19 @@ static int xgpu_nv_mailbox_rcv_irq(struct amdgpu_device *adev,
                if (amdgpu_sriov_runtime(adev))
                        schedule_work(&adev->virt.bad_pages_work);
                break;

+ case IDH_UNRECOV_ERR_NOTIFICATION:
+ xgpu_nv_mailbox_send_ack(adev);
+ if (!amdgpu_ras_is_rma(adev)) {
+ ras->is_rma = true;
+ dev_err(adev->dev, "VF is in an unrecoverable state. Runtime Services are halted.\n");
+ }
+
+ if (amdgpu_sriov_runtime(adev))
+ WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
+ &adev->virt.flr_work),
+ "Failed to queue work! at %s",
+ __func__);

+               break;
        case IDH_FLR_NOTIFICATION:
                if (amdgpu_sriov_runtime(adev))
                        WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h +1 -0
index 6d292a537c1b..5808689562cc 100644
@@ -61,6 +61,7 @@ enum idh_event {
        IDH_RAS_CPER_DUMP_READY = 14,
        IDH_RAS_BAD_PAGES_READY = 15,
        IDH_RAS_BAD_PAGES_NOTIFICATION = 16,
+       IDH_UNRECOV_ERR_NOTIFICATION = 17,
 
        IDH_TEXT_MESSAGE = 255,
 };

--
2.34.1