Skip to contentSkip to conversation

drm/amdgpu: Log deferred error separately

PATCH In review Candice Li 2 participants 3 messages Jan 10, 2024

Subsystems (3)
lore
Share & tools
lore

Candice Li

author
#0 lore raw

PATCH 1/2 drm/amdgpu: Log deferred error separately

Separate deferred error from UE and CE and log it
individually.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c       |  11 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c       | 116 +++++++++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h       |   6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c       |   1 +
 drivers/gpu/drm/amd/amdgpu/umc_v12_0.c        |  60 ++++-----
 drivers/gpu/drm/amd/amdgpu/umc_v12_0.h        |   3 +
 .../drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c  |   6 +-
 7 files changed, 142 insertions(+), 61 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c +8 -3
index 59fafb8392e0ba..666fd8fa39ad5e 100644
@@ -256,9 +256,14 @@ int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_blo
 		if (type == AMDGPU_MCA_ERROR_TYPE_UE)
 			amdgpu_ras_error_statistic_ue_count(err_data,
 				&mcm_info, &err_addr, (uint64_t)count);
-		else
-			amdgpu_ras_error_statistic_ce_count(err_data,
-				&mcm_info, &err_addr, (uint64_t)count);
+		else {
+			if (!!(MCA_REG__STATUS__DEFERRED(entry->regs[MCA_REG_IDX_STATUS])))
+				amdgpu_ras_error_statistic_de_count(err_data,
+					&mcm_info, &err_addr, (uint64_t)count);
+			else
+				amdgpu_ras_error_statistic_ce_count(err_data,
+					&mcm_info, &err_addr, (uint64_t)count);
+		}
 	}
 
 out_mca_release:
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +96 -20

179 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h +6 -0
index 9edc3909f17d7a..fae53d87578bd7 100644
@@ -476,6 +476,7 @@ struct ras_err_info {
 	struct amdgpu_smuio_mcm_config_info mcm_info;
 	u64 ce_count;
 	u64 ue_count;
+	u64 de_count;
 	struct ras_err_addr err_addr;
 };
 
@@ -487,6 +488,7 @@ struct ras_err_node {
 struct ras_err_data {
 	unsigned long ue_count;
 	unsigned long ce_count;
+	unsigned long de_count;
 	unsigned long err_addr_cnt;
 	struct eeprom_table_record *err_addr;
 	u32 err_list_count;
@@ -562,6 +564,7 @@ struct ras_query_if {
 	struct ras_common_if head;
 	unsigned long ue_count;
 	unsigned long ce_count;
+	unsigned long de_count;
 };
 
 struct ras_inject_if {
@@ -833,5 +836,8 @@ int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
 		struct amdgpu_smuio_mcm_config_info *mcm_info,
 		struct ras_err_addr *err_addr, u64 count);
+int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
+		struct amdgpu_smuio_mcm_config_info *mcm_info,
+		struct ras_err_addr *err_addr, u64 count);
 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances);
 #endif
drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +1 -0
index d65e21914d8c4a..848df7acdd3210 100644
@@ -210,6 +210,7 @@ int amdgpu_umc_poison_handler(struct amdgpu_device *adev, bool reset)
 		if (ret == AMDGPU_RAS_SUCCESS && obj) {
 			obj->err_data.ue_count += err_data.ue_count;
 			obj->err_data.ce_count += err_data.ce_count;
+			obj->err_data.de_count += err_data.de_count;
 		}
 
 		amdgpu_ras_error_data_fini(&err_data);
drivers/gpu/drm/amd/amdgpu/umc_v12_0.c +25 -35

111 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/umc_v12_0.h +3 -0
index e8de3a92251a2c..5973bfb14fceec 100644
@@ -121,9 +121,12 @@
 			(((_ipid_lo) >> 12) & 0xF))
 #define MCA_IPID_LO_2_UMC_INST(_ipid_lo) (((_ipid_lo) >> 21) & 0x7)
 
+bool umc_v12_0_is_deferred_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 bool umc_v12_0_is_uncorrectable_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 bool umc_v12_0_is_correctable_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 
+typedef bool (*check_error_type_func)(struct amdgpu_device *adev, uint64_t mc_umc_status);
+
 extern const uint32_t
 	umc_v12_0_channel_idx_tbl[]
 			[UMC_V12_0_UMC_INSTANCE_NUM]
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c +3 -3
index 4ebc6b421c2cb4..f13f957c8ac195 100644
@@ -2553,9 +2553,9 @@ static int mca_umc_mca_get_err_count(const struct mca_ras_info *mca_ras, struct
 		return 0;
 	}
 
-	if (type == AMDGPU_MCA_ERROR_TYPE_UE && umc_v12_0_is_uncorrectable_error(adev, status0))
-		*count = 1;
-	else if (type == AMDGPU_MCA_ERROR_TYPE_CE && umc_v12_0_is_correctable_error(adev, status0))
+	if ((type == AMDGPU_MCA_ERROR_TYPE_UE && umc_v12_0_is_uncorrectable_error(adev, status0)) ||
+	    (type == AMDGPU_MCA_ERROR_TYPE_CE && (umc_v12_0_is_correctable_error(adev, status0) ||
+	     umc_v12_0_is_deferred_error(adev, status0))))
 		*count = 1;
 
 	return 0;
Signature · 1 line
2.25.1
author
#1 lore raw

PATCH 2/2 drm/amdgpu: Do bad page retirement for deferred errors

Needs to do bad page retirement for deferred errors.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +7 -5
index 848df7acdd3210..df61df7e9b155f 100644
@@ -93,6 +93,7 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	int ret = 0;
+	unsigned long err_count;
 
 	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
 	ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc));
@@ -147,16 +148,17 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
 	}
 
 	/* only uncorrectable error needs gpu reset */
-	if (err_data->ue_count) {
-		dev_info(adev->dev, "%ld uncorrectable hardware errors "
-				"detected in UMC block\n",
-				err_data->ue_count);
+	if (err_data->ue_count || err_data->de_count) {
+		dev_info(adev->dev, "%ld uncorrectable hardware errors and "
+				"%ld deferred hardware errors detected in UMC block\n",
+				err_data->ue_count, err_data->de_count);
 
+		err_count = err_data->ue_count + err_data->de_count;
 		if ((amdgpu_bad_page_threshold != 0) &&
 			err_data->err_addr_cnt) {
 			amdgpu_ras_add_bad_pages(adev, err_data->err_addr,
 						err_data->err_addr_cnt);
-			amdgpu_ras_save_bad_pages(adev, &(err_data->ue_count));
+			amdgpu_ras_save_bad_pages(adev, &err_count);
 
 			amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
 
Signature · 1 line
2.25.1
Reviewed-by on v1 2/2
#2 lore raw

RE: [PATCH 2/2] drm/amdgpu: Do bad page retirement for deferred errors

[AMD Official Use Only - General]

Let's drop the following message.

+ dev_info(adev->dev, "%ld uncorrectable hardware errors and "
+ "%ld deferred hardware errors detected in UMC block\n",
+ err_data->ue_count, err_data->de_count);

With that fixed, the series is

Regards,
Hawking
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Candice Li
Sent: Wednesday, January 10, 2024 16:39
To: [email protected]
Cc: Li, Candice <[email protected]>
Subject: [PATCH 2/2] drm/amdgpu: Do bad page retirement for deferred errors

Needs to do bad page retirement for deferred errors.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +1 -0
index 848df7acdd3210..df61df7e9b155f 100644
@@ -93,6 +93,7 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
        struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
        struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
        int ret = 0;
+       unsigned long err_count;
 
        kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
        ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc)); @@ -147,16 +148,17 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
        }

        /* only uncorrectable error needs gpu reset */
-       if (err_data->ue_count) {
-               dev_info(adev->dev, "%ld uncorrectable hardware errors "
-                               "detected in UMC block\n",
-                               err_data->ue_count);
+       if (err_data->ue_count || err_data->de_count) {

+ dev_info(adev->dev, "%ld uncorrectable hardware errors and "
+ "%ld deferred hardware errors detected in UMC block\n",
+ err_data->ue_count, err_data->de_count);

+               err_count = err_data->ue_count + err_data->de_count;
                if ((amdgpu_bad_page_threshold != 0) &&
                        err_data->err_addr_cnt) {
                        amdgpu_ras_add_bad_pages(adev, err_data->err_addr,
                                                err_data->err_addr_cnt);

- amdgpu_ras_save_bad_pages(adev, &(err_data->ue_count));
+ amdgpu_ras_save_bad_pages(adev, &err_count);

                        amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);

--
2.25.1