Skip to contentSkip to conversation

drm/amdgpu: Register bad page handler for Aldebaran

PATCH In review Mukul Joshi 4 participants 21 messages May 12, 2021 – Sep 13, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: Register bad page handler for Aldebaran

On Aldebaran, GPU driver will handle bad page retirement even though UMC is host managed. As a result, register a bad page retirement handler on the mce notifier chain to retire bad pages on Aldebaran.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 154 ++++++++++++++++++++++++
 1 file changed, 154 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +154 -0
index b1c57a5b6e89..02263f509b36 100644
@@ -34,7 +34,9 @@
 #include "amdgpu_xgmi.h"
 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
 #include "atom.h"
+#include <asm/mce.h>
 
+static bool notifier_registered;
 static const char *RAS_FS_NAME = "ras";
 
 const char *ras_error_string[] = {
@@ -73,6 +75,11 @@ const char *ras_block_string[] = {
 /* typical ECC bad page rate(1 bad page per 100MB VRAM) */
 #define RAS_BAD_PAGE_RATE		(100 * 1024 * 1024ULL)
 
+#define GET_MCA_IPID_GPUID(m)		(((m) >> 44) & 0xF)
+#define GET_UMC_INST_NIBBLE(m)		(((m) >> 20) & 0xF)
+#define GET_CHAN_INDEX_NIBBLE(m)	(((m) >> 12) & 0xF)
+#define GPU_ID_OFFSET			8
+
 enum amdgpu_ras_retire_page_reservation {
 	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
 	AMDGPU_RAS_RETIRE_PAGE_PENDING,
@@ -85,6 +92,7 @@ static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
 				uint64_t addr);
 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
 				uint64_t addr);
+static void amdgpu_register_bad_pages_mca_notifier(void);
 
 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
 {
@@ -1978,6 +1986,10 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
 			goto free;
 	}
 
+	if ((adev->asic_type == CHIP_ALDEBARAN) &&
+	    (adev->gmc.xgmi.connected_to_cpu))
+		amdgpu_register_bad_pages_mca_notifier();
+
 	return 0;
 
 free:
@@ -2427,3 +2439,145 @@ void amdgpu_release_ras_context(struct amdgpu_device *adev)
 		kfree(con);
 	}
 }
+
+static struct amdgpu_device *find_adev(uint32_t node_id)
+{
+	struct amdgpu_gpu_instance *gpu_instance;
+	int i;
+	struct amdgpu_device *adev = NULL;
+
+	mutex_lock(&mgpu_info.mutex);
+
+	for (i = 0; i < mgpu_info.num_gpu; i++) {
+		gpu_instance = &(mgpu_info.gpu_ins[i]);
+		adev = gpu_instance->adev;
+
+		if (adev->gmc.xgmi.connected_to_cpu &&
+		    adev->gmc.xgmi.physical_node_id == node_id)
+			break;
+		adev = NULL;
+	}
+
+	mutex_unlock(&mgpu_info.mutex);
+
+	return adev;
+}
+
+static void find_umc_inst_chan_index(struct mce *m, uint32_t *umc_inst,
+				     uint32_t *chan_index)
+{
+	uint32_t val1 = 0;
+	uint32_t val2 = 0;
+	uint32_t rem = 0;
+
+	/*
+	 * Bit 20-23 provides the UMC instance nibble.
+	 * Bit 12-15 provides the channel index nibble.
+	 */
+	val1 = GET_UMC_INST_NIBBLE(m->ipid);
+	val2 = GET_CHAN_INDEX_NIBBLE(m->ipid);
+
+	*umc_inst = val1/2;
+	rem = val1%2;
+
+	*chan_index = (4*rem) + val2;
+}
+
+static int amdgpu_bad_page_notifier(struct notifier_block *nb,
+				    unsigned long val, void *data)
+{
+	struct mce *m = (struct mce *)data;
+	struct amdgpu_device *adev = NULL;
+	uint32_t gpu_id = 0;
+	uint32_t umc_inst = 0;
+	uint32_t chan_index = 0;
+	struct ras_err_data err_data = {0, 0, 0, NULL};
+	struct eeprom_table_record err_rec;
+	uint64_t retired_page;
+
+	/*
+	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
+	 * and error occurred in DramECC (Extended error code = 0) then only
+	 * process the error, else bail out.
+	 */
+	if (!m || !(is_smca_umc_v2(m->bank) && (XEC(m->status, 0x1f) == 0x0)))
+		return NOTIFY_DONE;
+
+	gpu_id = GET_MCA_IPID_GPUID(m->ipid);
+
+	/*
+	 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
+	 */
+	gpu_id -= GPU_ID_OFFSET;
+
+	adev = find_adev(gpu_id);
+	if (!adev) {
+		dev_warn(adev->dev, "%s: Unable to find adev for gpu_id: %d\n",
+				     __func__, gpu_id);
+		return NOTIFY_DONE;
+	}
+
+	/*
+	 * If it is correctable error, then print a message and return.
+	 */
+	if (mce_is_correctable(m)) {
+		dev_info(adev->dev, "%s: UMC Correctable error detected.",
+				    __func__);
+		return NOTIFY_OK;
+	}
+
+	/*
+	 * If it is uncorrectable error, then find out UMC instance and
+	 * channel index.
+	 */
+	find_umc_inst_chan_index(m, &umc_inst, &chan_index);
+
+	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d,"
+			    " chan_idx: %d", umc_inst, chan_index);
+
+	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
+
+	/*
+	 * Translate UMC channel address to Physical address
+	 */
+	retired_page = ADDR_OF_8KB_BLOCK(m->addr) |
+			ADDR_OF_256B_BLOCK(chan_index) |
+			OFFSET_IN_256B_BLOCK(m->addr);
+
+	err_rec.address = m->addr;
+	err_rec.retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
+	err_rec.ts = (uint64_t)ktime_get_real_seconds();
+	err_rec.err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
+	err_rec.cu = 0;
+	err_rec.mem_channel = chan_index;
+	err_rec.mcumc_id = umc_inst;
+
+	err_data.err_addr = &err_rec;
+	err_data.err_addr_cnt = 1;
+
+	if (amdgpu_bad_page_threshold != 0) {
+		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
+						err_data.err_addr_cnt);
+		amdgpu_ras_save_bad_pages(adev);
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block amdgpu_bad_page_nb = {
+	.notifier_call  = amdgpu_bad_page_notifier,
+	.priority       = MCE_PRIO_ACCEL,
+};
+
+static void amdgpu_register_bad_pages_mca_notifier(void)
+{
+	/*
+	 * Register the x86 notifier with MCE subsystem.
+	 * Please note a notifier can be registered only once
+	 * with the MCE subsystem.
+	 */
+	if (notifier_registered == false) {
+		mce_register_decode_chain(&amdgpu_bad_page_nb);
+		notifier_registered = true;
+	}
+}
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#1 lore raw

Hi,

so this is a drive-by review using the lore.kernel.org mail because I wasn't CCed on this.

Quoted 4 lines · Mukul Joshi
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +4 -0
+static int amdgpu_bad_page_notifier(struct notifier_block *nb,
+				    unsigned long val, void *data)
+{
+	struct mce *m = (struct mce *)data;

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2499

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10 -0
+	struct amdgpu_device *adev = NULL;
+	uint32_t gpu_id = 0;
+	uint32_t umc_inst = 0;
+	uint32_t chan_index = 0;
+	struct ras_err_data err_data = {0, 0, 0, NULL};
+	struct eeprom_table_record err_rec;
+	uint64_t retired_page;
+
+	/*
+	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,

Why does it matter if the error is a v2 UMC generated one?

IOW, can you detect the type of errors in GPU memory - I assume this is what this is supposed to handle - from the error signature itself instead of doing is_smca_umc_v2?

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +16 -0
+	 * and error occurred in DramECC (Extended error code = 0) then only
+	 * process the error, else bail out.
+	 */
+	if (!m || !(is_smca_umc_v2(m->bank) && (XEC(m->status, 0x1f) == 0x0)))
+		return NOTIFY_DONE;
+
+	gpu_id = GET_MCA_IPID_GPUID(m->ipid);
+
+	/*
+	 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
+	 */
+	gpu_id -= GPU_ID_OFFSET;
+
+	adev = find_adev(gpu_id);
+	if (!adev) {
+		dev_warn(adev->dev, "%s: Unable to find adev for gpu_id: %d\n",

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2525

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10 -0
+				     __func__, gpu_id);
+		return NOTIFY_DONE;
+	}
+
+	/*
+	 * If it is correctable error, then print a message and return.
+	 */
+	if (mce_is_correctable(m)) {
+		dev_info(adev->dev, "%s: UMC Correctable error detected.",
+				    __func__);

So put yourself in the shoes of a support engineer who starts getting all those calls about people's hardware getting correctable errors and should they replace it and should AMD RMA the GPUs and so on and so on..? Do you really wanna be on the receiving side of that call?

IOW, whom does printing the fact that the GPU had a corrected error which got corrected by the hardware, help and do you really want to upset people needlessly?

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2533

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +8 -0
+		return NOTIFY_OK;
+	}
+
+	/*
+	 * If it is uncorrectable error, then find out UMC instance and
+	 * channel index.
+	 */
+	find_umc_inst_chan_index(m, &umc_inst, &chan_index);

That's a void function but it could return a pointer to the instance and channel bundled in a struct maybe...

Quoted 26 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +26 -0
+
+	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d,"
+			    " chan_idx: %d", umc_inst, chan_index);
+
+	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
+
+	/*
+	 * Translate UMC channel address to Physical address
+	 */
+	retired_page = ADDR_OF_8KB_BLOCK(m->addr) |
+			ADDR_OF_256B_BLOCK(chan_index) |
+			OFFSET_IN_256B_BLOCK(m->addr);
+
+	err_rec.address = m->addr;
+	err_rec.retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
+	err_rec.ts = (uint64_t)ktime_get_real_seconds();
+	err_rec.err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
+	err_rec.cu = 0;
+	err_rec.mem_channel = chan_index;
+	err_rec.mcumc_id = umc_inst;
+
+	err_data.err_addr = &err_rec;
+	err_data.err_addr_cnt = 1;
+
+	if (amdgpu_bad_page_threshold != 0) {
+		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2569

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10 -0
+						err_data.err_addr_cnt);
+		amdgpu_ras_save_bad_pages(adev);
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block amdgpu_bad_page_nb = {
+	.notifier_call  = amdgpu_bad_page_notifier,
+	.priority       = MCE_PRIO_ACCEL,

Nothing ever explains why this needs to be a separate priority. So how about it?

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2578

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +9 -0
+};
+
+static void amdgpu_register_bad_pages_mca_notifier(void)
+{
+	/*
+	 * Register the x86 notifier with MCE subsystem.
+	 * Please note a notifier can be registered only once
+	 * with the MCE subsystem.
+	 */

Why do you need this? Other users don't need that. Do you need to call mce_unregister_decode_chain() when the driver gets removed or so?

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2579

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +1 -0
+	if (notifier_registered == false) {

This is just silly and should be fixed properly once the issue is understood.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#2 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 34 lines

-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Wednesday, May 12, 2021 5:37 AM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

[CAUTION: External Email]

Hi,

so this is a drive-by review using the lore.kernel.org mail because I wasn't CCed on this.

Quoted 4 lines · Mukul Joshi
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +4 -0
+static int amdgpu_bad_page_notifier(struct notifier_block *nb,
+                                 unsigned long val, void *data) {
+     struct mce *m = (struct mce *)data;
+     struct amdgpu_device *adev = NULL;

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10 -0
+     uint32_t gpu_id = 0;
+     uint32_t umc_inst = 0;
+     uint32_t chan_index = 0;
+     struct ras_err_data err_data = {0, 0, 0, NULL};
+     struct eeprom_table_record err_rec;
+     uint64_t retired_page;
+
+     /*
+      * If the error was generated in UMC_V2, which belongs to GPU
+ UMCs,

Why does it matter if the error is a v2 UMC generated one?

Quoted 3 lines

IOW, can you detect the type of errors in GPU memory - I assume this is what this is supposed to handle - from the error signature itself instead of doing is_smca_umc_v2?

SMCA UMCv2 corresponds to GPU's UMC MCA bank and the GPU driver is only interested in errors on GPU UMC. We cannot know this without is_smca_umc_v2.

Quoted 33 lines
Quoted 17 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +16 -0
+      * and error occurred in DramECC (Extended error code = 0) then only
+      * process the error, else bail out.
+      */
+     if (!m || !(is_smca_umc_v2(m->bank) && (XEC(m->status, 0x1f) == 0x0)))
+             return NOTIFY_DONE;
+
+     gpu_id = GET_MCA_IPID_GPUID(m->ipid);
+
+     /*
+      * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
+      */
+     gpu_id -= GPU_ID_OFFSET;
+
+     adev = find_adev(gpu_id);
+     if (!adev) {
+             dev_warn(adev->dev, "%s: Unable to find adev for gpu_id: %d\n",

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2525

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10 -0
+                                  __func__, gpu_id);
+             return NOTIFY_DONE;
+     }
+
+     /*
+      * If it is correctable error, then print a message and return.
+      */
+     if (mce_is_correctable(m)) {
+             dev_info(adev->dev, "%s: UMC Correctable error detected.",
+                                 __func__);

So put yourself in the shoes of a support engineer who starts getting all those calls about people's hardware getting correctable errors and should they replace it and should AMD RMA the GPUs and so on and so on..? Do you really wanna be on the receiving side of that call?

Quoted 3 lines

IOW, whom does printing the fact that the GPU had a corrected error which got corrected by the hardware, help and do you really want to upset people needlessly?

Agree. I will remove this debug print.

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2533

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +8 -0
+             return NOTIFY_OK;
+     }
+
+     /*
+      * If it is uncorrectable error, then find out UMC instance and
+      * channel index.
+      */
+     find_umc_inst_chan_index(m, &umc_inst, &chan_index);
Quoted 2 lines

That's a void function but it could return a pointer to the instance and channel bundled in a struct maybe...

Maybe. I hope its not too much of a concern if it stays the way it is.

Quoted 31 lines
Quoted 31 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +30 -0
+
+     dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d,"
+                         " chan_idx: %d", umc_inst, chan_index);
+
+     memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
+
+     /*
+      * Translate UMC channel address to Physical address
+      */
+     retired_page = ADDR_OF_8KB_BLOCK(m->addr) |
+                     ADDR_OF_256B_BLOCK(chan_index) |
+                     OFFSET_IN_256B_BLOCK(m->addr);
+
+     err_rec.address = m->addr;
+     err_rec.retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
+     err_rec.ts = (uint64_t)ktime_get_real_seconds();
+     err_rec.err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
+     err_rec.cu = 0;
+     err_rec.mem_channel = chan_index;
+     err_rec.mcumc_id = umc_inst;
+
+     err_data.err_addr = &err_rec;
+     err_data.err_addr_cnt = 1;
+
+     if (amdgpu_bad_page_threshold != 0) {
+             amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
+                                             err_data.err_addr_cnt);
+             amdgpu_ras_save_bad_pages(adev);
+     }
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2569

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +6 -0
+     return NOTIFY_OK;
+}
+
+static struct notifier_block amdgpu_bad_page_nb = {
+     .notifier_call  = amdgpu_bad_page_notifier,
+     .priority       = MCE_PRIO_ACCEL,

Nothing ever explains why this needs to be a separate priority. So how about it?

I wasn't really sure if I should use the EDAC priority here or create a new one for Accelerator devices. I thought using EDAC priority might not be accepted by the maintainers as EDAC and GPU (Accelerator) devices are two different class of devices.
That is the reason I create a new one.
I am OK to use EDAC priority if that is acceptable.

Quoted 14 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2578

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +9 -0
+};
+
+static void amdgpu_register_bad_pages_mca_notifier(void)
+{
+     /*
+      * Register the x86 notifier with MCE subsystem.
+      * Please note a notifier can be registered only once
+      * with the MCE subsystem.
+      */

Why do you need this? Other users don't need that. Do you need to call mce_unregister_decode_chain() when the driver gets removed or so?

Quoted 3 lines
Quoted 1 line

+ if (notifier_registered == false) {

This is just silly and should be fixed properly once the issue is understood.

A system can have multiple GPUs and we only want a single notifier registered. I will change the comment to explicitly state this.

Thanks,
Mukul

Quoted 5 lines

--
Regards/Gruss,
Boris.

Quoted 7 lines

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&amp;data=04%7C01%7Cmukul.joshi%40amd.com%7C1c231207786 446018e7208d915297942%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637564090158211378%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata =7LSOnoJWrTf5z96YFACxuRKZP1z4E4zkvtrNzjbTaPs%3D&amp;reserved=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#3 lore raw
Quoted 2 lines · Mukul

SMCA UMCv2 corresponds to GPU's UMC MCA bank and the GPU driver is only interested in errors on GPU UMC.

So that thing should be called SMCA_GPU_UMC not SMCA_UMC_V2.

Quoted 1 line

We cannot know this without is_smca_umc_v2.

You don't need it - just export smca_get_bank_type() and test the bank type at the call site.

Quoted 1 line

Maybe. I hope its not too much of a concern if it stays the way it is.

That was just a suggestion anyway - it is not code I maintain so not my call.

Quoted 5 lines

I wasn't really sure if I should use the EDAC priority here or create a new one for Accelerator devices. I thought using EDAC priority might not be accepted by the maintainers as EDAC and GPU (Accelerator) devices are two different class of devices.
That is the reason I create a new one.
I am OK to use EDAC priority if that is acceptable.

I don't know what's acceptable because I still am unclear as to what that thing is supposed to do. It seems you are interested only in uncorrectable errors.

How are those errors reported? #MC exception, deferred interrupt, simply logged in the bank and we find them by polling?

Then, the commit message is talking about some "bad page retirement". What does that do? What can the user do when she sees the

"Uncorrectable error detected in UMC..."

message?

It depends on what "retiring" of GPU pages means...

In any case, dmesg should issue a human-understandable message about the recovery action being done and what that means for the user: should she replace the GPU, should she ignore, etc, etc.

Quoted 2 lines

A system can have multiple GPUs and we only want a single notifier registered. I will change the comment to explicitly state this.

Actually, the notifier registration should be able to return a different retval to state that a callback has already been registered but that warns only currently so I'm guessing we're stuck with such ugly "workarounds" for their shortcomings.

I'm gonna take a look whether they can be fixed though so that you don't have to do this notifier_registered thing.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#4 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 15 lines

-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Wednesday, May 12, 2021 5:06 PM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

[CAUTION: External Email]

Quoted 2 lines · Mukul

SMCA UMCv2 corresponds to GPU's UMC MCA bank and the GPU driver is only interested in errors on GPU UMC.

Quoted 6 lines

So that thing should be called SMCA_GPU_UMC not SMCA_UMC_V2.

Quoted 1 line

We cannot know this without is_smca_umc_v2.

You don't need it - just export smca_get_bank_type() and test the bank type at the call site.

Exporting smca_get_bank_type() works fine when CONFIG_X86_MCE_AMD is defined. I would need to put #ifdef CONFIG_X86_MCE_AMD in my code to compile the amdgpu driver when CONFIG_X86_MCE_AMD is not defined.
I can avoid all that by using is_smca_umc_v2().
I think it would be cleaner with using is_smca_umc_v2().

Quoted 6 lines
Quoted 2 lines

Maybe. I hope its not too much of a concern if it stays the way it is.

That was just a suggestion anyway - it is not code I maintain so not my call.

Quoted 1 line

I wasn't really sure if I should use the EDAC priority here or create a new one

Quoted 9 lines

for Accelerator devices.

Quoted 4 lines

I thought using EDAC priority might not be accepted by the maintainers as EDAC and GPU (Accelerator) devices are two different class of devices. That is the reason I create a new one.
I am OK to use EDAC priority if that is acceptable.

I don't know what's acceptable because I still am unclear as to what that thing is supposed to do. It seems you are interested only in uncorrectable errors.

You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.

Quoted 3 lines

How are those errors reported? #MC exception, deferred interrupt, simply logged in the bank and we find them by polling?

It is a deferred interrupt that generates an MCE.

Quoted 5 lines

Then, the commit message is talking about some "bad page retirement". What does that do? What can the user do when she sees the

"Uncorrectable error detected in UMC..."

Quoted 8 lines

message?

It depends on what "retiring" of GPU pages means...

In any case, dmesg should issue a human-understandable message about the recovery action being done and what that means for the user: should she replace the GPU, should she ignore, etc, etc.

When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.
By retiring, we mean we reserve the page so that it is not available for allocations to any applications.
We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.

Hope it clears what "bad page retirement" is achieving.

Thanks,
Mukul

Quoted 14 lines
Quoted 2 lines

A system can have multiple GPUs and we only want a single notifier registered. I will change the comment to explicitly state this.

Actually, the notifier registration should be able to return a different retval to state that a callback has already been registered but that warns only currently so I'm guessing we're stuck with such ugly "workarounds" for their shortcomings.

I'm gonna take a look whether they can be fixed though so that you don't have to do this notifier_registered thing.

--
Regards/Gruss,
Boris.

Quoted 8 lines

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&amp;data=04%7C01%7CMukul.Joshi%40amd.com%7C5e305845504 14bc53c3008d91589b50b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637564503481206042%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata =fF2iaSQxAREAV2OwLi%2F8cN9uRuqNNKim2FpWJUBQS98%3D&amp;reserved= 0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#5 lore raw
Quoted 5 lines · Mukul

Exporting smca_get_bank_type() works fine when CONFIG_X86_MCE_AMD is defined. I would need to put #ifdef CONFIG_X86_MCE_AMD in my code to compile the amdgpu driver when CONFIG_X86_MCE_AMD is not defined.
I can avoid all that by using is_smca_umc_v2().
I think it would be cleaner with using is_smca_umc_v2().

See how smca_get_long_name() is exported and export that function the same way.

To save you some energy: is_smca_umc_v2() is not going to happen.

Quoted 2 lines

You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.

An EDAC "device", as you call it, is not interested in handling UEs. If anything, it counts them.

Quoted 1 line

It is a deferred interrupt that generates an MCE.

Is that the same deferred interrupt which calls amd_deferred_error_interrupt() ?

Quoted 3 lines

When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.

What page is that? Normal DRAM page or a page in some special GPU memory?

Quoted 2 lines

By retiring, we mean we reserve the page so that it is not available for allocations to any applications.

We do that for normal DRAM memory pages by poisoning them. I hope you don't mean that.

Looking at

amdgpu_ras_add_bad_pages
|-> amdgpu_vram_mgr_reserve_range

that's some VRAM thing so I'm guessing special memory on the GPU.

If so, what happens with all those "retired" pages when you reboot? They're getting used again and potentially trigger the same UEs and the same retiring happens?

Quoted 3 lines

We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.

Ok, I'm a user and I can access that information through sysfs. What can I do with it?

Quoted 1 line

Hope it clears what "bad page retirement" is achieving.

It is getting there.

Thx.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#6 lore raw
Quoted 21 lines · Borislav Petkov
Quoted 5 lines · Mukul

Exporting smca_get_bank_type() works fine when CONFIG_X86_MCE_AMD is defined. I would need to put #ifdef CONFIG_X86_MCE_AMD in my code to compile the amdgpu driver when CONFIG_X86_MCE_AMD is not defined.
I can avoid all that by using is_smca_umc_v2().
I think it would be cleaner with using is_smca_umc_v2().

See how smca_get_long_name() is exported and export that function the same way.

To save you some energy: is_smca_umc_v2() is not going to happen.

Quoted 2 lines

You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.

An EDAC "device", as you call it, is not interested in handling UEs. If anything, it counts them.

Quoted 1 line

It is a deferred interrupt that generates an MCE.

Quoted 8 lines

Is that the same deferred interrupt which calls amd_deferred_error_interrupt() ?

Quoted 3 lines

When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.

What page is that? Normal DRAM page or a page in some special GPU memory?

GPU memory.

Quoted 6 lines
Quoted 2 lines

By retiring, we mean we reserve the page so that it is not available for allocations to any applications.

We do that for normal DRAM memory pages by poisoning them. I hope you don't mean that.

Quoted 7 lines

Looking at

amdgpu_ras_add_bad_pages
|-> amdgpu_vram_mgr_reserve_range

that's some VRAM thing so I'm guessing special memory on the GPU.

Yes.

Quoted 3 lines

If so, what happens with all those "retired" pages when you reboot? They're getting used again and potentially trigger the same UEs and the same retiring happens?

The bad pages are stored in an EEPROM on the board and the next time the driver loads it reads the EEPROM so that it can reserve the bad pages at init time so they don't get used again.

Alex

Quoted 18 lines
Quoted 4 lines

We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.

Ok, I'm a user and I can access that information through sysfs. What can I do with it?

Quoted 1 line

Hope it clears what "bad page retirement" is achieving.

It is getting there.

Thx.

--
Regards/Gruss,
Boris.

Quoted 5 lines

https://people.kernel.org/tglx/notes-about-netiquette
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#7 lore raw
Quoted 3 lines · Alex Deucher

The bad pages are stored in an EEPROM on the board and the next time the driver loads it reads the EEPROM so that it can reserve the bad pages at init time so they don't get used again.

And that works automagically on the next boot? Because that sounds like the right thing to do.

So practically, what happens to a GPU in such a case where the VRAM starts going bad? It might get exhausted eventually and the driver will say something along the lines of:

"VRAM bad pages: 80%, consider replacing the GPU. It is operating currently with degrated performance."

or so?

Yap, from a RAS perspective, that makes good sense as you're prolonging the life of the component while still remains operational as good as it can and the only user interaction you need is she/he replacing it.

Sounds good.

Thx.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#8 lore raw
Quoted 8 lines · Borislav Petkov
Quoted 3 lines · Alex Deucher

The bad pages are stored in an EEPROM on the board and the next time the driver loads it reads the EEPROM so that it can reserve the bad pages at init time so they don't get used again.

And that works automagically on the next boot? Because that sounds like the right thing to do.

Yes, or driver reload, suspend/resume, etc.

Quoted 9 lines

So practically, what happens to a GPU in such a case where the VRAM starts going bad? It might get exhausted eventually and the driver will say something along the lines of:

"VRAM bad pages: 80%, consider replacing the GPU. It is operating currently with degrated performance."

or so?

Right. The sys admin can query the bad page count and decide when to retire the card.

Quoted 6 lines

Yap, from a RAS perspective, that makes good sense as you're prolonging the life of the component while still remains operational as good as it can and the only user interaction you need is she/he replacing it.

Sounds good.

Yes. That's the idea.

Alex

Quoted 8 lines

Thx.

--
Regards/Gruss,
Boris.

https://people.kernel.org/tglx/notes-about-netiquette

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#9 lore raw
Quoted 2 lines · Alex Deucher

Right. The sys admin can query the bad page count and decide when to retire the card.

Yap, although the driver should actively "tell" the sysadmin when some critical counts of retired VRAM pages are reached because I doubt all admins would go look at those counts on their own.

Btw, you say "admin" - am I to understand that those are some high end GPU cards with ECC memory? If consumer grade stuff has this too, then the driver should very much warn on such levels on its own because normal users won't know what and where to look.

Other than that, the big picture sounds good to me.

Thx.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#10 lore raw
Quoted 8 lines · Borislav Petkov
Quoted 2 lines · Alex Deucher

Right. The sys admin can query the bad page count and decide when to retire the card.

Yap, although the driver should actively "tell" the sysadmin when some critical counts of retired VRAM pages are reached because I doubt all admins would go look at those counts on their own.

I think we print something in the log as well when we hit the threshold. I need to double check the code.

Quoted 6 lines

Btw, you say "admin" - am I to understand that those are some high end GPU cards with ECC memory? If consumer grade stuff has this too, then the driver should very much warn on such levels on its own because normal users won't know what and where to look.

Currently it's only available on workstation and datacenter boards.

Quoted 1 line

Other than that, the big picture sounds good to me.

Thanks!

Alex

Quoted 8 lines

Thx.

--
Regards/Gruss,
Boris.

https://people.kernel.org/tglx/notes-about-netiquette

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#11 lore raw
In reply to Borislav Petkov

[AMD Official Use Only - Internal Distribution Only]

Quoted 19 lines

-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Thursday, May 13, 2021 10:58 AM
To: Alex Deucher <[email protected]>
Cc: Joshi, Mukul <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <linux- [email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

[CAUTION: External Email]

Quoted 2 lines · Alex Deucher

Right. The sys admin can query the bad page count and decide when to retire the card.

Yap, although the driver should actively "tell" the sysadmin when some critical counts of retired VRAM pages are reached because I doubt all admins would go look at those counts on their own.

Quoted 7 lines

Btw, you say "admin" - am I to understand that those are some high end GPU cards with ECC memory? If consumer grade stuff has this too, then the driver should very much warn on such levels on its own because normal users won't know what and where to look.

Other than that, the big picture sounds good to me.

Since now you are OK with how page retirement works, lets revisit the original question.

Are you OK with a new MCE priority (MCE_PRIO_ACCEL) or do you want us to use something else?

Thanks,
Mukul

Quoted 6 lines

Thx.

--
Regards/Gruss,
Boris.

Quoted 8 lines

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&amp;data=04%7C01%7CMukul.Joshi%40amd.com%7C50588f11ed5 3456b03e008d9161f765c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637565146658376385%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata =Es0FMDNzNEKgxvFiqe1kOo9aEPK6%2BOXrhI5aWs3QH9Q%3D&amp;reserved= 0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#12 lore raw
Quoted 2 lines · Mukul

Are you OK with a new MCE priority (MCE_PRIO_ACCEL) or do you want us to use something else?

I still don't know why a separate priority is needed. Maybe this still needs answering:

Quoted 1 line

It is a deferred interrupt that generates an MCE.

Is that the same deferred interrupt which calls
amd_deferred_error_interrupt() ?

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#13 lore raw

[AMD Official Use Only]

Quoted 15 lines

-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Friday, May 14, 2021 3:04 AM
To: Joshi, Mukul <[email protected]>
Cc: Alex Deucher <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <linux- [email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

[CAUTION: External Email]

Quoted 2 lines · Mukul

Are you OK with a new MCE priority (MCE_PRIO_ACCEL) or do you want us to use something else?

Quoted 7 lines

I still don't know why a separate priority is needed. Maybe this still needs answering:

Quoted 1 line

It is a deferred interrupt that generates an MCE.

Is that the same deferred interrupt which calls
amd_deferred_error_interrupt() ?

Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls amd_deferred_error_interrupt().

Thanks,
Mukul

Quoted 5 lines

--
Regards/Gruss,
Boris.

Quoted 7 lines

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&amp;data=04%7C01%7CMukul.Joshi%40amd.com%7C7e04d0ad3c6 147a8dfd008d916a66bbc%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637565726326540654%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata =AWHJvqIbtkGl6OPC86alAtnxsleq6UVe9mcM1MjxpUQ%3D&amp;reserved=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#14 lore raw

On Thu, May 27, 2021 at 03:54:27PM -0400, Joshi, Mukul wrote:
...

Quoted 6 lines
Quoted 2 lines

Is that the same deferred interrupt which calls
amd_deferred_error_interrupt() ?

Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls amd_deferred_error_interrupt().

Mukul,

Do you expect that the driver will need to mark pages with high correctable error counts as bad? I think the hardware folks may want the GPU memory errors to be handled more aggressively than CPU memory errors. The specific threshold may change from product to product, so it may make sense to hardcode this in the driver.

We have similar functionality in the Correctable Errors Collector. But enterprise users may prefer a direct approach done in the driver (based on the hardware experts' guidance) instead of configuring the kernel at runtime.

So I think having a separate priority may make sense if some special functionality, or combination of behaviors, is needed which don't fall under any exisiting things. In this case, "special functionality" could be that the GPU memory needs to be handled differently than CPU memory.

Another thing is that this behavior is similar to the NFIT behavior, i.e. there's a memory error on an external device that needs to be handled by the device's driver. So maybe we can rename MCE_PRIO_NFIT to be generic (MCE_PRIO_EXTERNAL?) and use that? Multiple notifiers with the same priority is okay, right?

Thanks,
Yazen
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#15 lore raw

[AMD Official Use Only]

Quoted 20 lines

-----Original Message-----
From: Ghannam, Yazen <[email protected]>
Sent: Thursday, June 3, 2021 5:13 PM
To: Joshi, Mukul <[email protected]>
Cc: Borislav Petkov <[email protected]>; Alex Deucher <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

On Thu, May 27, 2021 at 03:54:27PM -0400, Joshi, Mukul wrote:
...

Quoted 5 lines
Quoted 2 lines

Is that the same deferred interrupt which calls
amd_deferred_error_interrupt() ?

Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls

amd_deferred_error_interrupt().

Quoted 1 line
Quoted 8 lines

Mukul,

Do you expect that the driver will need to mark pages with high correctable error counts as bad? I think the hardware folks may want the GPU memory errors to be handled more aggressively than CPU memory errors. The specific threshold may change from product to product, so it may make sense to hardcode this in the driver.

Sorry I missed this email completely. Just saw it so responding now.

At the moment, we don't have a requirement to mark a page "bad" if there is a high correctable error counts. Our previous GPU ASICs which support RAS, also do not have such a feature.
But you make a good point. It might be worthwhile to go and ask the hardware folks about it.

Quoted 9 lines

We have similar functionality in the Correctable Errors Collector. But enterprise users may prefer a direct approach done in the driver (based on the hardware experts' guidance) instead of configuring the kernel at runtime.

So I think having a separate priority may make sense if some special functionality, or combination of behaviors, is needed which don't fall under any exisiting things. In this case, "special functionality" could be that the GPU memory needs to be handled differently than CPU memory.

Quoted 6 lines

Another thing is that this behavior is similar to the NFIT behavior, i.e. there's a memory error on an external device that needs to be handled by the device's driver. So maybe we can rename MCE_PRIO_NFIT to be generic (MCE_PRIO_EXTERNAL?) and use that? Multiple notifiers with the same priority is okay, right?

With respect to MCE priority, I was thinking of using the MCE_PRIO_EDAC instead of creating a new priority as the code in the GPU driver is doing error detection and handling the uncorrectable errors. Not sure if that aligns with the definition of EDAC device in the kernel.

What do you think?

Regards,
Mukul

Quoted 2 lines

Thanks,
Yazen

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#16 lore raw

[AMD Official Use Only]

Quoted 78 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Joshi, Mukul
Sent: Thursday, July 29, 2021 8:00 PM
To: Ghannam, Yazen <[email protected]>
Cc: x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <[email protected]>; [email protected]; Borislav Petkov <[email protected]>; Alex Deucher <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

[CAUTION: External Email]

[AMD Official Use Only]

Quoted 21 lines

-----Original Message-----
From: Ghannam, Yazen <[email protected]>
Sent: Thursday, June 3, 2021 5:13 PM
To: Joshi, Mukul <[email protected]>
Cc: Borislav Petkov <[email protected]>; Alex Deucher <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

On Thu, May 27, 2021 at 03:54:27PM -0400, Joshi, Mukul wrote:
...

Quoted 5 lines
Quoted 2 lines

Is that the same deferred interrupt which calls
amd_deferred_error_interrupt() ?

Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls

amd_deferred_error_interrupt().

Quoted 1 line
Quoted 8 lines

Mukul,

Do you expect that the driver will need to mark pages with high correctable error counts as bad? I think the hardware folks may want the GPU memory errors to be handled more aggressively than CPU memory errors. The specific threshold may change from product to product, so it may make sense to hardcode this in the driver.

Sorry I missed this email completely. Just saw it so responding now.

At the moment, we don't have a requirement to mark a page "bad" if there is a high correctable error counts.
Our previous GPU ASICs which support RAS, also do not have such a feature. But you make a good point. It might be worthwhile to go and ask the hardware folks about it.

Quoted 3 lines

We have similar functionality in the Correctable Errors Collector. But enterprise users may prefer a direct approach done in the driver (based on the hardware experts' guidance) instead of configuring the kernel at

runtime.

Quoted 5 lines

So I think having a separate priority may make sense if some special functionality, or combination of behaviors, is needed which don't fall under any exisiting things. In this case, "special functionality" could be that the GPU memory needs to be handled differently than CPU

memory.

Quoted 8 lines

Another thing is that this behavior is similar to the NFIT behavior, i.e. there's a memory error on an external device that needs to be handled by the device's driver. So maybe we can rename MCE_PRIO_NFIT to be generic
(MCE_PRIO_EXTERNAL?) and use that? Multiple notifiers with the same priority is okay, right?

With respect to MCE priority, I was thinking of using the MCE_PRIO_EDAC instead of creating a new priority as the code in the GPU driver is doing error detection and handling the uncorrectable errors.
Not sure if that aligns with the definition of EDAC device in the kernel.

Quoted 5 lines

What do you think?

Regards,
Mukul

After talking to Yazen, MCE_PRIO_UC might be a better choice for the MCE priority as we are dealing
only with uncorrectable errors.
I will be sending out a v2 patch with changes to use the MCE_PRIO_UC and drop the MCE_PRIO_ACCEL and see what the feedback is.

Thanks,
Mukul

Quoted 12 lines
Quoted 2 lines

Thanks,
Yazen

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.fre
edesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&amp;data=04%7C01%7Cmukul.joshi%40amd.com%7C7d32897fddef448ab0 aa08d952ecf41f%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376 31999953383488%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJ QIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=YWZz9 OYTMOhBl4183kV5ZYj01yw0xwNj%2BjTdXejFKH8%3D&amp;reserved=0

#17 lore raw
In reply to Borislav Petkov

[AMD Official Use Only - Internal Distribution Only]

Quoted 13 lines

-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Thursday, May 13, 2021 5:53 AM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

[CAUTION: External Email]

Quoted 1 line · Mukul

Exporting smca_get_bank_type() works fine when CONFIG_X86_MCE_AMD is

Quoted 9 lines

defined.

Quoted 4 lines

I would need to put #ifdef CONFIG_X86_MCE_AMD in my code to compile the amdgpu driver when CONFIG_X86_MCE_AMD is not defined. I can avoid all that by using is_smca_umc_v2().
I think it would be cleaner with using is_smca_umc_v2().

See how smca_get_long_name() is exported and export that function the same way.

That's probably not the best example to look at. smca_get_long_name() is used in drivers/edac/mce_amd.c and this file doesn't get compiled when CONFIG_X86_MCE_AMD is not defined.

And amdgpu driver has no dependency on CONFIG_X86_MCE_AMD.

So here is one option that we can try:
1. Export smca_get_bank_type().
2. I wrap my entire code in GPU driver with #ifdef CONFIG_X86_MCE_AMD

Will that work for you?

Thanks,
Mukul

Quoted 54 lines

To save you some energy: is_smca_umc_v2() is not going to happen.

Quoted 2 lines

You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.

An EDAC "device", as you call it, is not interested in handling UEs. If anything, it counts them.

Quoted 1 line

It is a deferred interrupt that generates an MCE.

Is that the same deferred interrupt which calls amd_deferred_error_interrupt() ?

Quoted 3 lines

When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.

What page is that? Normal DRAM page or a page in some special GPU memory?

Quoted 2 lines

By retiring, we mean we reserve the page so that it is not available for allocations to any applications.

We do that for normal DRAM memory pages by poisoning them. I hope you don't mean that.

Looking at

amdgpu_ras_add_bad_pages
|-> amdgpu_vram_mgr_reserve_range

that's some VRAM thing so I'm guessing special memory on the GPU.

If so, what happens with all those "retired" pages when you reboot? They're getting used again and potentially trigger the same UEs and the same retiring happens?

Quoted 3 lines

We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.

Ok, I'm a user and I can access that information through sysfs. What can I do with it?

Quoted 1 line

Hope it clears what "bad page retirement" is achieving.

It is getting there.

Thx.

--
Regards/Gruss,
Boris.

Quoted 8 lines

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&amp;data=04%7C01%7CMukul.Joshi%40amd.com%7Cd8c660fce3a2 4ce3c6d408d915f4efa6%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0% 7C637564964013263414%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwM DAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata= %2BnJ%2B99N%2FRljoHGALimZHZG%2Bmf9jL5zP2eA44I6pbzFY%3D&amp;reser ved=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#18 lore raw
Quoted 1 line · Mukul

That's probably not the best example to look at.

Oh, it is the *perfect* example but...

Quoted 4 lines

smca_get_long_name() is used in drivers/edac/mce_amd.c and this file doesn't get compiled when CONFIG_X86_MCE_AMD is not defined.

And amdgpu driver has no dependency on CONFIG_X86_MCE_AMD.

... maybe this will make you see it the right way: how much of the amdgpu RAS functionality you're adding, is going to even function without CONFIG_X86_MCE_AMD?

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette
#19 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 9 lines

-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Friday, May 14, 2021 3:06 AM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran

Quoted 7 lines

[CAUTION: External Email]

Quoted 1 line · Mukul

That's probably not the best example to look at.

Oh, it is the *perfect* example but...

I would prefer not to get into an argument of if it is the perfect example or not. I would prefer to get the job done and move forward.

Quoted 9 lines
Quoted 4 lines

smca_get_long_name() is used in drivers/edac/mce_amd.c and this file doesn't get compiled when CONFIG_X86_MCE_AMD is not defined.

And amdgpu driver has no dependency on CONFIG_X86_MCE_AMD.

... maybe this will make you see it the right way: how much of the amdgpu RAS functionality you're adding, is going to even function without CONFIG_X86_MCE_AMD?

We have RAS functionality in other ASICs that is not dependent on CONFIG_X86_MCE_AMD. So, I don't think we would want to do that just for one ASIC.
Maybe Alex D. has an opinion on it.

As I had mentioned in my previous comment, I am fine with wrapping around my code with #ifdef CONFIG_X86_MCE_AMD and exporting smca_get_bank_type().
I hope we can move forward with this approach.

Thanks,
Mukul

Quoted 4 lines

--
Regards/Gruss,
Boris.

Quoted 8 lines

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&amp;data=04%7C01%7CMukul.Joshi%40amd.com%7C7fded9165e6 64dde943808d916a6b33f%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637565727509040995%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata =5hNyAEla4H%2BN0kMdCDs6iL0Dr8mVQJNxKA0UXJU7%2F8c%3D&amp;reserv ed=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#20 lore raw
Quoted 3 lines · Mukul

We have RAS functionality in other ASICs that is not dependent on CONFIG_X86_MCE_AMD. So, I don't think we would want to do that just for one ASIC.

Lemme try again: you said that those errors do get reported through a deferred interrupt. Which is likely amd_deferred_error_interrupt().

If it is that interrupt and you don't have CONFIG_X86_MCE_AMD enabled, then you won't get any errors reported and your RAS functionality will simply sit there inactive.

So if that above is true - something to which I'm still not getting an answer but maybe one fine day... - so if that above is true, your RAS functionality *needs* CONFIG_X86_MCE_AMD to be enabled in order to *actually* function.

So you *must* make your RAS functionality depend on CONFIG_X86_MCE_AMD
- otherwise no deferred interrupts and no errors reported. It is that simple.

Signature · 4 lines
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette