drm/amdgpu: Register bad page handler for Aldebaran
Subsystems (2)
PATCH 1/1 drm/amdgpu: Register bad page handler for Aldebaran
On Aldebaran, GPU driver will handle bad page retirement even though UMC is host managed. As a result, register a bad page retirement handler on the mce notifier chain to retire bad pages on Aldebaran.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 154 ++++++++++++++++++++++++ 1 file changed, 154 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +154 -0
@@ -34,7 +34,9 @@#include "amdgpu_xgmi.h" #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" #include "atom.h" +#include <asm/mce.h> +static bool notifier_registered;static const char *RAS_FS_NAME ="ras" ;const char *ras_error_string[] = {
@@ -73,6 +75,11 @@ const char *ras_block_string[] = {/* typical ECC bad page rate(1 bad page per 100MB VRAM) */ #define RAS_BAD_PAGE_RATE (100 * 1024 * 1024ULL) +#define GET_MCA_IPID_GPUID (m) (((m) >> 44) & 0xF) +#define GET_UMC_INST_NIBBLE (m) (((m) >> 20) & 0xF) +#define GET_CHAN_INDEX_NIBBLE (m) (((m) >> 12) & 0xF) +#define GPU_ID_OFFSET 8 +enum amdgpu_ras_retire_page_reservation {AMDGPU_RAS_RETIRE_PAGE_RESERVED ,AMDGPU_RAS_RETIRE_PAGE_PENDING ,
@@ -85,6 +92,7 @@ static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, uint64_t addr);static bool amdgpu_ras_check_bad_page (struct amdgpu_device *adev,uint64_t addr); +static void amdgpu_register_bad_pages_mca_notifier (void );void amdgpu_ras_set_error_query_ready (struct amdgpu_device *adev,bool ready) {
@@ -1978,6 +1986,10 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev) goto free; } +if ((adev->asic_type ==CHIP_ALDEBARAN ) && + (adev->gmc.xgmi.connected_to_cpu)) +amdgpu_register_bad_pages_mca_notifier (); +return 0 ; free:
@@ -2427,3 +2439,145 @@ void amdgpu_release_ras_context(struct amdgpu_device *adev)kfree (con); } } + +static struct amdgpu_device *find_adev (uint32_t node_id) +{ +struct amdgpu_gpu_instance *gpu_instance; +int i; +struct amdgpu_device *adev =NULL ; + +mutex_lock (&mgpu_info.mutex); + +for (i =0 ; i < mgpu_info.num_gpu; i++) { + gpu_instance = &(mgpu_info.gpu_ins[i]); + adev = gpu_instance->adev; + +if (adev->gmc.xgmi.connected_to_cpu && + adev->gmc.xgmi.physical_node_id == node_id) +break ; + adev =NULL ; + } + +mutex_unlock (&mgpu_info.mutex); + +return adev; +} + +static void find_umc_inst_chan_index (struct mce *m,uint32_t *umc_inst, +uint32_t *chan_index) +{ +uint32_t val1 =0 ; +uint32_t val2 =0 ; +uint32_t rem =0 ; + +/* +* Bit 20-23 provides the UMC instance nibble. +* Bit 12-15 provides the channel index nibble. +*/ + val1 =GET_UMC_INST_NIBBLE (m->ipid); + val2 =GET_CHAN_INDEX_NIBBLE (m->ipid); + + *umc_inst = val1/2 ; + rem = val1%2 ; + + *chan_index = (4 *rem) + val2; +} + +static int amdgpu_bad_page_notifier (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct mce *m = (struct mce *)data; +struct amdgpu_device *adev =NULL ; +uint32_t gpu_id =0 ; +uint32_t umc_inst =0 ; +uint32_t chan_index =0 ; +struct ras_err_data err_data = {0 ,0 ,0 ,NULL }; +struct eeprom_table_record err_rec; +uint64_t retired_page; + +/* +* If the error was generated in UMC_V2, which belongs to GPU UMCs, +* and error occurred in DramECC (Extended error code = 0) then only +* process the error, else bail out. +*/ +if (!m || !(is_smca_umc_v2 (m->bank) && (XEC (m->status,0x1f ) ==0x0 ))) +return NOTIFY_DONE ; + + gpu_id =GET_MCA_IPID_GPUID (m->ipid); + +/* +* GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. +*/ + gpu_id -=GPU_ID_OFFSET ; + + adev =find_adev (gpu_id); +if (!adev) { +dev_warn (adev->dev,"%s: Unable to find adev for gpu_id: %d\n" , + __func__, gpu_id); +return NOTIFY_DONE ; + } + +/* +* If it is correctable error, then print a message and return. +*/ +if (mce_is_correctable (m)) { +dev_info (adev->dev,"%s: UMC Correctable error detected." , + __func__); +return NOTIFY_OK ; + } + +/* +* If it is uncorrectable error, then find out UMC instance and +* channel index. +*/ +find_umc_inst_chan_index (m, &umc_inst, &chan_index); + +dev_info (adev->dev,"Uncorrectable error detected in UMC inst: %d," +" chan_idx: %d" , umc_inst, chan_index); + +memset (&err_rec,0x0 ,sizeof (struct eeprom_table_record )); + +/* +* Translate UMC channel address to Physical address +*/ + retired_page =ADDR_OF_8KB_BLOCK (m->addr) | +ADDR_OF_256B_BLOCK (chan_index) | +OFFSET_IN_256B_BLOCK (m->addr); + + err_rec.address = m->addr; + err_rec.retired_page = retired_page >>AMDGPU_GPU_PAGE_SHIFT ; + err_rec.ts = (uint64_t )ktime_get_real_seconds (); + err_rec.err_type =AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE ; + err_rec.cu =0 ; + err_rec.mem_channel = chan_index; + err_rec.mcumc_id = umc_inst; + + err_data.err_addr = &err_rec; + err_data.err_addr_cnt =1 ; + +if (amdgpu_bad_page_threshold !=0 ) { +amdgpu_ras_add_bad_pages (adev, err_data.err_addr, + err_data.err_addr_cnt); +amdgpu_ras_save_bad_pages (adev); + } + +return NOTIFY_OK ; +} + +static struct notifier_block amdgpu_bad_page_nb = { + .notifier_call = amdgpu_bad_page_notifier, + .priority =MCE_PRIO_ACCEL , +}; + +static void amdgpu_register_bad_pages_mca_notifier (void ) +{ +/* +* Register the x86 notifier with MCE subsystem. +* Please note a notifier can be registered only once +* with the MCE subsystem. +*/ +if (notifier_registered == false) { +mce_register_decode_chain (&amdgpu_bad_page_nb); + notifier_registered = true; + } +}
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Hi,
so this is a drive-by review using the lore.kernel.org mail because I wasn't CCed on this.
Quoted 4 lines · Mukul Joshi
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +4
-0+static int amdgpu_bad_page_notifier (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct mce *m = (struct mce *)data;
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2499
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10
-0+struct amdgpu_device *adev =NULL ; +uint32_t gpu_id =0 ; +uint32_t umc_inst =0 ; +uint32_t chan_index =0 ; +struct ras_err_data err_data = {0 ,0 ,0 ,NULL }; +struct eeprom_table_record err_rec; +uint64_t retired_page; + +/* +* If the error was generated in UMC_V2, which belongs to GPU UMCs,
Why does it matter if the error is a v2 UMC generated one?
IOW, can you detect the type of errors in GPU memory - I assume this is what this is supposed to handle - from the error signature itself instead of doing is_smca_umc_v2?
Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +16
-0+* and error occurred in DramECC (Extended error code = 0) then only +* process the error, else bail out. +*/ +if (!m || !(is_smca_umc_v2 (m->bank) && (XEC (m->status,0x1f ) ==0x0 ))) +return NOTIFY_DONE ; + + gpu_id =GET_MCA_IPID_GPUID (m->ipid); + +/* +* GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. +*/ + gpu_id -=GPU_ID_OFFSET ; + + adev =find_adev (gpu_id); +if (!adev) { + dev_warn(adev->dev, "%s: Unable to find adev for gpu_id: %d\n",
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2525
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10
-0+ __func__, gpu_id); +return NOTIFY_DONE ; + } + +/* +* If it is correctable error, then print a message and return. +*/ +if (mce_is_correctable (m)) { +dev_info (adev->dev,"%s: UMC Correctable error detected." , + __func__);
So put yourself in the shoes of a support engineer who starts getting all those calls about people's hardware getting correctable errors and should they replace it and should AMD RMA the GPUs and so on and so on..? Do you really wanna be on the receiving side of that call?
IOW, whom does printing the fact that the GPU had a corrected error which got corrected by the hardware, help and do you really want to upset people needlessly?
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2533
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +8
-0+return NOTIFY_OK ; + } + +/* +* If it is uncorrectable error, then find out UMC instance and +* channel index. +*/ +find_umc_inst_chan_index (m, &umc_inst, &chan_index);
That's a void function but it could return a pointer to the instance and channel bundled in a struct maybe...
Quoted 26 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +26
-0+ +dev_info (adev->dev,"Uncorrectable error detected in UMC inst: %d," +" chan_idx: %d" , umc_inst, chan_index); + +memset (&err_rec,0x0 ,sizeof (struct eeprom_table_record )); + +/* +* Translate UMC channel address to Physical address +*/ + retired_page =ADDR_OF_8KB_BLOCK (m->addr) | +ADDR_OF_256B_BLOCK (chan_index) | +OFFSET_IN_256B_BLOCK (m->addr); + + err_rec.address = m->addr; + err_rec.retired_page = retired_page >>AMDGPU_GPU_PAGE_SHIFT ; + err_rec.ts = (uint64_t )ktime_get_real_seconds (); + err_rec.err_type =AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE ; + err_rec.cu =0 ; + err_rec.mem_channel = chan_index; + err_rec.mcumc_id = umc_inst; + + err_data.err_addr = &err_rec; + err_data.err_addr_cnt =1 ; + +if (amdgpu_bad_page_threshold !=0 ) { + amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2569
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10
-0+ err_data.err_addr_cnt); +amdgpu_ras_save_bad_pages (adev); + } + +return NOTIFY_OK ; +} + +static struct notifier_block amdgpu_bad_page_nb = { + .notifier_call = amdgpu_bad_page_notifier, + .priority =MCE_PRIO_ACCEL ,
Nothing ever explains why this needs to be a separate priority. So how about it?
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2578
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +9
-0+}; + +static void amdgpu_register_bad_pages_mca_notifier (void ) +{ +/* +* Register the x86 notifier with MCE subsystem. +* Please note a notifier can be registered only once +* with the MCE subsystem. +*/
Why do you need this? Other users don't need that. Do you need to call mce_unregister_decode_chain() when the driver gets removed or so?
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2579
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +1
-0+if (notifier_registered == false) {
This is just silly and should be fixed properly once the issue is understood.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette[AMD Official Use Only - Internal Distribution Only]
Quoted 34 lines
-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Wednesday, May 12, 2021 5:37 AM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran[CAUTION: External Email]
Hi,
so this is a drive-by review using the lore.kernel.org mail because I wasn't CCed on this.
Quoted 4 lines · Mukul Joshi
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +4
-0+static int amdgpu_bad_page_notifier (struct notifier_block *nb, +unsigned long val,void *data) { +struct mce *m = (struct mce *)data; +struct amdgpu_device *adev =NULL ;drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev)
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10
-0+uint32_t gpu_id =0 ; +uint32_t umc_inst =0 ; +uint32_t chan_index =0 ; +struct ras_err_data err_data = {0 ,0 ,0 ,NULL }; +struct eeprom_table_record err_rec; +uint64_t retired_page; + +/* +* If the error was generated in UMC_V2, which belongs to GPU +UMCs, Why does it matter if the error is a v2 UMC generated one?
Quoted 3 lines
IOW, can you detect the type of errors in GPU memory - I assume this is what this is supposed to handle - from the error signature itself instead of doing
is_smca_umc_v2?
SMCA UMCv2 corresponds to GPU's UMC MCA bank and the GPU driver is only interested in errors on GPU UMC. We cannot know this without is_smca_umc_v2.
Quoted 33 lines
Quoted 17 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +16
-0+* and error occurred in DramECC (Extended error code = 0) then only +* process the error, else bail out. +*/ +if (!m || !(is_smca_umc_v2 (m->bank) && (XEC (m->status,0x1f ) ==0x0 ))) +return NOTIFY_DONE ; + + gpu_id =GET_MCA_IPID_GPUID (m->ipid); + +/* +* GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. +*/ + gpu_id -=GPU_ID_OFFSET ; + + adev =find_adev (gpu_id); +if (!adev) { + dev_warn(adev->dev, "%s: Unable to find adev for gpu_id: %d\n",drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2525
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +10
-0+ __func__, gpu_id); +return NOTIFY_DONE ; + } + +/* +* If it is correctable error, then print a message and return. +*/ +if (mce_is_correctable (m)) { +dev_info (adev->dev,"%s: UMC Correctable error detected." , + __func__);So put yourself in the shoes of a support engineer who starts getting all those calls about people's hardware getting correctable errors and should they replace it and should AMD RMA the GPUs and so on and so on..? Do you really wanna be on the receiving side of that call?
Quoted 3 lines
IOW, whom does printing the fact that the GPU had a corrected error which got corrected by the hardware, help and do you really want to upset people needlessly?
Agree. I will remove this debug print.
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2533
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +8
-0+return NOTIFY_OK ; + } + +/* +* If it is uncorrectable error, then find out UMC instance and +* channel index. +*/ +find_umc_inst_chan_index (m, &umc_inst, &chan_index);
Quoted 2 lines
That's a void function but it could return a pointer to the instance and channel bundled in a struct maybe...
Maybe. I hope its not too much of a concern if it stays the way it is.
Quoted 31 lines
Quoted 31 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +30
-0+ +dev_info (adev->dev,"Uncorrectable error detected in UMC inst: %d," +" chan_idx: %d" , umc_inst, chan_index); + +memset (&err_rec,0x0 ,sizeof (struct eeprom_table_record )); + +/* +* Translate UMC channel address to Physical address +*/ + retired_page =ADDR_OF_8KB_BLOCK (m->addr) | +ADDR_OF_256B_BLOCK (chan_index) | +OFFSET_IN_256B_BLOCK (m->addr); + + err_rec.address = m->addr; + err_rec.retired_page = retired_page >>AMDGPU_GPU_PAGE_SHIFT ; + err_rec.ts = (uint64_t )ktime_get_real_seconds (); + err_rec.err_type =AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE ; + err_rec.cu =0 ; + err_rec.mem_channel = chan_index; + err_rec.mcumc_id = umc_inst; + + err_data.err_addr = &err_rec; + err_data.err_addr_cnt =1 ; + +if (amdgpu_bad_page_threshold !=0 ) { +amdgpu_ras_add_bad_pages (adev, err_data.err_addr, + err_data.err_addr_cnt); +amdgpu_ras_save_bad_pages (adev); + } +
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2569
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +6
-0+return NOTIFY_OK ; +} + +static struct notifier_block amdgpu_bad_page_nb = { + .notifier_call = amdgpu_bad_page_notifier, + .priority =MCE_PRIO_ACCEL ,Nothing ever explains why this needs to be a separate priority. So how about it?
I wasn't really sure if I should use the EDAC priority here or create a new one for Accelerator devices. I thought using EDAC priority might not be accepted by the maintainers as EDAC and GPU (Accelerator) devices are two different class of devices.
That is the reason I create a new one.
I am OK to use EDAC priority if that is acceptable.
Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c void amdgpu_release_ras_context(struct amdgpu_device *adev) · line 2578
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +9
-0+}; + +static void amdgpu_register_bad_pages_mca_notifier (void ) +{ +/* +* Register the x86 notifier with MCE subsystem. +* Please note a notifier can be registered only once +* with the MCE subsystem. +*/ Why do you need this? Other users don't need that. Do you need to call
mce_unregister_decode_chain()when the driver gets removed or so?
Quoted 3 lines
Quoted 1 line
+ if (
notifier_registered== false) {This is just silly and should be fixed properly once the issue is understood.
A system can have multiple GPUs and we only want a single notifier registered. I will change the comment to explicitly state this.
Thanks,
Mukul
Quoted 5 lines
--
Regards/Gruss,
Boris.
Quoted 7 lines
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&data=04%7C01%7Cmukul.joshi%40amd.com%7C1c231207786 446018e7208d915297942%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637564090158211378%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata =7LSOnoJWrTf5z96YFACxuRKZP1z4E4zkvtrNzjbTaPs%3D&reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 2 lines · Mukul
SMCA UMCv2 corresponds to GPU's UMC MCA bank and the GPU driver is only interested in errors on GPU UMC.
So that thing should be called SMCA_GPU_UMC not SMCA_UMC_V2.
Quoted 1 line
We cannot know this without
is_smca_umc_v2.
You don't need it - just export smca_get_bank_type() and test the bank type at the call site.
Quoted 1 line
Maybe. I hope its not too much of a concern if it stays the way it is.
That was just a suggestion anyway - it is not code I maintain so not my call.
Quoted 5 lines
I wasn't really sure if I should use the EDAC priority here or create a new one for Accelerator devices. I thought using EDAC priority might not be accepted by the maintainers as EDAC and GPU (Accelerator) devices are two different class of devices.
That is the reason I create a new one.
I am OK to use EDAC priority if that is acceptable.
I don't know what's acceptable because I still am unclear as to what that thing is supposed to do. It seems you are interested only in uncorrectable errors.
How are those errors reported? #MC exception, deferred interrupt, simply logged in the bank and we find them by polling?
Then, the commit message is talking about some "bad page retirement". What does that do? What can the user do when she sees the
"Uncorrectable error detected in UMC..."
message?
It depends on what "retiring" of GPU pages means...
In any case, dmesg should issue a human-understandable message about the recovery action being done and what that means for the user: should she replace the GPU, should she ignore, etc, etc.
Quoted 2 lines
A system can have multiple GPUs and we only want a single notifier registered. I will change the comment to explicitly state this.
Actually, the notifier registration should be able to return a different retval to state that a callback has already been registered but that warns only currently so I'm guessing we're stuck with such ugly "workarounds" for their shortcomings.
I'm gonna take a look whether they can be fixed though so that you don't have to do this notifier_registered thing.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette[AMD Official Use Only - Internal Distribution Only]
Quoted 15 lines
-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Wednesday, May 12, 2021 5:06 PM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran[CAUTION: External Email]
Quoted 2 lines · Mukul
SMCA UMCv2 corresponds to GPU's UMC MCA bank and the GPU driver is only interested in errors on GPU UMC.
Quoted 6 lines
So that thing should be called
SMCA_GPU_UMCnotSMCA_UMC_V2.Quoted 1 line
We cannot know this without
is_smca_umc_v2.You don't need it - just export
smca_get_bank_type()and test the bank type at the call site.
Exporting smca_get_bank_type() works fine when CONFIG_X86_MCE_AMD is defined. I would need to put #ifdef CONFIG_X86_MCE_AMD in my code to compile the amdgpu driver when CONFIG_X86_MCE_AMD is not defined.
I can avoid all that by using is_smca_umc_v2().
I think it would be cleaner with using is_smca_umc_v2().
Quoted 6 lines
Quoted 2 lines
Maybe. I hope its not too much of a concern if it stays the way it is.
That was just a suggestion anyway - it is not code I maintain so not my call.
Quoted 1 line
I wasn't really sure if I should use the EDAC priority here or create a new one
Quoted 9 lines
for Accelerator devices.
Quoted 4 lines
I thought using EDAC priority might not be accepted by the maintainers as EDAC and GPU (Accelerator) devices are two different class of devices. That is the reason I create a new one.
I am OK to use EDAC priority if that is acceptable.I don't know what's acceptable because I still am unclear as to what that thing is supposed to do. It seems you are interested only in uncorrectable errors.
You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.
Quoted 3 lines
How are those errors reported? #MC exception, deferred interrupt, simply logged in the bank and we find them by polling?
It is a deferred interrupt that generates an MCE.
Quoted 5 lines
Then, the commit message is talking about some "bad page retirement". What does that do? What can the user do when she sees the
"Uncorrectable error detected in UMC..."
Quoted 8 lines
message?
It depends on what "retiring" of GPU pages means...
In any case, dmesg should issue a human-understandable message about the recovery action being done and what that means for the user: should she replace the GPU, should she ignore, etc, etc.
When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.
By retiring, we mean we reserve the page so that it is not available for allocations to any applications.
We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.
Hope it clears what "bad page retirement" is achieving.
Thanks,
Mukul
Quoted 14 lines
Quoted 2 lines
A system can have multiple GPUs and we only want a single notifier registered. I will change the comment to explicitly state this.
Actually, the notifier registration should be able to return a different retval to state that a callback has already been registered but that warns only currently so I'm guessing we're stuck with such ugly "workarounds" for their shortcomings.
I'm gonna take a look whether they can be fixed though so that you don't have to do this
notifier_registeredthing.--
Regards/Gruss,
Boris.
Quoted 8 lines
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&data=04%7C01%7CMukul.Joshi%40amd.com%7C5e305845504 14bc53c3008d91589b50b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637564503481206042%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata =fF2iaSQxAREAV2OwLi%2F8cN9uRuqNNKim2FpWJUBQS98%3D&reserved= 0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 5 lines · Mukul
Exporting
smca_get_bank_type()works fine whenCONFIG_X86_MCE_AMDis defined. I would need to put #ifdefCONFIG_X86_MCE_AMDin my code to compile the amdgpu driver whenCONFIG_X86_MCE_AMDis not defined.
I can avoid all that by usingis_smca_umc_v2().
I think it would be cleaner with usingis_smca_umc_v2().
See how smca_get_long_name() is exported and export that function the same way.
To save you some energy: is_smca_umc_v2() is not going to happen.
Quoted 2 lines
You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.
An EDAC "device", as you call it, is not interested in handling UEs. If anything, it counts them.
Quoted 1 line
It is a deferred interrupt that generates an MCE.
Is that the same deferred interrupt which calls amd_deferred_error_interrupt() ?
Quoted 3 lines
When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.
What page is that? Normal DRAM page or a page in some special GPU memory?
Quoted 2 lines
By retiring, we mean we reserve the page so that it is not available for allocations to any applications.
We do that for normal DRAM memory pages by poisoning them. I hope you don't mean that.
Looking at
amdgpu_ras_add_bad_pages
|-> amdgpu_vram_mgr_reserve_range
that's some VRAM thing so I'm guessing special memory on the GPU.
If so, what happens with all those "retired" pages when you reboot? They're getting used again and potentially trigger the same UEs and the same retiring happens?
Quoted 3 lines
We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.
Ok, I'm a user and I can access that information through sysfs. What can I do with it?
Quoted 1 line
Hope it clears what "bad page retirement" is achieving.
It is getting there.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteQuoted 21 lines · Borislav Petkov
Quoted 5 lines · Mukul
Exporting
smca_get_bank_type()works fine whenCONFIG_X86_MCE_AMDis defined. I would need to put #ifdefCONFIG_X86_MCE_AMDin my code to compile the amdgpu driver whenCONFIG_X86_MCE_AMDis not defined.
I can avoid all that by usingis_smca_umc_v2().
I think it would be cleaner with usingis_smca_umc_v2().See how
smca_get_long_name()is exported and export that function the same way.To save you some energy:
is_smca_umc_v2()is not going to happen.Quoted 2 lines
You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.
An EDAC "device", as you call it, is not interested in handling UEs. If anything, it counts them.
Quoted 1 line
It is a deferred interrupt that generates an MCE.
Quoted 8 lines
Is that the same deferred interrupt which calls
amd_deferred_error_interrupt()?Quoted 3 lines
When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.
What page is that? Normal DRAM page or a page in some special GPU memory?
GPU memory.
Quoted 6 lines
Quoted 2 lines
By retiring, we mean we reserve the page so that it is not available for allocations to any applications.
We do that for normal DRAM memory pages by poisoning them. I hope you don't mean that.
Quoted 7 lines
Looking at
amdgpu_ras_add_bad_pages
|->amdgpu_vram_mgr_reserve_rangethat's some VRAM thing so I'm guessing special memory on the GPU.
Yes.
Quoted 3 lines
If so, what happens with all those "retired" pages when you reboot? They're getting used again and potentially trigger the same UEs and the same retiring happens?
The bad pages are stored in an EEPROM on the board and the next time the driver loads it reads the EEPROM so that it can reserve the bad pages at init time so they don't get used again.
Alex
Quoted 18 lines
Quoted 4 lines
We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.
Ok, I'm a user and I can access that information through sysfs. What can I do with it?
Quoted 1 line
Hope it clears what "bad page retirement" is achieving.
It is getting there.
Thx.
--
Regards/Gruss,
Boris.
Quoted 5 lines
https://people.kernel.org/tglx/notes-about-netiquette
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 3 lines · Alex Deucher
The bad pages are stored in an EEPROM on the board and the next time the driver loads it reads the EEPROM so that it can reserve the bad pages at init time so they don't get used again.
And that works automagically on the next boot? Because that sounds like the right thing to do.
So practically, what happens to a GPU in such a case where the VRAM starts going bad? It might get exhausted eventually and the driver will say something along the lines of:
"VRAM bad pages: 80%, consider replacing the GPU. It is operating currently with degrated performance."
or so?
Yap, from a RAS perspective, that makes good sense as you're prolonging the life of the component while still remains operational as good as it can and the only user interaction you need is she/he replacing it.
Sounds good.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteQuoted 8 lines · Borislav Petkov
Quoted 3 lines · Alex Deucher
The bad pages are stored in an EEPROM on the board and the next time the driver loads it reads the EEPROM so that it can reserve the bad pages at init time so they don't get used again.
And that works automagically on the next boot? Because that sounds like the right thing to do.
Yes, or driver reload, suspend/resume, etc.
Quoted 9 lines
So practically, what happens to a GPU in such a case where the VRAM starts going bad? It might get exhausted eventually and the driver will say something along the lines of:
"VRAM bad pages: 80%, consider replacing the GPU. It is operating currently with degrated performance."
or so?
Right. The sys admin can query the bad page count and decide when to retire the card.
Quoted 6 lines
Yap, from a RAS perspective, that makes good sense as you're prolonging the life of the component while still remains operational as good as it can and the only user interaction you need is she/he replacing it.
Sounds good.
Yes. That's the idea.
Alex
Quoted 8 lines
Thx.
--
Regards/Gruss,
Boris.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 2 lines · Alex Deucher
Right. The sys admin can query the bad page count and decide when to retire the card.
Yap, although the driver should actively "tell" the sysadmin when some critical counts of retired VRAM pages are reached because I doubt all admins would go look at those counts on their own.
Btw, you say "admin" - am I to understand that those are some high end GPU cards with ECC memory? If consumer grade stuff has this too, then the driver should very much warn on such levels on its own because normal users won't know what and where to look.
Other than that, the big picture sounds good to me.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteQuoted 8 lines · Borislav Petkov
Quoted 2 lines · Alex Deucher
Right. The sys admin can query the bad page count and decide when to retire the card.
Yap, although the driver should actively "tell" the sysadmin when some critical counts of retired VRAM pages are reached because I doubt all admins would go look at those counts on their own.
I think we print something in the log as well when we hit the threshold. I need to double check the code.
Quoted 6 lines
Btw, you say "admin" - am I to understand that those are some high end GPU cards with ECC memory? If consumer grade stuff has this too, then the driver should very much warn on such levels on its own because normal users won't know what and where to look.
Currently it's only available on workstation and datacenter boards.
Quoted 1 line
Other than that, the big picture sounds good to me.
Thanks!
Alex
Quoted 8 lines
Thx.
--
Regards/Gruss,
Boris.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
Quoted 19 lines
-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Thursday, May 13, 2021 10:58 AM
To: Alex Deucher <[email protected]>
Cc: Joshi, Mukul <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <linux- [email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran[CAUTION: External Email]
Quoted 2 lines · Alex Deucher
Right. The sys admin can query the bad page count and decide when to retire the card.
Yap, although the driver should actively "tell" the sysadmin when some critical counts of retired VRAM pages are reached because I doubt all admins would go look at those counts on their own.
Quoted 7 lines
Btw, you say "admin" - am I to understand that those are some high end GPU cards with ECC memory? If consumer grade stuff has this too, then the driver should very much warn on such levels on its own because normal users won't know what and where to look.
Other than that, the big picture sounds good to me.
Since now you are OK with how page retirement works, lets revisit the original question.
Are you OK with a new MCE priority (MCE_PRIO_ACCEL) or do you want us to use something else?
Thanks,
Mukul
Quoted 6 lines
Thx.
--
Regards/Gruss,
Boris.
Quoted 8 lines
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&data=04%7C01%7CMukul.Joshi%40amd.com%7C50588f11ed5 3456b03e008d9161f765c%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637565146658376385%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata =Es0FMDNzNEKgxvFiqe1kOo9aEPK6%2BOXrhI5aWs3QH9Q%3D&reserved= 0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 2 lines · Mukul
Are you OK with a new MCE priority (
MCE_PRIO_ACCEL) or do you want us to use something else?
I still don't know why a separate priority is needed. Maybe this still needs answering:
Quoted 1 line
It is a deferred interrupt that generates an MCE.
Is that the same deferred interrupt which callsamd_deferred_error_interrupt() ?
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette[AMD Official Use Only]
Quoted 15 lines
-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Friday, May 14, 2021 3:04 AM
To: Joshi, Mukul <[email protected]>
Cc: Alex Deucher <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <linux- [email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran[CAUTION: External Email]
Quoted 2 lines · Mukul
Are you OK with a new MCE priority (
MCE_PRIO_ACCEL) or do you want us to use something else?
Quoted 7 lines
I still don't know why a separate priority is needed. Maybe this still needs answering:
Quoted 1 line
It is a deferred interrupt that generates an MCE.
Is that the same deferred interrupt which calls
amd_deferred_error_interrupt()?
Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls amd_deferred_error_interrupt().
Thanks,
Mukul
Quoted 5 lines
--
Regards/Gruss,
Boris.
Quoted 7 lines
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&data=04%7C01%7CMukul.Joshi%40amd.com%7C7e04d0ad3c6 147a8dfd008d916a66bbc%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637565726326540654%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata =AWHJvqIbtkGl6OPC86alAtnxsleq6UVe9mcM1MjxpUQ%3D&reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On Thu, May 27, 2021 at 03:54:27PM -0400, Joshi, Mukul wrote:
...
Quoted 6 lines
Quoted 2 lines
Is that the same deferred interrupt which calls
amd_deferred_error_interrupt()?Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls
amd_deferred_error_interrupt().
Mukul,
Do you expect that the driver will need to mark pages with high correctable error counts as bad? I think the hardware folks may want the GPU memory errors to be handled more aggressively than CPU memory errors. The specific threshold may change from product to product, so it may make sense to hardcode this in the driver.
We have similar functionality in the Correctable Errors Collector. But enterprise users may prefer a direct approach done in the driver (based on the hardware experts' guidance) instead of configuring the kernel at runtime.
So I think having a separate priority may make sense if some special functionality, or combination of behaviors, is needed which don't fall under any exisiting things. In this case, "special functionality" could be that the GPU memory needs to be handled differently than CPU memory.
Another thing is that this behavior is similar to the NFIT behavior, i.e. there's a memory error on an external device that needs to be handled by the device's driver. So maybe we can rename MCE_PRIO_NFIT to be generic (MCE_PRIO_EXTERNAL?) and use that? Multiple notifiers with the same priority is okay, right?
Thanks,
Yazen
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only]
Quoted 20 lines
-----Original Message-----
From: Ghannam, Yazen <[email protected]>
Sent: Thursday, June 3, 2021 5:13 PM
To: Joshi, Mukul <[email protected]>
Cc: Borislav Petkov <[email protected]>; Alex Deucher <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for AldebaranOn Thu, May 27, 2021 at 03:54:27PM -0400, Joshi, Mukul wrote:
...Quoted 5 lines
Quoted 2 lines
Is that the same deferred interrupt which calls
amd_deferred_error_interrupt()?Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls
amd_deferred_error_interrupt().Quoted 1 line
Quoted 8 lines
Mukul,
Do you expect that the driver will need to mark pages with high correctable error counts as bad? I think the hardware folks may want the GPU memory errors to be handled more aggressively than CPU memory errors. The specific threshold may change from product to product, so it may make sense to hardcode this in the driver.
Sorry I missed this email completely. Just saw it so responding now.
At the moment, we don't have a requirement to mark a page "bad" if there is a high correctable error counts. Our previous GPU ASICs which support RAS, also do not have such a feature.
But you make a good point. It might be worthwhile to go and ask the hardware folks about it.
Quoted 9 lines
We have similar functionality in the Correctable Errors Collector. But enterprise users may prefer a direct approach done in the driver (based on the hardware experts' guidance) instead of configuring the kernel at runtime.
So I think having a separate priority may make sense if some special functionality, or combination of behaviors, is needed which don't fall under any exisiting things. In this case, "special functionality" could be that the GPU memory needs to be handled differently than CPU memory.
Quoted 6 lines
Another thing is that this behavior is similar to the NFIT behavior, i.e. there's a memory error on an external device that needs to be handled by the device's driver. So maybe we can rename
MCE_PRIO_NFITto be generic (MCE_PRIO_EXTERNAL?) and use that? Multiple notifiers with the same priority is okay, right?
With respect to MCE priority, I was thinking of using the MCE_PRIO_EDAC instead of creating a new priority as the code in the GPU driver is doing error detection and handling the uncorrectable errors. Not sure if that aligns with the definition of EDAC device in the kernel.
What do you think?
Regards,
Mukul
Quoted 2 lines
Thanks,
Yazen
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only]
Quoted 78 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Joshi, Mukul
Sent: Thursday, July 29, 2021 8:00 PM
To: Ghannam, Yazen <[email protected]>
Cc: x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <[email protected]>; [email protected]; Borislav Petkov <[email protected]>; Alex Deucher <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran[CAUTION: External Email]
[AMD Official Use Only]
Quoted 21 lines
-----Original Message-----
From: Ghannam, Yazen <[email protected]>
Sent: Thursday, June 3, 2021 5:13 PM
To: Joshi, Mukul <[email protected]>
Cc: Borislav Petkov <[email protected]>; Alex Deucher <[email protected]>; x86-ml <[email protected]>; Kasiviswanathan, Harish <[email protected]>; lkml <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for AldebaranOn Thu, May 27, 2021 at 03:54:27PM -0400, Joshi, Mukul wrote:
...Quoted 5 lines
Quoted 2 lines
Is that the same deferred interrupt which calls
amd_deferred_error_interrupt()?Sorry picking this up after sometime. I thought I had replied to this email. Yes it is the same deferred interrupt which calls
amd_deferred_error_interrupt().Quoted 1 line
Quoted 8 lines
Mukul,
Do you expect that the driver will need to mark pages with high correctable error counts as bad? I think the hardware folks may want the GPU memory errors to be handled more aggressively than CPU memory errors. The specific threshold may change from product to product, so it may make sense to hardcode this in the driver.
Sorry I missed this email completely. Just saw it so responding now.
At the moment, we don't have a requirement to mark a page "bad" if there is a high correctable error counts.
Our previous GPU ASICs which support RAS, also do not have such a feature. But you make a good point. It might be worthwhile to go and ask the hardware folks about it.Quoted 3 lines
We have similar functionality in the Correctable Errors Collector. But enterprise users may prefer a direct approach done in the driver (based on the hardware experts' guidance) instead of configuring the kernel at
runtime.
Quoted 5 lines
So I think having a separate priority may make sense if some special functionality, or combination of behaviors, is needed which don't fall under any exisiting things. In this case, "special functionality" could be that the GPU memory needs to be handled differently than CPU
memory.
Quoted 8 lines
Another thing is that this behavior is similar to the NFIT behavior, i.e. there's a memory error on an external device that needs to be handled by the device's driver. So maybe we can rename
MCE_PRIO_NFITto be generic
(MCE_PRIO_EXTERNAL?) and use that? Multiple notifiers with the same priority is okay, right?With respect to MCE priority, I was thinking of using the
MCE_PRIO_EDACinstead of creating a new priority as the code in the GPU driver is doing error detection and handling the uncorrectable errors.
Not sure if that aligns with the definition of EDAC device in the kernel.
Quoted 5 lines
What do you think?
Regards,
Mukul
After talking to Yazen, MCE_PRIO_UC might be a better choice for the MCE priority as we are dealing
only with uncorrectable errors.
I will be sending out a v2 patch with changes to use the MCE_PRIO_UC and drop the MCE_PRIO_ACCEL and see what the feedback is.
Thanks,
Mukul
Quoted 12 lines
Quoted 2 lines
Thanks,
Yazen_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.fre
edesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&data=04%7C01%7Cmukul.joshi%40amd.com%7C7d32897fddef448ab0 aa08d952ecf41f%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376 31999953383488%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJ QIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=YWZz9 OYTMOhBl4183kV5ZYj01yw0xwNj%2BjTdXejFKH8%3D&reserved=0
[AMD Official Use Only - Internal Distribution Only]
Quoted 13 lines
-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Thursday, May 13, 2021 5:53 AM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran[CAUTION: External Email]
Quoted 1 line · Mukul
Exporting
smca_get_bank_type()works fine whenCONFIG_X86_MCE_AMDis
Quoted 9 lines
defined.
Quoted 4 lines
I would need to put #ifdef
CONFIG_X86_MCE_AMDin my code to compile the amdgpu driver whenCONFIG_X86_MCE_AMDis not defined. I can avoid all that by usingis_smca_umc_v2().
I think it would be cleaner with usingis_smca_umc_v2().See how
smca_get_long_name()is exported and export that function the same way.
That's probably not the best example to look at. smca_get_long_name() is used in drivers/edac/mce_amd.c and this file doesn't get compiled when CONFIG_X86_MCE_AMD is not defined.
And amdgpu driver has no dependency on CONFIG_X86_MCE_AMD.
So here is one option that we can try:
1. Export smca_get_bank_type().
2. I wrap my entire code in GPU driver with #ifdef CONFIG_X86_MCE_AMD
Will that work for you?
Thanks,
Mukul
Quoted 54 lines
To save you some energy:
is_smca_umc_v2()is not going to happen.Quoted 2 lines
You can think of GPU device as a EDAC device here. It is mainly interested in handling uncorrectable errors.
An EDAC "device", as you call it, is not interested in handling UEs. If anything, it counts them.
Quoted 1 line
It is a deferred interrupt that generates an MCE.
Is that the same deferred interrupt which calls
amd_deferred_error_interrupt()?Quoted 3 lines
When an uncorrectable error is detected on the GPU UMC, all we are doing is determining the physical address where the error occurred and then "retiring" the page that address belongs to.
What page is that? Normal DRAM page or a page in some special GPU memory?
Quoted 2 lines
By retiring, we mean we reserve the page so that it is not available for allocations to any applications.
We do that for normal DRAM memory pages by poisoning them. I hope you don't mean that.
Looking at
amdgpu_ras_add_bad_pages
|->amdgpu_vram_mgr_reserve_rangethat's some VRAM thing so I'm guessing special memory on the GPU.
If so, what happens with all those "retired" pages when you reboot? They're getting used again and potentially trigger the same UEs and the same retiring happens?
Quoted 3 lines
We are providing information to the user by storing all the information about the retired pages in EEPROM. This can be accessed through sysfs.
Ok, I'm a user and I can access that information through sysfs. What can I do with it?
Quoted 1 line
Hope it clears what "bad page retirement" is achieving.
It is getting there.
Thx.
--
Regards/Gruss,
Boris.
Quoted 8 lines
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&data=04%7C01%7CMukul.Joshi%40amd.com%7Cd8c660fce3a2 4ce3c6d408d915f4efa6%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0% 7C637564964013263414%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwM DAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata= %2BnJ%2B99N%2FRljoHGALimZHZG%2Bmf9jL5zP2eA44I6pbzFY%3D&reser ved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 1 line · Mukul
That's probably not the best example to look at.
Oh, it is the *perfect* example but...
Quoted 4 lines
smca_get_long_name()is used indrivers/edac/mce_amd.cand this file doesn't get compiled whenCONFIG_X86_MCE_AMDis not defined.And amdgpu driver has no dependency on
CONFIG_X86_MCE_AMD.
... maybe this will make you see it the right way: how much of the amdgpu RAS functionality you're adding, is going to even function without CONFIG_X86_MCE_AMD?
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette[AMD Official Use Only - Internal Distribution Only]
Quoted 9 lines
-----Original Message-----
From: Borislav Petkov <[email protected]>
Sent: Friday, May 14, 2021 3:06 AM
To: Joshi, Mukul <[email protected]>
Cc: [email protected]; Kasiviswanathan, Harish <[email protected]>; x86-ml <[email protected]>; lkml <linux- [email protected]>
Subject: Re: [PATCH] drm/amdgpu: Register bad page handler for Aldebaran
Quoted 7 lines
[CAUTION: External Email]
Quoted 1 line · Mukul
That's probably not the best example to look at.
Oh, it is the *perfect* example but...
I would prefer not to get into an argument of if it is the perfect example or not. I would prefer to get the job done and move forward.
Quoted 9 lines
Quoted 4 lines
smca_get_long_name()is used indrivers/edac/mce_amd.cand this file doesn't get compiled whenCONFIG_X86_MCE_AMDis not defined.And amdgpu driver has no dependency on
CONFIG_X86_MCE_AMD.... maybe this will make you see it the right way: how much of the amdgpu RAS functionality you're adding, is going to even function without
CONFIG_X86_MCE_AMD?
We have RAS functionality in other ASICs that is not dependent on CONFIG_X86_MCE_AMD. So, I don't think we would want to do that just for one ASIC.
Maybe Alex D. has an opinion on it.
As I had mentioned in my previous comment, I am fine with wrapping around my code with #ifdef CONFIG_X86_MCE_AMD and exporting smca_get_bank_type().
I hope we can move forward with this approach.
Thanks,
Mukul
Quoted 4 lines
--
Regards/Gruss,
Boris.
Quoted 8 lines
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.
kernel.org%2Ftglx%2Fnotes-about- netiquette&data=04%7C01%7CMukul.Joshi%40amd.com%7C7fded9165e6 64dde943808d916a6b33f%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0 %7C637565727509040995%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata =5hNyAEla4H%2BN0kMdCDs6iL0Dr8mVQJNxKA0UXJU7%2F8c%3D&reserv ed=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 3 lines · Mukul
We have RAS functionality in other ASICs that is not dependent on
CONFIG_X86_MCE_AMD. So, I don't think we would want to do that just for one ASIC.
Lemme try again: you said that those errors do get reported through a deferred interrupt. Which is likely amd_deferred_error_interrupt().
If it is that interrupt and you don't have CONFIG_X86_MCE_AMD enabled, then you won't get any errors reported and your RAS functionality will simply sit there inactive.
So if that above is true - something to which I'm still not getting an answer but maybe one fine day... - so if that above is true, your RAS functionality *needs* CONFIG_X86_MCE_AMD to be enabled in order to *actually* function.
So you *must* make your RAS functionality depend on CONFIG_X86_MCE_AMD
- otherwise no deferred interrupts and no errors reported. It is that simple.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette