drm/amdgpu: add mode2 reset for sienna_cichlid
PATCH 1/5 drm/amdgpu: add mode2 reset for sienna_cichlid
To meet the requirement for multi container usecase which needs a quicker reset and not causing VRAM lost, adding the Mode2 reset handler for sienna_cichlid. Adding a AMDGPU_SKIP_MODE2_RESET flag so driver can fallback to default reset method when mode2 reset failed and retry.
- add mode2 reset handler for sienna_cichlid
- introduce AMDGPU_SKIP_MODE2_RESET flag
- let mode2 reset fallback to default reset method if failed
--- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 13 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c | 1 + drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 297 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h | 32 ++ .../pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h | 4 +- drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h | 3 +- .../amd/pm/swsmu/smu11/sienna_cichlid_ppt.c | 54 ++++ 15 files changed, 414 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c create mode 100644 drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
@@ -75,7 +75,7 @@ amdgpu-y += \ vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \ vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o arct_reg_init.o mxgpu_nv.o \ nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o \ - nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o + sienna_cichlid.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o # add DF block amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -0
@@ -135,6 +135,7 @@ static void amdgpu_amdkfd_reset_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -1
@@ -5146,6 +5146,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, reset_context->job = job; reset_context->hive = hive; +/* * Build list of devices to reset. * In case we are in XGMI hive mode, resort the device list
@@ -5265,8 +5266,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,amdgpu_ras_resume (adev); }else { r =amdgpu_do_asic_reset (device_list_handle, reset_context); -if (r && r == -EAGAIN ) +if (r && r == -EAGAIN ) { +set_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags); + adev->asic_reset_res =0 ; goto retry; + } } skip_hw_reset:
@@ -5694,6 +5698,7 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev;set_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags);set_bit (AMDGPU_SKIP_HW_RESET , &reset_context.flags); +set_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags); adev->no_hw_access = true; r =amdgpu_device_pre_asic_reset (adev, &reset_context);
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -0
@@ -71,6 +71,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags); r =amdgpu_device_gpu_recover (ring->adev, job, &reset_context);if (r)
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +1 -0
@@ -1949,6 +1949,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); }
drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +13 -0
@@ -23,6 +23,7 @@#include "amdgpu_reset.h" #include "aldebaran.h" +#include "sienna_cichlid.h" int amdgpu_reset_add_handler (struct amdgpu_reset_control *reset_ctl,struct amdgpu_reset_handler *handler)
@@ -40,6 +41,9 @@ int amdgpu_reset_init(struct amdgpu_device *adev)case IP_VERSION (13 ,0 ,2 ): ret =aldebaran_reset_init (adev);break ; +case IP_VERSION (11 ,0 ,7 ): + ret =sienna_cichlid_reset_init (adev); +break ;default :break ; }
@@ -55,6 +59,9 @@ int amdgpu_reset_fini(struct amdgpu_device *adev)case IP_VERSION (13 ,0 ,2 ): ret =aldebaran_reset_fini (adev);break ; +case IP_VERSION (11 ,0 ,7 ): + ret =sienna_cichlid_reset_fini (adev); +break ;default :break ; }
@@ -67,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags)) +return -ENOSYS ; +if (adev->reset_cntl && adev->reset_cntl->get_reset_handler) reset_handler = adev->reset_cntl->get_reset_handler ( adev->reset_cntl, reset_context);
@@ -83,6 +93,9 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,int ret;struct amdgpu_reset_handler *reset_handler =NULL ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags)) +return -ENOSYS ; +if (adev->reset_cntl) reset_handler = adev->reset_cntl->get_reset_handler ( adev->reset_cntl, reset_context);
drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h +1 -0
@@ -30,6 +30,7 @@ enum AMDGPU_RESET_FLAGS {AMDGPU_NEED_FULL_RESET =0 ,AMDGPU_SKIP_HW_RESET =1 , + AMDGPU_SKIP_MODE2_RESET = 2, }; struct amdgpu_reset_context {
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -0
@@ -290,6 +290,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1 -0
@@ -317,6 +317,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +1 -0
@@ -529,6 +529,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +297 -0
297 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +* Copyright 2021 Advanced Micro Devices, Inc. +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +* OTHER DEALINGS IN THE SOFTWARE. +* +*/ + +#ifndef __SIENNA_CICHLID_H__ +#define __SIENNA_CICHLID_H__ + +#include "amdgpu.h" + +int sienna_cichlid_reset_init (struct amdgpu_device *adev); +int sienna_cichlid_reset_fini (struct amdgpu_device *adev); + +#endif
drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h +2 -2
@@ -137,7 +137,7 @@#define PPSMC_MSG_DisallowGpo 0x56#define PPSMC_MSG_Enable2ndUSB20Port 0x57 - -#define PPSMC_Message_Count 0x58 +#define PPSMC_MSG_DriverMode2Reset 0x5D +#define PPSMC_Message_Count 0x5E#endif
drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h +2 -1
@@ -235,7 +235,8 @@__SMU_DUMMY_MAP (UnforceGfxVid), \__SMU_DUMMY_MAP (HeavySBR), \__SMU_DUMMY_MAP (SetBadHBMPagesRetiredFlagsPerChannel), \ -__SMU_DUMMY_MAP (EnableGfxImu), +__SMU_DUMMY_MAP (EnableGfxImu), \ +__SMU_DUMMY_MAP (DriverMode2Reset), #undef __SMU_DUMMY_MAP#define __SMU_DUMMY_MAP (type) SMU_MSG_##type
drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +54 -0
72 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH 2/5 drm/amdgpu: add debugfs amdgpu_reset_level
Introduce amdgpu_reset_level debugfs in order to help debug and test specific type of reset. Also helps blocking unwanted type of resets.
By default, mode2 reset will not be enabled
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 20 ++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 3 +++ 5 files changed, 34 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +4 -0
@@ -238,6 +238,7 @@ extern int amdgpu_si_support;extern int amdgpu_cik_support;#endif extern int amdgpu_num_kcq; +extern uint amdgpu_reset_level_mask;#define AMDGPU_VCNFW_LOG_SIZE (32 * 1024)extern int amdgpu_vcnfw_log;
@@ -274,6 +275,9 @@ extern int amdgpu_vcnfw_log;#define AMDGPU_RESET_VCE (1 << 13)#define AMDGPU_RESET_VCE1 (1 << 14) +#define AMDGPU_RESET_LEVEL_SOFT_RECOVERY (1 << 0) +#define AMDGPU_RESET_LEVEL_MODE2 (1 << 1) +/* max cursor sizes (in pixels) */ #define CIK_CURSOR_WIDTH 128#define CIK_CURSOR_HEIGHT 128
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +20 -0
@@ -1661,12 +1661,29 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val)return ret; } +static int amdgpu_debugfs_reset_level_get (void *data,u64 *val) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)data; + *val = amdgpu_reset_level_mask; +return 0 ; +} + +static int amdgpu_debugfs_reset_level_set (void *data,u64 val) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)data; + amdgpu_reset_level_mask = val; +return 0 ; +} +DEFINE_DEBUGFS_ATTRIBUTE (fops_ib_preempt,NULL , amdgpu_debugfs_ib_preempt,"%llu\n" );DEFINE_DEBUGFS_ATTRIBUTE (fops_sclk_set,NULL , amdgpu_debugfs_sclk_set,"%llu\n" ); +DEFINE_DEBUGFS_ATTRIBUTE (fops_reset_level, amdgpu_debugfs_reset_level_get, + amdgpu_debugfs_reset_level_set,"%llu\n" ); +static ssize_t amdgpu_reset_dump_register_list_read (struct file *f,char __user *buf,size_t size,loff_t *pos) {
@@ -1785,6 +1802,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)return PTR_ERR (ent); } +debugfs_create_file ("amdgpu_reset_level" ,0200 , root, adev, + &fops_reset_level); +/* Register debugfs entries for amdgpu_ttm */ amdgpu_ttm_debugfs_init (adev);amdgpu_debugfs_pm_init (adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1 -0
@@ -198,6 +198,7 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .timeout_fatal_disable = false, .period = 0x0, /* default to 0x0 (timeout disable) */ }; +uint amdgpu_reset_level_mask =0x1 ;/** * DOC: vramlimit (int)
drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +6 -0
@@ -74,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;
@@ -93,6 +96,9 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,int ret;struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +3 -0
@@ -405,6 +405,9 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, {ktime_t deadline =ktime_add_us (ktime_get (),10000 ); +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_SOFT_RECOVERY )) +return false; +if (amdgpu_sriov_vf (ring->adev) || !ring->funcs->soft_recovery || !fence)return false;
Signature · 1 line
2.25.1
Christian König
maintainerRe: [PATCH 2/5] drm/amdgpu: add debugfs amdgpu_reset_level
Well NAK to the debugfs approach, stuff like that is usually a module parameter.
Apart from that this series needs to be reviewed by Andrey.
Regards,
Christian.
Am 22.07.22 um 09:34 schrieb Victor Zhao:
Quoted 131 lines
Introduce
amdgpu_reset_leveldebugfs in order to help debug and test specific type of reset. Also helps blocking unwanted type of resets.By default, mode2 reset will not be enabled
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 20 ++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 3 +++ 5 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +4
-0@@ -238,6 +238,7 @@ extern int amdgpu_si_support;extern int amdgpu_cik_support;#endif extern int amdgpu_num_kcq; +extern uint amdgpu_reset_level_mask;#define AMDGPU_VCNFW_LOG_SIZE (32 * 1024)extern int amdgpu_vcnfw_log;@@ -274,6 +275,9 @@ extern int amdgpu_vcnfw_log;#define AMDGPU_RESET_VCE (1 << 13)#define AMDGPU_RESET_VCE1 (1 << 14) +#define AMDGPU_RESET_LEVEL_SOFT_RECOVERY (1 << 0) +#define AMDGPU_RESET_LEVEL_MODE2 (1 << 1) +/* max cursor sizes (in pixels) */ #define CIK_CURSOR_WIDTH 128#define CIK_CURSOR_HEIGHT 128drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +20
-0@@ -1661,12 +1661,29 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val)return ret; } +static int amdgpu_debugfs_reset_level_get (void *data,u64 *val) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)data; + *val = amdgpu_reset_level_mask; +return 0 ; +} + +static int amdgpu_debugfs_reset_level_set (void *data,u64 val) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)data; + amdgpu_reset_level_mask = val; +return 0 ; +} +DEFINE_DEBUGFS_ATTRIBUTE (fops_ib_preempt,NULL , amdgpu_debugfs_ib_preempt,"%llu\n" );DEFINE_DEBUGFS_ATTRIBUTE (fops_sclk_set,NULL , amdgpu_debugfs_sclk_set,"%llu\n" ); +DEFINE_DEBUGFS_ATTRIBUTE (fops_reset_level, amdgpu_debugfs_reset_level_get, + amdgpu_debugfs_reset_level_set,"%llu\n" ); +static ssize_t amdgpu_reset_dump_register_list_read (struct file *f,char __user *buf,size_t size,loff_t *pos) {@@ -1785,6 +1802,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)return PTR_ERR (ent); } +debugfs_create_file ("amdgpu_reset_level" ,0200 , root, adev, + &fops_reset_level); +/* Register debugfs entries for amdgpu_ttm */ amdgpu_ttm_debugfs_init (adev);amdgpu_debugfs_pm_init (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1
-0@@ -198,6 +198,7 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .timeout_fatal_disable = false, .period = 0x0, /* default to 0x0 (timeout disable) */ }; +uint amdgpu_reset_level_mask =0x1 ;/** * DOC: vramlimit (int) drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +6
-0@@ -74,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;@@ -93,6 +96,9 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,int ret;struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +3
-0@@ -405,6 +405,9 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, {ktime_t deadline =ktime_add_us (ktime_get (),10000 ); +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_SOFT_RECOVERY )) +return false; +if (amdgpu_sriov_vf (ring->adev) || !ring->funcs->soft_recovery || !fence)return false;
RE: [PATCH 2/5] drm/amdgpu: add debugfs amdgpu_reset_level
[AMD Official Use Only - General]
Hi @Grodzovsky, Andrey,
Please help review the series, thanks a lot.
Hi @Koenig, Christian,
I thought a module parameter will be exposed to a common user, this control was added to help debug and test so I put it as a debugfs. I can make it module parameter if more appropriate.
Thanks,
Victor
-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, July 22, 2022 4:20 PM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deng, Emily <[email protected]>; Deucher, Alexander <[email protected]>
Subject: Re: [PATCH 2/5] drm/amdgpu: add debugfs amdgpu_reset_level
Well NAK to the debugfs approach, stuff like that is usually a module parameter.
Apart from that this series needs to be reviewed by Andrey.
Regards,
Christian.
Am 22.07.22 um 09:34 schrieb Victor Zhao:
Quoted 133 lines
Introduce
amdgpu_reset_leveldebugfs in order to help debug and test specific type of reset. Also helps blocking unwanted type of resets.By default, mode2 reset will not be enabled
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 20 ++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 3 +++ 5 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +4
-0@@ -238,6 +238,7 @@ extern int amdgpu_si_support;extern int amdgpu_cik_support;#endif extern int amdgpu_num_kcq; +extern uint amdgpu_reset_level_mask;#define AMDGPU_VCNFW_LOG_SIZE (32 * 1024)extern int amdgpu_vcnfw_log;@@ -274,6 +275,9 @@ extern int amdgpu_vcnfw_log;#define AMDGPU_RESET_VCE (1 << 13)#define AMDGPU_RESET_VCE1 (1 << 14) +#define AMDGPU_RESET_LEVEL_SOFT_RECOVERY (1 << 0) #define +AMDGPU_RESET_LEVEL_MODE2 (1 <<1 ) + /* max cursor sizes (in pixels) */#define CIK_CURSOR_WIDTH 128#define CIK_CURSOR_HEIGHT 128drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +18
-0@@ -1661,12 +1661,29 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val)return ret; } +static int amdgpu_debugfs_reset_level_get (void *data,u64 *val) { +struct amdgpu_device *adev = (struct amdgpu_device *)data; + *val = amdgpu_reset_level_mask; +return 0 ; +} + +static int amdgpu_debugfs_reset_level_set (void *data,u64 val) { +struct amdgpu_device *adev = (struct amdgpu_device *)data; + amdgpu_reset_level_mask = val; +return 0 ; +} +DEFINE_DEBUGFS_ATTRIBUTE (fops_ib_preempt,NULL , amdgpu_debugfs_ib_preempt,"%llu\n" );DEFINE_DEBUGFS_ATTRIBUTE (fops_sclk_set,NULL , amdgpu_debugfs_sclk_set,"%llu\n" ); +DEFINE_DEBUGFS_ATTRIBUTE (fops_reset_level, amdgpu_debugfs_reset_level_get, + amdgpu_debugfs_reset_level_set,"%llu\n" ); +static ssize_t amdgpu_reset_dump_register_list_read (struct file *f,char __user *buf,size_t size,loff_t *pos) {@@ -1785,6 +1802,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)return PTR_ERR (ent); } +debugfs_create_file ("amdgpu_reset_level" ,0200 , root, adev, + &fops_reset_level); +/* Register debugfs entries for amdgpu_ttm */ amdgpu_ttm_debugfs_init (adev);amdgpu_debugfs_pm_init (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1
-0@@ -198,6 +198,7 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .timeout_fatal_disable = false, .period = 0x0, /* default to 0x0 (timeout disable) */ }; +uint amdgpu_reset_level_mask =0x1 ;/** * DOC: vramlimit (int) drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +6
-0@@ -74,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;@@ -93,6 +96,9 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,int ret;struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +3
-0@@ -405,6 +405,9 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, {ktime_t deadline =ktime_add_us (ktime_get (),10000 ); +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_SOFT_RECOVERY )) +return false; +if (amdgpu_sriov_vf (ring->adev) || !ring->funcs->soft_recovery || !fence)return false;
Christian König
maintainerRe: [PATCH 2/5] drm/amdgpu: add debugfs amdgpu_reset_level
Hi Victor,
Am 25.07.22 um 12:45 schrieb Zhao, Victor:
Quoted 9 lines
[AMD Official Use Only - General]
Hi @Grodzovsky, Andrey,
Please help review the series, thanks a lot.
Hi @Koenig, Christian,
I thought a module parameter will be exposed to a common user, this control was added to help debug and test so I put it as a debugfs. I can make it module parameter if more appropriate.
That's a really good argument. I leave the decision if we should use a module parameter or debugfs file to Andrey.
If you go with debugfs then using the debugfs_create_u32() or debugfs_create_u64() function would be more appropriate I think. And then don't make that global, but rather a per device flag.
Regards,
Christian.
Quoted 155 lines
Thanks,
Victor-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, July 22, 2022 4:20 PM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deng, Emily <[email protected]>; Deucher, Alexander <[email protected]>
Subject: Re: [PATCH 2/5] drm/amdgpu: add debugfsamdgpu_reset_levelWell NAK to the debugfs approach, stuff like that is usually a module parameter.
Apart from that this series needs to be reviewed by Andrey.
Regards,
Christian.Am 22.07.22 um 09:34 schrieb Victor Zhao:
Quoted 133 lines
Introduce
amdgpu_reset_leveldebugfs in order to help debug and test specific type of reset. Also helps blocking unwanted type of resets.By default, mode2 reset will not be enabled
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 20 ++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 3 +++ 5 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +4
-0@@ -238,6 +238,7 @@ extern int amdgpu_si_support;extern int amdgpu_cik_support;#endif extern int amdgpu_num_kcq; +extern uint amdgpu_reset_level_mask;#define AMDGPU_VCNFW_LOG_SIZE (32 * 1024)extern int amdgpu_vcnfw_log;@@ -274,6 +275,9 @@ extern int amdgpu_vcnfw_log;#define AMDGPU_RESET_VCE (1 << 13)#define AMDGPU_RESET_VCE1 (1 << 14) +#define AMDGPU_RESET_LEVEL_SOFT_RECOVERY (1 << 0) #define +AMDGPU_RESET_LEVEL_MODE2 (1 <<1 ) + /* max cursor sizes (in pixels) */#define CIK_CURSOR_WIDTH 128#define CIK_CURSOR_HEIGHT 128drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +18
-0@@ -1661,12 +1661,29 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val)return ret; } +static int amdgpu_debugfs_reset_level_get (void *data,u64 *val) { +struct amdgpu_device *adev = (struct amdgpu_device *)data; + *val = amdgpu_reset_level_mask; +return 0 ; +} + +static int amdgpu_debugfs_reset_level_set (void *data,u64 val) { +struct amdgpu_device *adev = (struct amdgpu_device *)data; + amdgpu_reset_level_mask = val; +return 0 ; +} +DEFINE_DEBUGFS_ATTRIBUTE (fops_ib_preempt,NULL , amdgpu_debugfs_ib_preempt,"%llu\n" );DEFINE_DEBUGFS_ATTRIBUTE (fops_sclk_set,NULL , amdgpu_debugfs_sclk_set,"%llu\n" ); +DEFINE_DEBUGFS_ATTRIBUTE (fops_reset_level, amdgpu_debugfs_reset_level_get, + amdgpu_debugfs_reset_level_set,"%llu\n" ); +static ssize_t amdgpu_reset_dump_register_list_read (struct file *f,char __user *buf,size_t size,loff_t *pos) {@@ -1785,6 +1802,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)return PTR_ERR (ent); } +debugfs_create_file ("amdgpu_reset_level" ,0200 , root, adev, + &fops_reset_level); +/* Register debugfs entries for amdgpu_ttm */ amdgpu_ttm_debugfs_init (adev);amdgpu_debugfs_pm_init (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1
-0@@ -198,6 +198,7 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .timeout_fatal_disable = false, .period = 0x0, /* default to 0x0 (timeout disable) */ }; +uint amdgpu_reset_level_mask =0x1 ;/** * DOC: vramlimit (int) drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +6
-0@@ -74,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;@@ -93,6 +96,9 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,int ret;struct amdgpu_reset_handler *reset_handler =NULL ; +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2 )) +return -ENOSYS ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags))return -ENOSYS ;drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +3
-0@@ -405,6 +405,9 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, {ktime_t deadline =ktime_add_us (ktime_get (),10000 ); +if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_SOFT_RECOVERY )) +return false; +if (amdgpu_sriov_vf (ring->adev) || !ring->funcs->soft_recovery || !fence)return false;
Re: [PATCH 2/5] drm/amdgpu: add debugfs amdgpu_reset_level
Quoted 11 lines · Christian König
Hi Victor,
Am 25.07.22 um 12:45 schrieb Zhao, Victor:
Quoted 8 lines
[AMD Official Use Only - General]
Hi @Grodzovsky, Andrey,
Please help review the series, thanks a lot.
Hi @Koenig, Christian,
Quoted 8 lines
Quoted 1 line
I thought a module parameter will be exposed to a common user, this control was added to help debug and test so I put it as a debugfs. I can make it module parameter if more appropriate.
That's a really good argument. I leave the decision if we should use a module parameter or debugfs file to Andrey.
If you go with debugfs then using the
debugfs_create_u32()ordebugfs_create_u64()function would be more appropriate I think. And then don't make that global, but rather a per device flag.Regards,
Christian.
Makes sense to me too.
Andrey
Quoted 150 lines
Quoted 149 lines
Thanks,
Victor-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, July 22, 2022 4:20 PM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deng, Emily <[email protected]>; Deucher, Alexander <[email protected]>
Subject: Re: [PATCH 2/5] drm/amdgpu: add debugfsamdgpu_reset_levelWell NAK to the debugfs approach, stuff like that is usually a module parameter.
Apart from that this series needs to be reviewed by Andrey.
Regards,
Christian.Am 22.07.22 um 09:34 schrieb Victor Zhao:
Quoted 126 lines
Introduce
amdgpu_reset_leveldebugfs in order to help debug and test specific type of reset. Also helps blocking unwanted type of resets.By default, mode2 reset will not be enabled
Signed-off-by Victor Zhao <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++++drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c| 20
++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 3 +++5 files changed, 34 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -238,6 +238,7 @@ extern int amdgpu_si_support; extern int amdgpu_cik_support; #endif extern int amdgpu_num_kcq; +extern uint amdgpu_reset_level_mask; #define AMDGPU_VCNFW_LOG_SIZE (32 * 1024) extern int amdgpu_vcnfw_log;@@ -274,6 +275,9 @@ extern int amdgpu_vcnfw_log; #define AMDGPU_RESET_VCE (1 << 13) #define AMDGPU_RESET_VCE1 (1 << 14)+#define
AMDGPU_RESET_LEVEL_SOFT_RECOVERY(1 << 0) #definedrivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0+AMDGPU_RESET_LEVEL_MODE2 (1 <<1 ) +/* max cursor sizes (in pixels) */
#defineCIK_CURSOR_WIDTH128
#defineCIK_CURSOR_HEIGHT128drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +15
-0@@ -1661,12 +1661,29 @@ static int amdgpu_debugfs_sclk_set(void *data, u64 val) return ret; } +static int amdgpu_debugfs_reset_level_get(void *data, u64 *val) { + struct amdgpu_device *adev = (struct amdgpu_device *)data; + *val = amdgpu_reset_level_mask; + return 0; +} + +static int amdgpu_debugfs_reset_level_set (void *data,u64 val) { + struct amdgpu_device *adev = (struct amdgpu_device *)data; + amdgpu_reset_level_mask = val; + return 0; +} + DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL, amdgpu_debugfs_ib_preempt, "%llu\n"); DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL, amdgpu_debugfs_sclk_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_reset_level, amdgpu_debugfs_reset_level_get, + amdgpu_debugfs_reset_level_set, "%llu\n"); + static ssize_t amdgpu_reset_dump_register_list_read(struct file *f, char __user *buf, size_t size, loff_t *pos) {@@ -1785,6 +1802,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) return PTR_ERR(ent); } + debugfs_create_file("amdgpu_reset_level", 0200, root, adev, + &fops_reset_level); + /* Register debugfs entries for amdgpu_ttm */ amdgpu_ttm_debugfs_init(adev);
amdgpu_debugfs_pm_init(adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +0
-0@@ -198,6 +198,7 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .timeout_fatal_disable = false,.period =
0x0, /* default to0x0(timeout disable) */ };
+uintamdgpu_reset_level_mask=0x1;
/**
* DOC: vramlimit (int)drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +2
-0@@ -74,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, { struct amdgpu_reset_handler *reset_handler = NULL; + if (!(amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_MODE2)) + return -ENOSYS; + if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags)) return -ENOSYS;@@ -93,6 +96,9 @@ int amdgpu_reset_perform_reset(struct
amdgpu_device*adev, int ret;
structamdgpu_reset_handler*reset_handler= NULL;
+ if (!(amdgpu_reset_level_mask &AMDGPU_RESET_LEVEL_MODE2)) + return-ENOSYS;
+
if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags)) return-ENOSYS;
diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
indexd3558c34d406..1ffdc050a077100644drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +2
-0@@ -405,6 +405,9 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, { ktime_t deadline = ktime_add_us(ktime_get(), 10000); + if (!(amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_SOFT_RECOVERY)) + return false; + if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) return false;
PATCH 3/5 drm/amdgpu: save and restore gc hub regs
In reply to Victor ZhaoSave and restore gfxhub regs as they will be reset during mode 2
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h | 2 + drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 26 +++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 72 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 7 +- .../include/asic_reg/gc/gc_10_3_0_offset.h | 4 ++ 5 files changed, 110 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +2 -0
@@ -35,6 +35,8 @@ struct amdgpu_gfxhub_funcs {void (*init)(struct amdgpu_device *adev);int (*get_xgmi_info)(struct amdgpu_device *adev);void (*utcl2_harvest)(struct amdgpu_device *adev); +void (*mode2_save_regs)(struct amdgpu_device *adev); +void (*mode2_restore_regs)(struct amdgpu_device *adev); };struct amdgpu_gfxhub {
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +26 -0
@@ -264,6 +264,32 @@ struct amdgpu_gmc {u64 mall_size;/* number of UMC instances */ int num_umc; +/* mode2 save restore */ +u64 VM_L2_CNTL ; +u64 VM_L2_CNTL2 ; +u64 VM_DUMMY_PAGE_FAULT_CNTL ; +u64 VM_DUMMY_PAGE_FAULT_ADDR_LO32 ; +u64 VM_DUMMY_PAGE_FAULT_ADDR_HI32 ; +u64 VM_L2_PROTECTION_FAULT_CNTL ; +u64 VM_L2_PROTECTION_FAULT_CNTL2 ; +u64 VM_L2_PROTECTION_FAULT_MM_CNTL3 ; +u64 VM_L2_PROTECTION_FAULT_MM_CNTL4 ; +u64 VM_L2_PROTECTION_FAULT_ADDR_LO32 ; +u64 VM_L2_PROTECTION_FAULT_ADDR_HI32 ; +u64 VM_DEBUG ; +u64 VM_L2_MM_GROUP_RT_CLASSES ; +u64 VM_L2_BANK_SELECT_RESERVED_CID ; +u64 VM_L2_BANK_SELECT_RESERVED_CID2 ; +u64 VM_L2_CACHE_PARITY_CNTL ; +u64 VM_L2_IH_LOG_CNTL ; +u64 VM_CONTEXT_CNTL [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32 [16 ]; +u64 MC_VM_MX_L1_TLB_CNTL ; };#define amdgpu_gmc_flush_gpu_tlb (adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +72 -0
82 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +6 -1
@@ -94,8 +94,11 @@ sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,int r =0 ;struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; -if (!amdgpu_sriov_vf (adev)) +if (!amdgpu_sriov_vf (adev)) { +if (adev->gfxhub.funcs->mode2_save_regs) + adev->gfxhub.funcs->mode2_save_regs (adev); r =sienna_cichlid_mode2_suspend_ip (adev); + }return r; }
@@ -152,6 +155,8 @@ static int sienna_cichlid_mode2_restore_ip(struct amdgpu_device *adev) }/* Reinit GFXHUB */ +if (adev->gfxhub.funcs->mode2_restore_regs) + adev->gfxhub.funcs->mode2_restore_regs (adev); adev->gfxhub.funcs->init (adev); r = adev->gfxhub.funcs->gart_enable (adev);if (r) {
drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h +4 -0
@@ -3129,6 +3129,8 @@#define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX 0#define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32 0x15cc#define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX 0 +#define mmGCVM_DEBUG 0x15cd +#define mmGCVM_DEBUG_BASE_IDX 0#define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32 0x15ce#define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX 0#define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32 0x15cf
@@ -3151,6 +3153,8 @@#define mmGCVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX 0#define mmGCVM_L2_CACHE_PARITY_CNTL 0x15d8#define mmGCVM_L2_CACHE_PARITY_CNTL_BASE_IDX 0 +#define mmGCVM_L2_IH_LOG_CNTL 0x15d9 +#define mmGCVM_L2_IH_LOG_CNTL_BASE_IDX 0#define mmGCVM_L2_CNTL5 0x15dc#define mmGCVM_L2_CNTL5_BASE_IDX 0#define mmGCVM_L2_GCR_CNTL 0x15dd
Signature · 1 line
2.25.1
Re: [PATCH 3/5] drm/amdgpu: save and restore gc hub regs
Andrey
Quoted 197 lines · Victor Zhao
Save and restore gfxhub regs as they will be reset during mode 2
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h | 2 + drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 26 +++++++ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 72 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 7 +- .../include/asic_reg/gc/gc_10_3_0_offset.h | 4 ++ 5 files changed, 110 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +2
-0@@ -35,6 +35,8 @@ struct amdgpu_gfxhub_funcs {void (*init)(struct amdgpu_device *adev);int (*get_xgmi_info)(struct amdgpu_device *adev);void (*utcl2_harvest)(struct amdgpu_device *adev); +void (*mode2_save_regs)(struct amdgpu_device *adev); +void (*mode2_restore_regs)(struct amdgpu_device *adev); };struct amdgpu_gfxhub {drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +26
-0@@ -264,6 +264,32 @@ struct amdgpu_gmc {u64 mall_size;/* number of UMC instances */ int num_umc; +/* mode2 save restore */ +u64 VM_L2_CNTL ; +u64 VM_L2_CNTL2 ; +u64 VM_DUMMY_PAGE_FAULT_CNTL ; +u64 VM_DUMMY_PAGE_FAULT_ADDR_LO32 ; +u64 VM_DUMMY_PAGE_FAULT_ADDR_HI32 ; +u64 VM_L2_PROTECTION_FAULT_CNTL ; +u64 VM_L2_PROTECTION_FAULT_CNTL2 ; +u64 VM_L2_PROTECTION_FAULT_MM_CNTL3 ; +u64 VM_L2_PROTECTION_FAULT_MM_CNTL4 ; +u64 VM_L2_PROTECTION_FAULT_ADDR_LO32 ; +u64 VM_L2_PROTECTION_FAULT_ADDR_HI32 ; +u64 VM_DEBUG ; +u64 VM_L2_MM_GROUP_RT_CLASSES ; +u64 VM_L2_BANK_SELECT_RESERVED_CID ; +u64 VM_L2_BANK_SELECT_RESERVED_CID2 ; +u64 VM_L2_CACHE_PARITY_CNTL ; +u64 VM_L2_IH_LOG_CNTL ; +u64 VM_CONTEXT_CNTL [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32 [16 ]; +u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32 [16 ]; +u64 MC_VM_MX_L1_TLB_CNTL ; };#define amdgpu_gmc_flush_gpu_tlb (adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +72
-0@@ -576,6 +576,76 @@ static void gfxhub_v2_1_utcl2_harvest(struct amdgpu_device *adev) } } +static void gfxhub_v2_1_save_regs (struct amdgpu_device *adev) +{ +int i; + adev->gmc.VM_L2_CNTL =RREG32_SOC15 (GC ,0 , mmGCVM_L2_CNTL); + adev->gmc.VM_L2_CNTL2 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_CNTL2); + adev->gmc.VM_DUMMY_PAGE_FAULT_CNTL =RREG32_SOC15 (GC ,0 , mmGCVM_DUMMY_PAGE_FAULT_CNTL); + adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_LO32 =RREG32_SOC15 (GC ,0 , mmGCVM_DUMMY_PAGE_FAULT_ADDR_LO32); + adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_HI32 =RREG32_SOC15 (GC ,0 , mmGCVM_DUMMY_PAGE_FAULT_ADDR_HI32); + adev->gmc.VM_L2_PROTECTION_FAULT_CNTL =RREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_CNTL); + adev->gmc.VM_L2_PROTECTION_FAULT_CNTL2 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_CNTL2); + adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL3 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_MM_CNTL3); + adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL4 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_MM_CNTL4); + adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_LO32 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_ADDR_LO32); + adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_HI32 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_ADDR_HI32); + adev->gmc.VM_DEBUG =RREG32_SOC15 (GC ,0 , mmGCVM_DEBUG); + adev->gmc.VM_L2_MM_GROUP_RT_CLASSES =RREG32_SOC15 (GC ,0 , mmGCVM_L2_MM_GROUP_RT_CLASSES); + adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID =RREG32_SOC15 (GC ,0 , mmGCVM_L2_BANK_SELECT_RESERVED_CID); + adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID2 =RREG32_SOC15 (GC ,0 , mmGCVM_L2_BANK_SELECT_RESERVED_CID2); + adev->gmc.VM_L2_CACHE_PARITY_CNTL =RREG32_SOC15 (GC ,0 , mmGCVM_L2_CACHE_PARITY_CNTL); + adev->gmc.VM_L2_IH_LOG_CNTL =RREG32_SOC15 (GC ,0 , mmGCVM_L2_IH_LOG_CNTL); + +for (i =0 ; i <=15 ; i++) { + adev->gmc.VM_CONTEXT_CNTL[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_CNTL, i); + adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, i *2 ); + adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, i *2 ); + adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, i *2 ); + adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, i *2 ); + adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, i *2 ); + adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[i] =RREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, i *2 ); + } + + adev->gmc.MC_VM_MX_L1_TLB_CNTL =RREG32_SOC15 (GC ,0 , mmGCMC_VM_MX_L1_TLB_CNTL); +} + +static void gfxhub_v2_1_restore_regs (struct amdgpu_device *adev) +{ +int i; +WREG32_SOC15 (GC ,0 , mmGCVM_L2_CNTL, adev->gmc.VM_L2_CNTL); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_CNTL2, adev->gmc.VM_L2_CNTL2); +WREG32_SOC15 (GC ,0 , mmGCVM_DUMMY_PAGE_FAULT_CNTL, adev->gmc.VM_DUMMY_PAGE_FAULT_CNTL); +WREG32_SOC15 (GC ,0 , mmGCVM_DUMMY_PAGE_FAULT_ADDR_LO32, adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_LO32); +WREG32_SOC15 (GC ,0 , mmGCVM_DUMMY_PAGE_FAULT_ADDR_HI32, adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_HI32); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_CNTL, adev->gmc.VM_L2_PROTECTION_FAULT_CNTL); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_CNTL2, adev->gmc.VM_L2_PROTECTION_FAULT_CNTL2); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_MM_CNTL3, adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL3); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_MM_CNTL4, adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL4); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_ADDR_LO32, adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_LO32); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_PROTECTION_FAULT_ADDR_HI32, adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_HI32); +WREG32_SOC15 (GC ,0 , mmGCVM_DEBUG, adev->gmc.VM_DEBUG); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_MM_GROUP_RT_CLASSES, adev->gmc.VM_L2_MM_GROUP_RT_CLASSES); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_BANK_SELECT_RESERVED_CID, adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_BANK_SELECT_RESERVED_CID2, adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID2); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_CACHE_PARITY_CNTL, adev->gmc.VM_L2_CACHE_PARITY_CNTL); +WREG32_SOC15 (GC ,0 , mmGCVM_L2_IH_LOG_CNTL, adev->gmc.VM_L2_IH_LOG_CNTL); + +for (i =0 ; i <=15 ; i++) { +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_CNTL, i, adev->gmc.VM_CONTEXT_CNTL[i]); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, i *2 , adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[i]); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, i *2 , adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[i]); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, i *2 , adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[i]); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, i *2 , adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[i]); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, i *2 , adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[i]); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, i *2 , adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[i]); + } + +WREG32_SOC15 (GC ,0 , mmGCMC_VM_FB_LOCATION_BASE, adev->gmc.vram_start >>24 ); +WREG32_SOC15 (GC ,0 , mmGCMC_VM_FB_LOCATION_TOP, adev->gmc.vram_end >>24 ); +WREG32_SOC15 (GC ,0 , mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL); +} +const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .get_fb_location = gfxhub_v2_1_get_fb_location, .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset,@@ -586,4 +656,6 @@ const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .init = gfxhub_v2_1_init, .get_xgmi_info = gfxhub_v2_1_get_xgmi_info, .utcl2_harvest = gfxhub_v2_1_utcl2_harvest, + .mode2_save_regs = gfxhub_v2_1_save_regs, + .mode2_restore_regs = gfxhub_v2_1_restore_regs, };drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +6
-1@@ -94,8 +94,11 @@ sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,int r =0 ;struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; -if (!amdgpu_sriov_vf (adev)) +if (!amdgpu_sriov_vf (adev)) { +if (adev->gfxhub.funcs->mode2_save_regs) + adev->gfxhub.funcs->mode2_save_regs (adev); r =sienna_cichlid_mode2_suspend_ip (adev); + }return r; }@@ -152,6 +155,8 @@ static int sienna_cichlid_mode2_restore_ip(struct amdgpu_device *adev) }/* Reinit GFXHUB */ +if (adev->gfxhub.funcs->mode2_restore_regs) + adev->gfxhub.funcs->mode2_restore_regs (adev); adev->gfxhub.funcs->init (adev); r = adev->gfxhub.funcs->gart_enable (adev);if (r) {drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h +4
-0@@ -3129,6 +3129,8 @@#define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX 0#define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32 0x15cc#define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX 0 +#define mmGCVM_DEBUG 0x15cd +#define mmGCVM_DEBUG_BASE_IDX 0#define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32 0x15ce#define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX 0#define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32 0x15cf@@ -3151,6 +3153,8 @@#define mmGCVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX 0#define mmGCVM_L2_CACHE_PARITY_CNTL 0x15d8#define mmGCVM_L2_CACHE_PARITY_CNTL_BASE_IDX 0 +#define mmGCVM_L2_IH_LOG_CNTL 0x15d9 +#define mmGCVM_L2_IH_LOG_CNTL_BASE_IDX 0#define mmGCVM_L2_CNTL5 0x15dc#define mmGCVM_L2_CNTL5_BASE_IDX 0#define mmGCVM_L2_GCR_CNTL 0x15dd
PATCH 4/5 drm/amdgpu: revert context to stop engine before mode2 reset
In reply to Victor ZhaoFor some hang caused by slow tests, engine cannot be stopped which may cause resume failure after reset. In this case, force halt engine by reverting context addresses
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h | 1 + drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 2 ++ 4 files changed, 40 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -5037,6 +5037,7 @@ static void amdgpu_device_recheck_guilty_jobs(/* set guilty */ drm_sched_increase_karma (s_job); +amdgpu_reset_prepare_hwcontext (adev, reset_context); retry:/* do hw reset */ if (amdgpu_sriov_vf (adev)) {
drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +1 -0
@@ -37,6 +37,7 @@ struct amdgpu_gfxhub_funcs {void (*utcl2_harvest)(struct amdgpu_device *adev);void (*mode2_save_regs)(struct amdgpu_device *adev);void (*mode2_restore_regs)(struct amdgpu_device *adev); +void (*halt)(struct amdgpu_device *adev); };struct amdgpu_gfxhub {
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +36 -0
@@ -646,6 +646,41 @@ static void gfxhub_v2_1_restore_regs(struct amdgpu_device *adev)WREG32_SOC15 (GC ,0 , mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL); } +static void gfxhub_v2_1_halt (struct amdgpu_device *adev) +{ +struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0 ]; +int i; +uint32_t tmp; +int time =1000 ; + +gfxhub_v2_1_set_fault_enable_default (adev, false); + +for (i =0 ; i <=14 ; i++) { +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, + i * hub->ctx_addr_distance, +0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, + i * hub->ctx_addr_distance, +0 ); + } + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); +while ((tmp & (GRBM_STATUS2__EA_BUSY_MASK | +GRBM_STATUS2__EA_LINK_BUSY_MASK )) !=0 && + time) { +udelay (100 ); + time--; + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); + } + +if (!time) { +DRM_WARN ("failed to wait for GRBM(EA) idle\n" ); + } +} +const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .get_fb_location = gfxhub_v2_1_get_fb_location, .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset,
@@ -658,4 +693,5 @@ const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .utcl2_harvest = gfxhub_v2_1_utcl2_harvest, .mode2_save_regs = gfxhub_v2_1_save_regs, .mode2_restore_regs = gfxhub_v2_1_restore_regs, + .halt = gfxhub_v2_1_halt, };
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +2 -0
@@ -97,6 +97,8 @@ sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,if (!amdgpu_sriov_vf (adev)) {if (adev->gfxhub.funcs->mode2_save_regs) adev->gfxhub.funcs->mode2_save_regs (adev); +if (adev->gfxhub.funcs->halt) + adev->gfxhub.funcs->halt (adev); r =sienna_cichlid_mode2_suspend_ip (adev); }
Signature · 1 line
2.25.1
Re: [PATCH 4/5] drm/amdgpu: revert context to stop engine before mode2 reset
Quoted 3 lines · Victor Zhao
For some hang caused by slow tests, engine cannot be stopped which may cause resume failure after reset. In this case, force halt engine by reverting context addresses
Can you maybe explain a bit more what exactly you mean by slow test and why engine cannot be stopped in this case ?
Andrey
Quoted 98 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h | 1 + drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 2 ++ 4 files changed, 40 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0@@ -5037,6 +5037,7 @@ static void amdgpu_device_recheck_guilty_jobs(/* set guilty */ drm_sched_increase_karma (s_job); +amdgpu_reset_prepare_hwcontext (adev, reset_context); retry:/* do hw reset */ if (amdgpu_sriov_vf (adev)) {drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +1
-0@@ -37,6 +37,7 @@ struct amdgpu_gfxhub_funcs {void (*utcl2_harvest)(struct amdgpu_device *adev);void (*mode2_save_regs)(struct amdgpu_device *adev);void (*mode2_restore_regs)(struct amdgpu_device *adev); +void (*halt)(struct amdgpu_device *adev); };struct amdgpu_gfxhub {drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +36
-0@@ -646,6 +646,41 @@ static void gfxhub_v2_1_restore_regs(struct amdgpu_device *adev)WREG32_SOC15 (GC ,0 , mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL); } +static void gfxhub_v2_1_halt (struct amdgpu_device *adev) +{ +struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0 ]; +int i; +uint32_t tmp; +int time =1000 ; + +gfxhub_v2_1_set_fault_enable_default (adev, false); + +for (i =0 ; i <=14 ; i++) { +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, + i * hub->ctx_addr_distance, +0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, + i * hub->ctx_addr_distance, +0 ); + } + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); +while ((tmp & (GRBM_STATUS2__EA_BUSY_MASK | +GRBM_STATUS2__EA_LINK_BUSY_MASK )) !=0 && + time) { +udelay (100 ); + time--; + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); + } + +if (!time) { +DRM_WARN ("failed to wait for GRBM(EA) idle\n" ); + } +} +const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .get_fb_location = gfxhub_v2_1_get_fb_location, .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset,@@ -658,4 +693,5 @@ const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .utcl2_harvest = gfxhub_v2_1_utcl2_harvest, .mode2_save_regs = gfxhub_v2_1_save_regs, .mode2_restore_regs = gfxhub_v2_1_restore_regs, + .halt = gfxhub_v2_1_halt, };drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +2
-0@@ -97,6 +97,8 @@ sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,if (!amdgpu_sriov_vf (adev)) {if (adev->gfxhub.funcs->mode2_save_regs) adev->gfxhub.funcs->mode2_save_regs (adev); +if (adev->gfxhub.funcs->halt) + adev->gfxhub.funcs->halt (adev); r =sienna_cichlid_mode2_suspend_ip (adev); }
RE: [PATCH 4/5] drm/amdgpu: revert context to stop engine before mode2 reset
[AMD Official Use Only - General]
Hi Andrey,
For slow tests I mean the slow hang tests by quark tool.
An example here:
hang_vm_gfx_dispatch_slow.lua - This script runs on a graphics engine using compute engine and has a hacked CS program which is massive and duplicates standard CS program move code hundreds of thousands of times. The effect is a very slowly executing CS program.
It's not a bad job but just need a very long time to finish. I suppose we don’t have a way to stop shader here. And the running apps will be affected when reset is done.
Thanks,
Victor
-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:20 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 4/5] drm/amdgpu: revert context to stop engine before mode2 reset
Quoted 3 lines · Victor Zhao
For some hang caused by slow tests, engine cannot be stopped which may cause resume failure after reset. In this case, force halt engine by reverting context addresses
Can you maybe explain a bit more what exactly you mean by slow test and why engine cannot be stopped in this case ?
Andrey
Quoted 101 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h | 1 + drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 2 ++ 4 files changed, 40 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0@@ -5037,6 +5037,7 @@ static void amdgpu_device_recheck_guilty_jobs(/* set guilty */ drm_sched_increase_karma (s_job); +amdgpu_reset_prepare_hwcontext (adev, reset_context); retry:/* do hw reset */ if (amdgpu_sriov_vf (adev)) {drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +1
-0@@ -37,6 +37,7 @@ struct amdgpu_gfxhub_funcs {void (*utcl2_harvest)(struct amdgpu_device *adev);void (*mode2_save_regs)(struct amdgpu_device *adev);void (*mode2_restore_regs)(struct amdgpu_device *adev); +void (*halt)(struct amdgpu_device *adev); };struct amdgpu_gfxhub {drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +34
-0@@ -646,6 +646,41 @@ static void gfxhub_v2_1_restore_regs(struct amdgpu_device *adev)WREG32_SOC15 (GC ,0 , mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL); } +static void gfxhub_v2_1_halt (struct amdgpu_device *adev) { +struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0 ]; +int i; +uint32_t tmp; +int time =1000 ; + +gfxhub_v2_1_set_fault_enable_default (adev, false); + +for (i =0 ; i <=14 ; i++) { +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, + i * hub->ctx_addr_distance, +0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, + i * hub->ctx_addr_distance, +0 ); + } + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); +while ((tmp & (GRBM_STATUS2__EA_BUSY_MASK | +GRBM_STATUS2__EA_LINK_BUSY_MASK )) !=0 && + time) { +udelay (100 ); + time--; + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); + } + +if (!time) { +DRM_WARN ("failed to wait for GRBM(EA) idle\n" ); + } +} +const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .get_fb_location = gfxhub_v2_1_get_fb_location, .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset, @@ -658,4 +693,5@@ const struct
amdgpu_gfxhub_funcsgfxhub_v2_1_funcs= {
.utcl2_harvest =gfxhub_v2_1_utcl2_harvest, .mode2_save_regs =gfxhub_v2_1_save_regs, .mode2_restore_regs =gfxhub_v2_1_restore_regs,drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +1
-0+ .halt = gfxhub_v2_1_halt, };drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +2
-0@@ -97,6 +97,8 @@ sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,if (!amdgpu_sriov_vf (adev)) {if (adev->gfxhub.funcs->mode2_save_regs) adev->gfxhub.funcs->mode2_save_regs (adev); +if (adev->gfxhub.funcs->halt) + adev->gfxhub.funcs->halt (adev); r =sienna_cichlid_mode2_suspend_ip (adev); }
Andrey Grodzovsky
Acked-byRe: [PATCH 4/5] drm/amdgpu: revert context to stop engine before mode2 reset
Got it
Andrey
Quoted 134 lines · Victor
[AMD Official Use Only - General]
Hi Andrey,
For slow tests I mean the slow hang tests by quark tool.
An example here:
hang_vm_gfx_dispatch_slow.lua - This script runs on a graphics engine using compute engine and has a hacked CS program which is massive and duplicates standard CS program move code hundreds of thousands of times. The effect is a very slowly executing CS program.It's not a bad job but just need a very long time to finish. I suppose we don’t have a way to stop shader here. And the running apps will be affected when reset is done.
Thanks,
Victor-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:20 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 4/5] drm/amdgpu: revert context to stop engine before mode2 resetQuoted 3 lines · Victor Zhao
For some hang caused by slow tests, engine cannot be stopped which may cause resume failure after reset. In this case, force halt engine by reverting context addresses
Can you maybe explain a bit more what exactly you mean by slow test and why engine cannot be stopped in this case ?
Andrey
Quoted 100 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h | 1 + drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 2 ++ 4 files changed, 40 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0@@ -5037,6 +5037,7 @@ static void amdgpu_device_recheck_guilty_jobs(/* set guilty */ drm_sched_increase_karma (s_job); +amdgpu_reset_prepare_hwcontext (adev, reset_context); retry:/* do hw reset */ if (amdgpu_sriov_vf (adev)) {drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +1
-0@@ -37,6 +37,7 @@ struct amdgpu_gfxhub_funcs {void (*utcl2_harvest)(struct amdgpu_device *adev);void (*mode2_save_regs)(struct amdgpu_device *adev);void (*mode2_restore_regs)(struct amdgpu_device *adev); +void (*halt)(struct amdgpu_device *adev); };struct amdgpu_gfxhub {drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +34
-0@@ -646,6 +646,41 @@ static void gfxhub_v2_1_restore_regs(struct amdgpu_device *adev)WREG32_SOC15 (GC ,0 , mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL); } +static void gfxhub_v2_1_halt (struct amdgpu_device *adev) { +struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0 ]; +int i; +uint32_t tmp; +int time =1000 ; + +gfxhub_v2_1_set_fault_enable_default (adev, false); + +for (i =0 ; i <=14 ; i++) { +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, + i * hub->ctx_addr_distance, ~0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, + i * hub->ctx_addr_distance, +0 ); +WREG32_SOC15_OFFSET (GC ,0 , mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, + i * hub->ctx_addr_distance, +0 ); + } + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); +while ((tmp & (GRBM_STATUS2__EA_BUSY_MASK | +GRBM_STATUS2__EA_LINK_BUSY_MASK )) !=0 && + time) { +udelay (100 ); + time--; + tmp =RREG32_SOC15 (GC ,0 , mmGRBM_STATUS2); + } + +if (!time) { +DRM_WARN ("failed to wait for GRBM(EA) idle\n" ); + } +} +const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .get_fb_location = gfxhub_v2_1_get_fb_location, .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset, @@ -658,4 +693,5@@ const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { .utcl2_harvest = gfxhub_v2_1_utcl2_harvest, .mode2_save_regs = gfxhub_v2_1_save_regs, .mode2_restore_regs = gfxhub_v2_1_restore_regs,drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +1
-0+ .halt = gfxhub_v2_1_halt, };drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +2
-0@@ -97,6 +97,8 @@ sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,if (!amdgpu_sriov_vf (adev)) {if (adev->gfxhub.funcs->mode2_save_regs) adev->gfxhub.funcs->mode2_save_regs (adev); +if (adev->gfxhub.funcs->halt) + adev->gfxhub.funcs->halt (adev); r =sienna_cichlid_mode2_suspend_ip (adev); }
PATCH 5/5 drm/amdgpu: reduce reset time
In reply to Victor ZhaoIn multi container use case, reset time is important, so skip ring tests and cp halt wait during ip suspending for reset as they are going to fail and cost more time on reset
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 26 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1 -1
@@ -477,7 +477,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.compute_ring[i],RESET_QUEUES ,0 ,0 ); -if (adev->gfx.kiq.ring.sched.ready) +if (adev->gfx.kiq.ring.sched.ready && !amdgpu_in_reset (adev)) r =amdgpu_ring_test_helper (kiq_ring);spin_unlock (&adev->gfx.kiq.ring_lock);
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +16 -10
@@ -5971,16 +5971,19 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)WREG32_SOC15 (GC ,0 , mmCP_ME_CNTL, tmp); } -for (i =0 ; i < adev->usec_timeout; i++) { -if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("failed to %s cp gfx\n" , enable ?"unhalt" :"halt" ); +if (!amdgpu_in_reset (adev)) { +for (i =0 ; i < adev->usec_timeout; i++) { +if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) +break ; +udelay (1 ); + } +if (i >= adev->usec_timeout) +DRM_ERROR ("failed to %s cp gfx\n" , + enable ?"unhalt" :"halt" ); + }return 0 ; + }static int gfx_v10_0_cp_gfx_load_pfp_microcode (struct amdgpu_device *adev)
@@ -7569,8 +7572,10 @@ static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)for (i =0 ; i < adev->gfx.num_gfx_rings; i++) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.gfx_ring[i],PREEMPT_QUEUES ,0 ,0 ); - - return amdgpu_ring_test_helper(kiq_ring); +if (!amdgpu_in_reset (adev)) +return amdgpu_ring_test_helper (kiq_ring); +else + return 0; }#endif
@@ -7610,6 +7615,7 @@ static int gfx_v10_0_hw_fini(void *handle)return 0 ; } +gfx_v10_0_cp_enable (adev, false);gfx_v10_0_enable_gui_idle_interrupt (adev, false);
Signature · 1 line
2.25.1
Re: [PATCH 5/5] drm/amdgpu: reduce reset time
Quoted 3 lines · Victor Zhao
In multi container use case, reset time is important, so skip ring tests and cp halt wait during ip suspending for reset as they are going to fail and cost more time on reset
Why are they failing in this case ? Skipping ring tests is not the best idea as you loose important indicator of system's sanity. Is there any way to make them work ?
Quoted 40 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 26 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-1@@ -477,7 +477,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.compute_ring[i],RESET_QUEUES ,0 ,0 ); -if (adev->gfx.kiq.ring.sched.ready) +if (adev->gfx.kiq.ring.sched.ready && !amdgpu_in_reset (adev)) r =amdgpu_ring_test_helper (kiq_ring);spin_unlock (&adev->gfx.kiq.ring_lock);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-8@@ -5971,16 +5971,19 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)WREG32_SOC15 (GC ,0 , mmCP_ME_CNTL, tmp); } -for (i =0 ; i < adev->usec_timeout; i++) { -if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("failed to %s cp gfx\n" , enable ?"unhalt" :"halt" ); +if (!amdgpu_in_reset (adev)) { +for (i =0 ; i < adev->usec_timeout; i++) { +if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) +break ;
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) · line 5987
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +7
-0+udelay (1 ); + } +if (i >= adev->usec_timeout) +DRM_ERROR ("failed to %s cp gfx\n" , + enable ?"unhalt" :"halt" ); + }return 0 ; + }
This change has impact beyond container case no ? We had no issue
with this code during regular reset cases so why we would give up on this code which confirms CP is idle ? What is the side effect of skipping this during all GPU resets ?
Andrey
Quoted 23 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-2static int gfx_v10_0_cp_gfx_load_pfp_microcode (struct amdgpu_device *adev)@@ -7569,8 +7572,10 @@ static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)for (i =0 ; i < adev->gfx.num_gfx_rings; i++) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.gfx_ring[i],PREEMPT_QUEUES ,0 ,0 ); - - return amdgpu_ring_test_helper(kiq_ring); +if (!amdgpu_in_reset (adev)) +return amdgpu_ring_test_helper (kiq_ring); +else + return 0; }#endif @@ -7610,6 +7615,7 @@ static int gfx_v10_0_hw_fini(void *handle)return 0 ; } +gfx_v10_0_cp_enable (adev, false);gfx_v10_0_enable_gui_idle_interrupt (adev, false);
RE: [PATCH 5/5] drm/amdgpu: reduce reset time
[AMD Official Use Only - General]
Hi Andrey,
Reply inline.
Thanks,
Victor
-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:18 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 5/5] drm/amdgpu: reduce reset time
Quoted 3 lines · Victor Zhao
In multi container use case, reset time is important, so skip ring tests and cp halt wait during ip suspending for reset as they are going to fail and cost more time on reset
Why are they failing in this case ? Skipping ring tests is not the best idea as you loose important indicator of system's sanity. Is there any way to make them work ?
[Victor]: I've seen gfx ring test fail every time after a gfx engine hang. I thought it should be expected as gfx is in a bad state. Do you know the reason we have ring tests before reset? As we are going to reset the asic anyway. Another approach could be to make the skip mode2 only or reduce the wait time here.
Quoted 42 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 26 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-1@@ -477,7 +477,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.compute_ring[i],RESET_QUEUES ,0 ,0 ); -if (adev->gfx.kiq.ring.sched.ready) +if (adev->gfx.kiq.ring.sched.ready && !amdgpu_in_reset (adev)) r =amdgpu_ring_test_helper (kiq_ring);spin_unlock (&adev->gfx.kiq.ring_lock);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-8@@ -5971,16 +5971,19 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)WREG32_SOC15 (GC ,0 , mmCP_ME_CNTL, tmp); } -for (i =0 ; i < adev->usec_timeout; i++) { -if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("failed to %s cp gfx\n" , enable ?"unhalt" :"halt" ); +if (!amdgpu_in_reset (adev)) { +for (i =0 ; i < adev->usec_timeout; i++) { +if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) +break ;
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) · line 5987
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +7
-0+udelay (1 ); + } +if (i >= adev->usec_timeout) +DRM_ERROR ("failed to %s cp gfx\n" , + enable ?"unhalt" :"halt" ); + }return 0 ; + }
This change has impact beyond container case no ? We had no issue with this code during regular reset cases so why we would give up on this code which confirms CP is idle ? What is the side effect of skipping this during all GPU resets ?
Andrey
[Victor]: I see "failed to halt cp gfx" with regular reset cases as well when doing a gfx hang test using quark. I haven't seen a side effect with Mode1 reset yet but maybe shorten the wait time could be better?
Quoted 23 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0static int gfx_v10_0_cp_gfx_load_pfp_microcode (struct amdgpu_device *adev) @@ -7569,8 +7572,10 @@ static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)for (i = 0; i <
adev->gfx.num_gfx_rings; i++)
kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
PREEMPT_QUEUES, 0, 0);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-2- - return amdgpu_ring_test_helper(kiq_ring); +if (!amdgpu_in_reset (adev)) +return amdgpu_ring_test_helper (kiq_ring); +else + return 0; }#endif @@ -7610,6 +7615,7 @@ static int gfx_v10_0_hw_fini(void *handle)return 0 ; } +gfx_v10_0_cp_enable (adev, false);gfx_v10_0_enable_gui_idle_interrupt (adev, false);
Re: [PATCH 5/5] drm/amdgpu: reduce reset time
Quoted 21 lines · Victor
[AMD Official Use Only - General]
Hi Andrey,
Reply inline.
Thanks,
Victor-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:18 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 5/5] drm/amdgpu: reduce reset timeOn 2022-07-22 03:34, Victor Zhao wrote:
Quoted 8 lines
Quoted 3 lines
In multi container use case, reset time is important, so skip ring tests and cp halt wait during ip suspending for reset as they are going to fail and cost more time on reset
Why are they failing in this case ? Skipping ring tests is not the best idea as you loose important indicator of system's sanity. Is there any way to make them work ?
[Victor]: I've seen gfx ring test fail every time after a gfx engine hang. I thought it should be expected as gfx is in a bad state. Do you know the reason we have ring tests before reset? As we are going to reset the asic anyway. Another approach could be to make the skip mode2 only or reduce the wait time here.
I dug down in history and according to commit 'drm/amdgpu:unmap KCQ in gfx hw_fini(v2)' you need to write to scratch register for completion of queue unmap operation so you defently don't want to just skip it. I agree in case that the ring is hung this has no point but remember that GPU reset can happen not only to a hunged ring but for other reasons (RAS, manual reset e.t.c.) in which case you probably want to shut down gracefully here ? I see we have adev->ip_blocks[i].status.hang flag which you maybe can use here instead ?
Quoted 56 lines
Quoted 43 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 26 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-1@@ -477,7 +477,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.compute_ring[i],RESET_QUEUES ,0 ,0 ); -if (adev->gfx.kiq.ring.sched.ready) +if (adev->gfx.kiq.ring.sched.ready && !amdgpu_in_reset (adev)) r =amdgpu_ring_test_helper (kiq_ring);spin_unlock (&adev->gfx.kiq.ring_lock);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-8@@ -5971,16 +5971,19 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)WREG32_SOC15 (GC ,0 , mmCP_ME_CNTL, tmp); } -for (i =0 ; i < adev->usec_timeout; i++) { -if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("failed to %s cp gfx\n" , enable ?"unhalt" :"halt" ); +if (!amdgpu_in_reset (adev)) { +for (i =0 ; i < adev->usec_timeout; i++) { +if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) +break ;drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) · line 5987
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +7
-0+udelay (1 ); + } +if (i >= adev->usec_timeout) +DRM_ERROR ("failed to %s cp gfx\n" , + enable ?"unhalt" :"halt" ); + }return 0 ; + }This change has impact beyond container case no ? We had no issue with this code during regular reset cases so why we would give up on this code which confirms CP is idle ? What is the side effect of skipping this during all GPU resets ?
Quoted 3 lines
Andrey
[Victor]: I see "failed to halt cp gfx" with regular reset cases as well when doing a gfx hang test using quark. I haven't seen a side effect with Mode1 reset yet but maybe shorten the wait time could be better?
Same as above i guess, it would indeed time out for a hung ring but GPU reset happens not only because of hung rings but for other reasons.
Andrey
Quoted 24 lines
Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0static int gfx_v10_0_cp_gfx_load_pfp_microcode (struct amdgpu_device *adev) @@ -7569,8 +7572,10 @@ static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)for (i = 0; i < adev->gfx.num_gfx_rings; i++) kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i], PREEMPT_QUEUES, 0, 0);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-2- - return amdgpu_ring_test_helper(kiq_ring); +if (!amdgpu_in_reset (adev)) +return amdgpu_ring_test_helper (kiq_ring); +else + return 0; }#endif @@ -7610,6 +7615,7 @@ static int gfx_v10_0_hw_fini(void *handle)return 0 ; } +gfx_v10_0_cp_enable (adev, false);gfx_v10_0_enable_gui_idle_interrupt (adev, false);
RE: [PATCH 5/5] drm/amdgpu: reduce reset time
[AMD Official Use Only - General]
Hi Andrey,
Problem with status.hang is that it is set at amdgpu_device_ip_check_soft_reset, which is not implemented in nv or gfx10. They have to be nicely implemented first.
Another option I thought is to mark status.hang or add a flag to amdgpu_gfx when job timeout reported on gfx/comp ring. And this will require some logic to map the relationship for ring and ip blocks. This way does not look good as well.
Thanks,
Victor
-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Wednesday, July 27, 2022 12:57 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 5/5] drm/amdgpu: reduce reset time
Quoted 22 lines · Victor
[AMD Official Use Only - General]
Hi Andrey,
Reply inline.
Thanks,
Victor-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:18 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 5/5] drm/amdgpu: reduce reset timeOn 2022-07-22 03:34, Victor Zhao wrote:
Quoted 8 lines
Quoted 3 lines
In multi container use case, reset time is important, so skip ring tests and cp halt wait during ip suspending for reset as they are going to fail and cost more time on reset
Why are they failing in this case ? Skipping ring tests is not the best idea as you loose important indicator of system's sanity. Is there any way to make them work ?
[Victor]: I've seen gfx ring test fail every time after a gfx engine hang. I thought it should be expected as gfx is in a bad state. Do you know the reason we have ring tests before reset? As we are going to reset the asic anyway. Another approach could be to make the skip mode2 only or reduce the wait time here.
I dug down in history and according to commit 'drm/amdgpu:unmap KCQ in gfx hw_fini(v2)' you need to write to scratch register for completion of queue unmap operation so you defently don't want to just skip it. I agree in case that the ring is hung this has no point but remember that GPU reset can happen not only to a hunged ring but for other reasons (RAS, manual reset e.t.c.) in which case you probably want to shut down gracefully here ? I see we have adev->ip_blocks[i].status.hang flag which you maybe can use here instead ?
Quoted 56 lines
Quoted 43 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 26 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-1@@ -477,7 +477,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.compute_ring[i],RESET_QUEUES ,0 ,0 ); -if (adev->gfx.kiq.ring.sched.ready) +if (adev->gfx.kiq.ring.sched.ready && !amdgpu_in_reset (adev)) r =amdgpu_ring_test_helper (kiq_ring);spin_unlock (&adev->gfx.kiq.ring_lock);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-8@@ -5971,16 +5971,19 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)WREG32_SOC15 (GC ,0 , mmCP_ME_CNTL, tmp); } -for (i =0 ; i < adev->usec_timeout; i++) { -if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("failed to %s cp gfx\n" , enable ?"unhalt" :"halt" ); +if (!amdgpu_in_reset (adev)) { +for (i =0 ; i < adev->usec_timeout; i++) { +if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) +break ;drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) · line 5987
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +7
-0+udelay (1 ); + } +if (i >= adev->usec_timeout) +DRM_ERROR ("failed to %s cp gfx\n" , + enable ?"unhalt" :"halt" ); + }return 0 ; + }This change has impact beyond container case no ? We had no issue with this code during regular reset cases so why we would give up on this code which confirms CP is idle ? What is the side effect of skipping this during all GPU resets ?
Quoted 3 lines
Andrey
[Victor]: I see "failed to halt cp gfx" with regular reset cases as well when doing a gfx hang test using quark. I haven't seen a side effect with Mode1 reset yet but maybe shorten the wait time could be better?
Same as above i guess, it would indeed time out for a hung ring but GPU reset happens not only because of hung rings but for other reasons.
Andrey
Quoted 25 lines
Quoted 25 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0static int gfx_v10_0_cp_gfx_load_pfp_microcode(struct
amdgpu_device
*adev) @@ -7569,8 +7572,10 @@ static intgfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)
for (i = 0; i <adev->gfx.num_gfx_rings; i++)
kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
PREEMPT_QUEUES, 0, 0);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-2- - return amdgpu_ring_test_helper(kiq_ring); +if (!amdgpu_in_reset (adev)) +return amdgpu_ring_test_helper (kiq_ring); +else + return 0; }#endif @@ -7610,6 +7615,7 @@ static int gfx_v10_0_hw_fini(void *handle)return 0 ; } +gfx_v10_0_cp_enable (adev, false);gfx_v10_0_enable_gui_idle_interrupt (adev, false);
Re: [PATCH 5/5] drm/amdgpu: reduce reset time
Quoted 6 lines · Victor
[AMD Official Use Only - General]
Hi Andrey,
Problem with status.hang is that it is set at
amdgpu_device_ip_check_soft_reset, which is not implemented in nv or gfx10. They have to be nicely implemented first.
Another option I thought is to mark status.hang or add a flag toamdgpu_gfxwhen job timeout reported on gfx/comp ring. And this will require some logic to map the relationship for ring and ip blocks. This way does not look good as well.
I don't think we need this at the ring level, its enough to know that the reset you are going through is because of one of rings are hanged to apply this skip logic, it's pretty easy if we add 'bool hang' flag to adev->reset_domain which u can set in the beginning amdgpu_job_timedout and clear in the end. No protection is required as all the resets from all origins are serialized with timeout handler in a single threaded queue.
Andrey
Quoted 137 lines
Thanks,
Victor-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Wednesday, July 27, 2022 12:57 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 5/5] drm/amdgpu: reduce reset timeQuoted 21 lines · Victor
[AMD Official Use Only - General]
Hi Andrey,
Reply inline.
Thanks,
Victor-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:18 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 5/5] drm/amdgpu: reduce reset timeQuoted 8 lines
Quoted 3 lines · Victor Zhao
In multi container use case, reset time is important, so skip ring tests and cp halt wait during ip suspending for reset as they are going to fail and cost more time on reset
Why are they failing in this case ? Skipping ring tests is not the best idea as you loose important indicator of system's sanity. Is there any way to make them work ?
[Victor]: I've seen gfx ring test fail every time after a gfx engine hang. I thought it should be expected as gfx is in a bad state. Do you know the reason we have ring tests before reset? As we are going to reset the asic anyway. Another approach could be to make the skip mode2 only or reduce the wait time here.
I dug down in history and according to commit 'drm/amdgpu:unmap KCQ in gfx
hw_fini(v2)' you need to write to scratch register for completion of queue unmap operation so you defently don't want to just skip it. I agree in case that the ring is hung this has no point but remember that GPU reset can happen not only to a hunged ring but for other reasons (RAS, manual resete.t.c.) in which case you probably want to shut down gracefully here ? I see we have adev->ip_blocks[i].status.hang flag which you maybe can use here instead ?Quoted 54 lines
Quoted 42 lines
Signed-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 26 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-1@@ -477,7 +477,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev) kiq->pmf->kiq_unmap_queues (kiq_ring, &adev->gfx.compute_ring[i],RESET_QUEUES ,0 ,0 ); -if (adev->gfx.kiq.ring.sched.ready) +if (adev->gfx.kiq.ring.sched.ready && !amdgpu_in_reset (adev)) r =amdgpu_ring_test_helper (kiq_ring);spin_unlock (&adev->gfx.kiq.ring_lock);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-8@@ -5971,16 +5971,19 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)WREG32_SOC15 (GC ,0 , mmCP_ME_CNTL, tmp); } -for (i =0 ; i < adev->usec_timeout; i++) { -if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("failed to %s cp gfx\n" , enable ?"unhalt" :"halt" ); +if (!amdgpu_in_reset (adev)) { +for (i =0 ; i < adev->usec_timeout; i++) { +if (RREG32_SOC15 (GC ,0 , mmCP_STAT) ==0 ) +break ;drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) · line 5987
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +7
-0+udelay (1 ); + } +if (i >= adev->usec_timeout) +DRM_ERROR ("failed to %s cp gfx\n" , + enable ?"unhalt" :"halt" ); + }return 0 ; + }This change has impact beyond container case no ? We had no issue with this code during regular reset cases so why we would give up on this code which confirms CP is idle ? What is the side effect of skipping this during all GPU resets ?
Quoted 3 lines
Andrey
[Victor]: I see "failed to halt cp gfx" with regular reset cases as well when doing a gfx hang test using quark. I haven't seen a side effect with Mode1 reset yet but maybe shorten the wait time could be better?
Same as above i guess, it would indeed time out for a hung ring but GPU reset happens not only because of hung rings but for other reasons.
Andrey
Quoted 24 lines
Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0static int gfx_v10_0_cp_gfx_load_pfp_microcode(struct
amdgpu_device
*adev) @@ -7569,8 +7572,10 @@ static intgfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)for (i = 0; i < adev->gfx.num_gfx_rings; i++) kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i], PREEMPT_QUEUES, 0, 0);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-2- - return amdgpu_ring_test_helper(kiq_ring); +if (!amdgpu_in_reset (adev)) +return amdgpu_ring_test_helper (kiq_ring); +else + return 0; }#endif @@ -7610,6 +7615,7 @@ static int gfx_v10_0_hw_fini(void *handle)return 0 ; } +gfx_v10_0_cp_enable (adev, false);gfx_v10_0_enable_gui_idle_interrupt (adev, false);
Quoted 7 lines · Victor Zhao
To meet the requirement for multi container usecase which needs a quicker reset and not causing VRAM lost, adding the Mode2 reset handler for
sienna_cichlid. Adding aAMDGPU_SKIP_MODE2_RESETflag so driver can fallback to default reset method when mode2 reset failed and retry.- add mode2 reset handler for
sienna_cichlid
Seems to me ASIC specific changes should be in a seperate patch
Quoted 62 lines
- introduce
AMDGPU_SKIP_MODE2_RESETflag
- let mode2 reset fallback to default reset method if failedSigned-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 13 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c | 1 + drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 297 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h | 32 ++ .../pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h | 4 +- drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h | 3 +- .../amd/pm/swsmu/smu11/sienna_cichlid_ppt.c | 54 ++++ 15 files changed, 414 insertions(+), 5 deletions(-)create mode 100644
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.ccreate mode 100644drivers/gpu/drm/amd/amdgpu/sienna_cichlid.hdrivers/gpu/drm/amd/amdgpu/Makefile +1
-1@@ -75,7 +75,7 @@ amdgpu-y += \ vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \ vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o arct_reg_init.o mxgpu_nv.o \ nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o \ - nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o + sienna_cichlid.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o # add DF block amdgpu-y += \drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1
-0@@ -135,6 +135,7 @@ static void amdgpu_amdkfd_reset_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0@@ -5146,6 +5146,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, reset_context->job = job; reset_context->hive = hive; +/* * Build list of devices to reset. * In case we are in XGMI hive mode, resort the device list
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 5271
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3
-1@@ -5265,8 +5266,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,amdgpu_ras_resume (adev); }else { r =amdgpu_do_asic_reset (device_list_handle, reset_context); -if (r && r == -EAGAIN ) +if (r && r == -EAGAIN ) { +set_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags); + adev->asic_reset_res =0 ;
See my question bellow related to this set
Quoted 257 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-0goto retry; + } } skip_hw_reset:@@ -5694,6 +5698,7 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev;set_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags);set_bit (AMDGPU_SKIP_HW_RESET , &reset_context.flags); +set_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags); adev->no_hw_access = true; r =amdgpu_device_pre_asic_reset (adev, &reset_context);drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0@@ -71,6 +71,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags); r =amdgpu_device_gpu_recover (ring->adev, job, &reset_context);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +1
-0@@ -1949,6 +1949,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +13
-0@@ -23,6 +23,7 @@#include "amdgpu_reset.h" #include "aldebaran.h" +#include "sienna_cichlid.h" int amdgpu_reset_add_handler (struct amdgpu_reset_control *reset_ctl,struct amdgpu_reset_handler *handler)@@ -40,6 +41,9 @@ int amdgpu_reset_init(struct amdgpu_device *adev)case IP_VERSION (13 ,0 ,2 ): ret =aldebaran_reset_init (adev);break ; +case IP_VERSION (11 ,0 ,7 ): + ret =sienna_cichlid_reset_init (adev); +break ;default :break ; }@@ -55,6 +59,9 @@ int amdgpu_reset_fini(struct amdgpu_device *adev)case IP_VERSION (13 ,0 ,2 ): ret =aldebaran_reset_fini (adev);break ; +case IP_VERSION (11 ,0 ,7 ): + ret =sienna_cichlid_reset_fini (adev); +break ;default :break ; }@@ -67,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags)) +return -ENOSYS ; +if (adev->reset_cntl && adev->reset_cntl->get_reset_handler) reset_handler = adev->reset_cntl->get_reset_handler ( adev->reset_cntl, reset_context);@@ -83,6 +93,9 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,int ret;struct amdgpu_reset_handler *reset_handler =NULL ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags)) +return -ENOSYS ; +if (adev->reset_cntl) reset_handler = adev->reset_cntl->get_reset_handler ( adev->reset_cntl, reset_context);drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h +1
-0@@ -30,6 +30,7 @@ enum AMDGPU_RESET_FLAGS {AMDGPU_NEED_FULL_RESET =0 ,AMDGPU_SKIP_HW_RESET =1 , + AMDGPU_SKIP_MODE2_RESET = 2, }; struct amdgpu_reset_context {drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1
-0@@ -290,6 +290,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1
-0@@ -317,6 +317,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +1
-0@@ -529,6 +529,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +114
-0@@ -0,0 +1,297 @@ +/* +* Copyright 2021 Advanced Micro Devices, Inc. +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +* OTHER DEALINGS IN THE SOFTWARE. +* +*/ + +#include "sienna_cichlid.h" +#include "amdgpu_reset.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_dpm.h" +#include "amdgpu_job.h" +#include "amdgpu_ring.h" +#include "amdgpu_ras.h" +#include "amdgpu_psp.h" +#include "amdgpu_xgmi.h" + +static struct amdgpu_reset_handler * +sienna_cichlid_get_reset_handler (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) +{ +struct amdgpu_reset_handler *handler; +struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; + +if (reset_context->method !=AMD_RESET_METHOD_NONE ) { +list_for_each_entry (handler, &reset_ctl->reset_handlers, + handler_list) { +if (handler->reset_method == reset_context->method) +return handler; + } + }else { +list_for_each_entry (handler, &reset_ctl->reset_handlers, + handler_list) { +if (handler->reset_method ==AMD_RESET_METHOD_MODE2 && + adev->pm.fw_version >=0x3a5500 && + !amdgpu_sriov_vf (adev)) { + reset_context->method =AMD_RESET_METHOD_MODE2 ; +return handler; + } + } + } + +return NULL ; +} + +static int sienna_cichlid_mode2_suspend_ip (struct amdgpu_device *adev) +{ +int r, i; + +amdgpu_device_set_pg_state (adev,AMD_PG_STATE_UNGATE ); +amdgpu_device_set_cg_state (adev,AMD_CG_STATE_UNGATE ); + +for (i = adev->num_ip_blocks -1 ; i >=0 ; i--) { +if (!(adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_GFX || + adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_SDMA )) +continue ; + + r = adev->ip_blocks[i].version->funcs->suspend (adev); + +if (r) { +dev_err (adev->dev, +"suspend of IP block <%s> failed %d\n" , + adev->ip_blocks[i].version->funcs->name, r); +return r; + } + adev->ip_blocks[i].status.hw = false; + } + +return r; +} + +static int +sienna_cichlid_mode2_prepare_hwcontext (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) +{ +int r =0 ; +struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; + +if (!amdgpu_sriov_vf (adev)) + r =sienna_cichlid_mode2_suspend_ip (adev); + +return r; +} + +static void sienna_cichlid_async_reset (struct work_struct *work) +{ +struct amdgpu_reset_handler *handler; +struct amdgpu_reset_control *reset_ctl = + container_of(work, struct amdgpu_reset_control, reset_work); +struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; + +list_for_each_entry (handler, &reset_ctl->reset_handlers, + handler_list) { +if (handler->reset_method == reset_ctl->active_reset) { +dev_dbg (adev->dev,"Resetting device\n" ); + handler->do_reset (adev);
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c line 124
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +10
-0+break ; + } + } +} + +static int sienna_cichlid_mode2_reset (struct amdgpu_device *adev) +{ +/* disable BM */ +pci_clear_master (adev->pdev); + adev->asic_reset_res =amdgpu_dpm_mode2_reset (adev);
Why do you need to set adev->asic_reset_res here and not just return back result ?
Quoted 318 lines
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +173
-0+return adev->asic_reset_res; +} + +static int +sienna_cichlid_mode2_perform_reset (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; +int r; + + r =sienna_cichlid_mode2_reset (adev); +if (r) { +dev_err (adev->dev, +"ASIC reset failed with error, %d " , r); + } +return r; +} + +static int sienna_cichlid_mode2_restore_ip (struct amdgpu_device *adev) +{ +int i, r; +struct psp_context *psp = &adev->psp; + + r =psp_rlc_autoload_start (psp); +if (r) { +dev_err (adev->dev,"Failed to start rlc autoload\n" ); +return r; + } + +/* Reinit GFXHUB */ + adev->gfxhub.funcs->init (adev); + r = adev->gfxhub.funcs->gart_enable (adev); +if (r) { +dev_err (adev->dev,"GFXHUB gart reenable failed after reset\n" ); +return r; + } + +for (i =0 ; i < adev->num_ip_blocks; i++) { +if (adev->ip_blocks[i].version->type ==AMD_IP_BLOCK_TYPE_IH ) + r = adev->ip_blocks[i].version->funcs->resume (adev); +if (r) { +dev_err (adev->dev, +"resume of IP block <%s> failed %d\n" , + adev->ip_blocks[i].version->funcs->name, r); +return r; + } + + adev->ip_blocks[i].status.hw = true; + } + +for (i =0 ; i < adev->num_ip_blocks; i++) { +if (!(adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_GFX || + adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_SDMA )) +continue ; + r = adev->ip_blocks[i].version->funcs->resume (adev); +if (r) { +dev_err (adev->dev, +"resume of IP block <%s> failed %d\n" , + adev->ip_blocks[i].version->funcs->name, r); +return r; + } + + adev->ip_blocks[i].status.hw = true; + } + +for (i =0 ; i < adev->num_ip_blocks; i++) { +if (!(adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_GFX || + adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_SDMA )) +continue ; + +if (adev->ip_blocks[i].version->funcs->late_init) { + r = adev->ip_blocks[i].version->funcs->late_init ( + (void *)adev); +if (r) { +dev_err (adev->dev, +"late_init of IP block <%s> failed %d after reset\n" , + adev->ip_blocks[i].version->funcs->name, + r); +return r; + } + } + adev->ip_blocks[i].status.late_initialized = true; + } + +amdgpu_device_set_cg_state (adev,AMD_CG_STATE_GATE ); +amdgpu_device_set_pg_state (adev,AMD_PG_STATE_GATE ); + +return r; +} + +static int +sienna_cichlid_mode2_restore_hwcontext (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) +{ +int r; +struct amdgpu_device *tmp_adev = (struct amdgpu_device *)reset_ctl->handle; + +dev_info (tmp_adev->dev, +"GPU reset succeeded, trying to resume\n" ); + r =sienna_cichlid_mode2_restore_ip (tmp_adev); +if (r) + goto end; + +/* +* Add this ASIC as tracked as reset was already +* complete successfully. +*/ +amdgpu_register_gpu_instance (tmp_adev); + +/* Resume RAS */ +amdgpu_ras_resume (tmp_adev); + +amdgpu_irq_gpu_reset_resume_helper (tmp_adev); + + r =amdgpu_ib_ring_tests (tmp_adev); +if (r) { +dev_err (tmp_adev->dev, +"ib ring test failed (%d).\n" , r); + r = -EAGAIN ; + tmp_adev->asic_reset_res = r; + goto end; + } + +end: +if (r) +return -EAGAIN ; +else +return r; +} + +static struct amdgpu_reset_handler sienna_cichlid_mode2_handler = { + .reset_method =AMD_RESET_METHOD_MODE2 , + .prepare_env =NULL , + .prepare_hwcontext = sienna_cichlid_mode2_prepare_hwcontext, + .perform_reset = sienna_cichlid_mode2_perform_reset, + .restore_hwcontext = sienna_cichlid_mode2_restore_hwcontext, + .restore_env =NULL , + .do_reset = sienna_cichlid_mode2_reset, +}; + +int sienna_cichlid_reset_init (struct amdgpu_device *adev) +{ +struct amdgpu_reset_control *reset_ctl; + + reset_ctl =kzalloc (sizeof (*reset_ctl),GFP_KERNEL ); +if (!reset_ctl) +return -ENOMEM ; + + reset_ctl->handle = adev; + reset_ctl->async_reset = sienna_cichlid_async_reset; + reset_ctl->active_reset =AMD_RESET_METHOD_NONE ; + reset_ctl->get_reset_handler = sienna_cichlid_get_reset_handler; + +INIT_LIST_HEAD (&reset_ctl->reset_handlers); +INIT_WORK (&reset_ctl->reset_work, reset_ctl->async_reset); +/* Only mode2 is handled through reset control now */ +amdgpu_reset_add_handler (reset_ctl, &sienna_cichlid_mode2_handler); + + adev->reset_cntl = reset_ctl; + +return 0 ; +} + +int sienna_cichlid_reset_fini (struct amdgpu_device *adev) +{ +kfree (adev->reset_cntl); + adev->reset_cntl =NULL ; +return 0 ; +}drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h new file +32
-0@@ -0,0 +1,32 @@ +/* +* Copyright 2021 Advanced Micro Devices, Inc. +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR +* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +* OTHER DEALINGS IN THE SOFTWARE. +* +*/ + +#ifndef __SIENNA_CICHLID_H__ +#define __SIENNA_CICHLID_H__ + +#include "amdgpu.h" + +int sienna_cichlid_reset_init (struct amdgpu_device *adev); +int sienna_cichlid_reset_fini (struct amdgpu_device *adev); + +#endif drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h +2
-2@@ -137,7 +137,7 @@#define PPSMC_MSG_DisallowGpo 0x56#define PPSMC_MSG_Enable2ndUSB20Port 0x57 - -#define PPSMC_Message_Count 0x58 +#define PPSMC_MSG_DriverMode2Reset 0x5D +#define PPSMC_Message_Count 0x5E#endif drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h +2
-1@@ -235,7 +235,8 @@__SMU_DUMMY_MAP (UnforceGfxVid), \__SMU_DUMMY_MAP (HeavySBR), \__SMU_DUMMY_MAP (SetBadHBMPagesRetiredFlagsPerChannel), \ -__SMU_DUMMY_MAP (EnableGfxImu), +__SMU_DUMMY_MAP (EnableGfxImu), \ +__SMU_DUMMY_MAP (DriverMode2Reset), #undef __SMU_DUMMY_MAP#define __SMU_DUMMY_MAP (type) SMU_MSG_##typedrivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +54
-0@@ -154,6 +154,7 @@ static struct cmn2asic_msg_mapping sienna_cichlid_message_map[SMU_MSG_MAX_COUNT]MSG_MAP (SetGpoFeaturePMask, PPSMC_MSG_SetGpoFeaturePMask,0 ),MSG_MAP (DisallowGpo, PPSMC_MSG_DisallowGpo,0 ),MSG_MAP (Enable2ndUSB20Port, PPSMC_MSG_Enable2ndUSB20Port,0 ), + MSG_MAP(DriverMode2Reset, PPSMC_MSG_DriverMode2Reset, 0), }; static struct cmn2asic_mapping sienna_cichlid_clk_map[SMU_CLK_COUNT] = {@@ -4254,6 +4255,57 @@ static int sienna_cichlid_stb_get_data_direct(struct smu_context *smu,return 0 ; } +static bool sienna_cichlid_is_mode2_reset_supported (struct smu_context *smu) +{ +return true; +} + +static int sienna_cichlid_mode2_reset (struct smu_context *smu) +{ +u32 smu_version; +int ret =0 , index; +struct amdgpu_device *adev = smu->adev; +int timeout =100 ; + +smu_cmn_get_smc_version (smu,NULL , &smu_version); + + index =smu_cmn_to_asic_specific_index (smu,CMN2ASIC_MAPPING_MSG , + SMU_MSG_DriverMode2Reset); + +mutex_lock (&smu->message_lock); + + ret =smu_cmn_send_msg_without_waiting (smu, (uint16_t )index, +SMU_RESET_MODE_2 ); + + ret =smu_cmn_wait_for_response (smu); +while (ret !=0 && timeout) { + ret =smu_cmn_wait_for_response (smu); +/* Wait a bit more time for getting ACK */ +if (ret !=0 ) { + --timeout; +usleep_range (500 ,1000 ); +continue ; + }else { +break ; + } + } + +if (!timeout) { +dev_err (adev->dev, +"failed to send mode2 message \tparam: 0x%08x response %#x\n" , +SMU_RESET_MODE_2 , ret); + goto out; + } + +dev_info (smu->adev->dev,"restore config space...\n" ); +/* Restore the config space saved during init */ +amdgpu_device_load_pci_state (adev->pdev); +out: +mutex_unlock (&smu->message_lock); + +return ret; +} +static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_allowed_feature_mask = sienna_cichlid_get_allowed_feature_mask, .set_default_dpm_table = sienna_cichlid_set_default_dpm_table,@@ -4348,6 +4400,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_default_config_table_settings = sienna_cichlid_get_default_config_table_settings, .set_config_table = sienna_cichlid_set_config_table, .get_unique_id = sienna_cichlid_get_unique_id, + .mode2_reset_is_support = sienna_cichlid_is_mode2_reset_supported, + .mode2_reset = sienna_cichlid_mode2_reset, };void sienna_cichlid_set_ppt_funcs (struct smu_context *smu)
[AMD Official Use Only - General]
Will do.
Thanks,
Victor
-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, July 26, 2022 5:06 AM
To: Zhao, Victor <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>; Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>
Subject: Re: [PATCH 1/5] drm/amdgpu: add mode2 reset for sienna_cichlid
Quoted 7 lines · Victor Zhao
To meet the requirement for multi container usecase which needs a quicker reset and not causing VRAM lost, adding the Mode2 reset handler for
sienna_cichlid. Adding aAMDGPU_SKIP_MODE2_RESETflag so driver can fallback to default reset method when mode2 reset failed and retry.- add mode2 reset handler for
sienna_cichlid
Seems to me ASIC specific changes should be in a seperate patch
Quoted 70 lines
- introduce
AMDGPU_SKIP_MODE2_RESETflag
- let mode2 reset fallback to default reset method if failedSigned-off-by Victor Zhao <[email protected]>--- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 13 + drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 1 + drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c | 1 + drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 297 ++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h | 32 ++ .../pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h | 4 +- drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h | 3 +- .../amd/pm/swsmu/smu11/sienna_cichlid_ppt.c | 54 ++++ 15 files changed, 414 insertions(+), 5 deletions(-)create mode 100644
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.ccreate mode 100644drivers/gpu/drm/amd/amdgpu/sienna_cichlid.hdrivers/gpu/drm/amd/amdgpu/Makefile +2
-1@@ -75,7 +75,7 @@ amdgpu-y += \ vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \ vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o arct_reg_init.o mxgpu_nv.o \ nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o \ - nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o + sienna_cichlid.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o +lsdma_v6_0.o # add DF block amdgpu-y += \drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1
-0@@ -135,6 +135,7 @@ static void amdgpu_amdkfd_reset_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0@@ -5146,6 +5146,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, reset_context->job = job; reset_context->hive = hive; +/* * Build list of devices to reset. * In case we are in XGMI hive mode, resort the device list @@
-5265,8 +5266,11 @@ int amdgpu_device_gpu_recover(structamdgpu_device*adev,
amdgpu_ras_resume(adev);
} else {
r =amdgpu_do_asic_reset(device_list_handle, reset_context);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 5271
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3
-1-if (r && r == -EAGAIN ) +if (r && r == -EAGAIN ) { +set_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags); + adev->asic_reset_res =0 ;
See my question bellow related to this set
Quoted 276 lines
goto retry;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-0+ } } skip_hw_reset:@@ -5694,6 +5698,7 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev;set_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags);set_bit (AMDGPU_SKIP_HW_RESET , &reset_context.flags); +set_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags); adev->no_hw_access = true; r = amdgpu_device_pre_asic_reset(adev, &reset_context); diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index10fdd12cf853..9844d99075e9100644drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1
-0@@ -71,6 +71,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags); r =amdgpu_device_gpu_recover (ring->adev, job, &reset_context);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +1
-0@@ -1949,6 +1949,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (ras->adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +1
-0@@ -23,6 +23,7 @@#include "amdgpu_reset.h" #include "aldebaran.h" +#include "sienna_cichlid.h" int amdgpu_reset_add_handler (struct amdgpu_reset_control *reset_ctl, struct amdgpu_reset_handler *handler) @@ -40,6 +41,9 @@ int
amdgpu_reset_init(struct amdgpu_device *adev)
caseIP_VERSION(13, 0, 2):
ret =aldebaran_reset_init(adev);
break;drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +9
-0+case IP_VERSION (11 ,0 ,7 ): + ret =sienna_cichlid_reset_init (adev); +break ;default :break ; }@@ -55,6 +59,9 @@ int amdgpu_reset_fini(struct amdgpu_device *adev)case IP_VERSION (13 ,0 ,2 ): ret =aldebaran_reset_fini (adev);break ; +case IP_VERSION (11 ,0 ,7 ): + ret =sienna_cichlid_reset_fini (adev); +break ;default :break ; }@@ -67,6 +74,9 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, {struct amdgpu_reset_handler *reset_handler =NULL ; +if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags)) +return -ENOSYS ; +if (adev->reset_cntl && adev->reset_cntl->get_reset_handler) reset_handler = adev->reset_cntl->get_reset_handler ( adev->reset_cntl, reset_context); @@ -83,6 +93,9 @@ intamdgpu_reset_perform_reset(struct
amdgpu_device*adev,
int ret;
structamdgpu_reset_handler*reset_handler= NULL;drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +3
-0+if (test_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context->flags)) +return -ENOSYS ; +if (adev->reset_cntl) reset_handler = adev->reset_cntl->get_reset_handler ( adev->reset_cntl, reset_context); diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.hb/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
index9e55a5d7a825..cc4b2eeb24cf100644drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h +1
-0@@ -30,6 +30,7 @@ enum AMDGPU_RESET_FLAGS {AMDGPU_NEED_FULL_RESET =0 ,AMDGPU_SKIP_HW_RESET =1 , + AMDGPU_SKIP_MODE2_RESET = 2, }; struct amdgpu_reset_context {drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1
-0@@ -290,6 +290,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1
-0@@ -317,6 +317,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +1
-0@@ -529,6 +529,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work) reset_context.method =AMD_RESET_METHOD_NONE ; reset_context.reset_req_dev = adev;clear_bit (AMDGPU_NEED_FULL_RESET , &reset_context.flags); +clear_bit (AMDGPU_SKIP_MODE2_RESET , &reset_context.flags);amdgpu_device_gpu_recover (adev,NULL , &reset_context); }drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +125
-0@@ -0,0 +1,297 @@ +/* +* Copyright 2021 Advanced Micro Devices, Inc. +* +* Permission is hereby granted, free of charge, to any person +obtaining a +* copy of this software and associated documentation files (the +"Software"), +* to deal in the Software without restriction, including without +limitation +* the rights to use, copy, modify, merge, publish, distribute, +sublicense, +* and/or sell copies of the Software, and to permit persons to whom +the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be +included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL +* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, +DAMAGES OR +* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, +* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR +* OTHER DEALINGS IN THE SOFTWARE. +* +*/ + +#include "sienna_cichlid.h" +#include "amdgpu_reset.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_dpm.h" +#include "amdgpu_job.h" +#include "amdgpu_ring.h" +#include "amdgpu_ras.h" +#include "amdgpu_psp.h" +#include "amdgpu_xgmi.h" + +static struct amdgpu_reset_handler * +sienna_cichlid_get_reset_handler (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) { +struct amdgpu_reset_handler *handler; +struct amdgpu_device *adev = (struct amdgpu_device +*)reset_ctl->handle; + +if (reset_context->method !=AMD_RESET_METHOD_NONE ) { +list_for_each_entry (handler, &reset_ctl->reset_handlers, + handler_list) { +if (handler->reset_method == reset_context->method) +return handler; + } + }else { +list_for_each_entry (handler, &reset_ctl->reset_handlers, + handler_list) { +if (handler->reset_method ==AMD_RESET_METHOD_MODE2 && + adev->pm.fw_version >=0x3a5500 && + !amdgpu_sriov_vf (adev)) { + reset_context->method =AMD_RESET_METHOD_MODE2 ; +return handler; + } + } + } + +return NULL ; +} + +static int sienna_cichlid_mode2_suspend_ip (struct amdgpu_device +*adev) { +int r, i; + +amdgpu_device_set_pg_state (adev,AMD_PG_STATE_UNGATE ); +amdgpu_device_set_cg_state (adev,AMD_CG_STATE_UNGATE ); + +for (i = adev->num_ip_blocks -1 ; i >=0 ; i--) { +if (!(adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_GFX || + adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_SDMA )) +continue ; + + r = adev->ip_blocks[i].version->funcs->suspend (adev); + +if (r) { +dev_err (adev->dev, +"suspend of IP block <%s> failed %d\n" , + adev->ip_blocks[i].version->funcs->name, r); +return r; + } + adev->ip_blocks[i].status.hw = false; + } + +return r; +} + +static int +sienna_cichlid_mode2_prepare_hwcontext (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) { +int r =0 ; +struct amdgpu_device *adev = (struct amdgpu_device +*)reset_ctl->handle; + +if (!amdgpu_sriov_vf (adev)) + r =sienna_cichlid_mode2_suspend_ip (adev); + +return r; +} + +static void sienna_cichlid_async_reset (struct work_struct *work) { +struct amdgpu_reset_handler *handler; +struct amdgpu_reset_control *reset_ctl = + container_of(work, struct amdgpu_reset_control, reset_work); +struct amdgpu_device *adev = (struct amdgpu_device +*)reset_ctl->handle; + +list_for_each_entry (handler, &reset_ctl->reset_handlers, + handler_list) { +if (handler->reset_method == reset_ctl->active_reset) { +dev_dbg (adev->dev,"Resetting device\n" );
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c line 135
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +10
-0+ handler->do_reset (adev); +break ; + } + } +} + +static int sienna_cichlid_mode2_reset (struct amdgpu_device *adev) { +/* disable BM */ +pci_clear_master (adev->pdev); + adev->asic_reset_res =amdgpu_dpm_mode2_reset (adev);
Why do you need to set adev->asic_reset_res here and not just return back result ?
Quoted 331 lines
drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c new file +170
-0+return adev->asic_reset_res; +} + +static int +sienna_cichlid_mode2_perform_reset (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) { +struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; +int r; + + r =sienna_cichlid_mode2_reset (adev); +if (r) { +dev_err (adev->dev, +"ASIC reset failed with error, %d " , r); + } +return r; +} + +static int sienna_cichlid_mode2_restore_ip (struct amdgpu_device +*adev) { +int i, r; +struct psp_context *psp = &adev->psp; + + r =psp_rlc_autoload_start (psp); +if (r) { +dev_err (adev->dev,"Failed to start rlc autoload\n" ); +return r; + } + +/* Reinit GFXHUB */ + adev->gfxhub.funcs->init (adev); + r = adev->gfxhub.funcs->gart_enable (adev); +if (r) { +dev_err (adev->dev,"GFXHUB gart reenable failed after reset\n" ); +return r; + } + +for (i =0 ; i < adev->num_ip_blocks; i++) { +if (adev->ip_blocks[i].version->type ==AMD_IP_BLOCK_TYPE_IH ) + r = adev->ip_blocks[i].version->funcs->resume (adev); +if (r) { +dev_err (adev->dev, +"resume of IP block <%s> failed %d\n" , + adev->ip_blocks[i].version->funcs->name, r); +return r; + } + + adev->ip_blocks[i].status.hw = true; + } + +for (i =0 ; i < adev->num_ip_blocks; i++) { +if (!(adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_GFX || + adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_SDMA )) +continue ; + r = adev->ip_blocks[i].version->funcs->resume (adev); +if (r) { +dev_err (adev->dev, +"resume of IP block <%s> failed %d\n" , + adev->ip_blocks[i].version->funcs->name, r); +return r; + } + + adev->ip_blocks[i].status.hw = true; + } + +for (i =0 ; i < adev->num_ip_blocks; i++) { +if (!(adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_GFX || + adev->ip_blocks[i].version->type == +AMD_IP_BLOCK_TYPE_SDMA )) +continue ; + +if (adev->ip_blocks[i].version->funcs->late_init) { + r = adev->ip_blocks[i].version->funcs->late_init ( + (void *)adev); +if (r) { +dev_err (adev->dev, +"late_init of IP block <%s> failed %d after reset\n" , + adev->ip_blocks[i].version->funcs->name, + r); +return r; + } + } + adev->ip_blocks[i].status.late_initialized = true; + } + +amdgpu_device_set_cg_state (adev,AMD_CG_STATE_GATE ); +amdgpu_device_set_pg_state (adev,AMD_PG_STATE_GATE ); + +return r; +} + +static int +sienna_cichlid_mode2_restore_hwcontext (struct amdgpu_reset_control *reset_ctl, +struct amdgpu_reset_context *reset_context) { +int r; +struct amdgpu_device *tmp_adev = (struct amdgpu_device +*)reset_ctl->handle; + +dev_info (tmp_adev->dev, +"GPU reset succeeded, trying to resume\n" ); + r =sienna_cichlid_mode2_restore_ip (tmp_adev); +if (r) + goto end; + +/* +* Add this ASIC as tracked as reset was already +* complete successfully. +*/ +amdgpu_register_gpu_instance (tmp_adev); + +/* Resume RAS */ +amdgpu_ras_resume (tmp_adev); + +amdgpu_irq_gpu_reset_resume_helper (tmp_adev); + + r =amdgpu_ib_ring_tests (tmp_adev); +if (r) { +dev_err (tmp_adev->dev, +"ib ring test failed (%d).\n" , r); + r = -EAGAIN ; + tmp_adev->asic_reset_res = r; + goto end; + } + +end: +if (r) +return -EAGAIN ; +else +return r; +} + +static struct amdgpu_reset_handler sienna_cichlid_mode2_handler = { + .reset_method =AMD_RESET_METHOD_MODE2 , + .prepare_env =NULL , + .prepare_hwcontext = sienna_cichlid_mode2_prepare_hwcontext, + .perform_reset = sienna_cichlid_mode2_perform_reset, + .restore_hwcontext = sienna_cichlid_mode2_restore_hwcontext, + .restore_env =NULL , + .do_reset = sienna_cichlid_mode2_reset, +}; + +int sienna_cichlid_reset_init (struct amdgpu_device *adev) { +struct amdgpu_reset_control *reset_ctl; + + reset_ctl =kzalloc (sizeof (*reset_ctl),GFP_KERNEL ); +if (!reset_ctl) +return -ENOMEM ; + + reset_ctl->handle = adev; + reset_ctl->async_reset = sienna_cichlid_async_reset; + reset_ctl->active_reset =AMD_RESET_METHOD_NONE ; + reset_ctl->get_reset_handler = sienna_cichlid_get_reset_handler; + +INIT_LIST_HEAD (&reset_ctl->reset_handlers); +INIT_WORK (&reset_ctl->reset_work, reset_ctl->async_reset); +/* Only mode2 is handled through reset control now */ +amdgpu_reset_add_handler (reset_ctl, &sienna_cichlid_mode2_handler); + + adev->reset_cntl = reset_ctl; + +return 0 ; +} + +int sienna_cichlid_reset_fini (struct amdgpu_device *adev) { +kfree (adev->reset_cntl); + adev->reset_cntl =NULL ; +return 0 ; +}drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h new file +44
-0@@ -0,0 +1,32 @@ +/* +* Copyright 2021 Advanced Micro Devices, Inc. +* +* Permission is hereby granted, free of charge, to any person +obtaining a +* copy of this software and associated documentation files (the +"Software"), +* to deal in the Software without restriction, including without +limitation +* the rights to use, copy, modify, merge, publish, distribute, +sublicense, +* and/or sell copies of the Software, and to permit persons to whom +the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be +included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL +* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, +DAMAGES OR +* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, +* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR +* OTHER DEALINGS IN THE SOFTWARE. +* +*/ + +#ifndef __SIENNA_CICHLID_H__ +#define __SIENNA_CICHLID_H__ + +#include "amdgpu.h" + +int sienna_cichlid_reset_init (struct amdgpu_device *adev);int +sienna_cichlid_reset_fini (struct amdgpu_device *adev); + +#endif (quoted diff) +0
-0
a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.hb/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.hindexd2e10a724560..82cf9e563065100644drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h +2
-2@@ -137,7 +137,7 @@#define PPSMC_MSG_DisallowGpo 0x56#define PPSMC_MSG_Enable2ndUSB20Port 0x57 - -#define PPSMC_Message_Count 0x58 +#define PPSMC_MSG_DriverMode2Reset 0x5D +#define PPSMC_Message_Count 0x5E#endif drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h +2
-1@@ -235,7 +235,8 @@__SMU_DUMMY_MAP (UnforceGfxVid), \__SMU_DUMMY_MAP (HeavySBR), \__SMU_DUMMY_MAP (SetBadHBMPagesRetiredFlagsPerChannel), \ -__SMU_DUMMY_MAP (EnableGfxImu), +__SMU_DUMMY_MAP (EnableGfxImu), \ +__SMU_DUMMY_MAP (DriverMode2Reset), #undef __SMU_DUMMY_MAP#define __SMU_DUMMY_MAP (type) SMU_MSG_##typedrivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +1
-0@@ -154,6 +154,7 @@ static struct cmn2asic_msg_mapping sienna_cichlid_message_map[SMU_MSG_MAX_COUNT]MSG_MAP (SetGpoFeaturePMask, PPSMC_MSG_SetGpoFeaturePMask,0 ),MSG_MAP (DisallowGpo, PPSMC_MSG_DisallowGpo,0 ),MSG_MAP (Enable2ndUSB20Port, PPSMC_MSG_Enable2ndUSB20Port,0 ), + MSG_MAP(DriverMode2Reset, PPSMC_MSG_DriverMode2Reset, 0), };static struct cmn2asic_mapping sienna_cichlid_clk_map[SMU_CLK_COUNT ]= { @@ -4254,6 +4255,57 @@ static int sienna_cichlid_stb_get_data_direct(struct
smu_context*smu,
return 0;
}drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +52
-0+static bool sienna_cichlid_is_mode2_reset_supported (struct +smu_context *smu) { +return true; +} + +static int sienna_cichlid_mode2_reset (struct smu_context *smu) { +u32 smu_version; +int ret =0 , index; +struct amdgpu_device *adev = smu->adev; +int timeout =100 ; + +smu_cmn_get_smc_version (smu,NULL , &smu_version); + + index =smu_cmn_to_asic_specific_index (smu,CMN2ASIC_MAPPING_MSG , + SMU_MSG_DriverMode2Reset); + +mutex_lock (&smu->message_lock); + + ret =smu_cmn_send_msg_without_waiting (smu, (uint16_t )index, +SMU_RESET_MODE_2 ); + + ret =smu_cmn_wait_for_response (smu); +while (ret !=0 && timeout) { + ret =smu_cmn_wait_for_response (smu); +/* Wait a bit more time for getting ACK */ +if (ret !=0 ) { + --timeout; +usleep_range (500 ,1000 ); +continue ; + }else { +break ; + } + } + +if (!timeout) { +dev_err (adev->dev, +"failed to send mode2 message \tparam: 0x%08x response %#x\n" , +SMU_RESET_MODE_2 , ret); + goto out; + } + +dev_info (smu->adev->dev,"restore config space...\n" ); +/* Restore the config space saved during init */ +amdgpu_device_load_pci_state (adev->pdev); +out: +mutex_unlock (&smu->message_lock); + +return ret; +} +static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_allowed_feature_mask = sienna_cichlid_get_allowed_feature_mask, .set_default_dpm_table = sienna_cichlid_set_default_dpm_table,@@ -4348,6 +4400,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_default_config_table_settings = sienna_cichlid_get_default_config_table_settings, .set_config_table = sienna_cichlid_set_config_table, .get_unique_id = sienna_cichlid_get_unique_id, + .mode2_reset_is_support = sienna_cichlid_is_mode2_reset_supported, + .mode2_reset = sienna_cichlid_mode2_reset, };void sienna_cichlid_set_ppt_funcs (struct smu_context *smu)