Skip to contentSkip to conversation

drm/amd/pm: expose swctf threshold setting for legacy powerplay

PATCH In review Evan Quan 2 participants 5 messages Jun 26, 2023 – Jun 27, 2023

Subsystems (3)
lore
Share & tools
lore

Evan Quan

author
#0 lore raw

PATCH 1/2 drm/amd/pm: expose swctf threshold setting for legacy powerplay

Preparation for coming optimization which eliminates the influence of GPU temperature momentary fluctuation.

---
 drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h                |  2 ++
 .../gpu/drm/amd/pm/powerplay/hwmgr/hardwaremanager.c   |  4 +++-
 drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c    |  2 ++
 drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c  | 10 ++++++++++
 drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c  |  4 ++++
 drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c  |  4 ++++
 drivers/gpu/drm/amd/pm/powerplay/inc/power_state.h     |  1 +
 7 files changed, 26 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h +2 -0
index a9161f3da8b5..7faad759a6cc 100644
@@ -90,6 +90,8 @@ struct amdgpu_dpm_thermal {
 	int                max_mem_crit_temp;
 	/* memory max emergency(shutdown) temp */
 	int                max_mem_emergency_temp;
+	/* SWCTF threshold */
+	int                sw_ctf_threshold;
 	/* was last interrupt low to high or high to low */
 	bool               high_to_low;
 	/* interrupt source */
drivers/gpu/drm/amd/pm/powerplay/hwmgr/hardwaremanager.c +3 -1
index 981dc8c7112d..90452b66e107 100644
@@ -241,7 +241,8 @@ int phm_start_thermal_controller(struct pp_hwmgr *hwmgr)
 		TEMP_RANGE_MAX,
 		TEMP_RANGE_MIN,
 		TEMP_RANGE_MAX,
-		TEMP_RANGE_MAX};
+		TEMP_RANGE_MAX,
+		0};
 	struct amdgpu_device *adev = hwmgr->adev;
 
 	if (!hwmgr->not_vf)
@@ -265,6 +266,7 @@ int phm_start_thermal_controller(struct pp_hwmgr *hwmgr)
 	adev->pm.dpm.thermal.min_mem_temp = range.mem_min;
 	adev->pm.dpm.thermal.max_mem_crit_temp = range.mem_crit_max;
 	adev->pm.dpm.thermal.max_mem_emergency_temp = range.mem_emergency_max;
+	adev->pm.dpm.thermal.sw_ctf_threshold = range.sw_ctf_threshold;
 
 	return ret;
 }
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +2 -0
index d82767866ac1..6d887ead2967 100644
@@ -5433,6 +5433,8 @@ static int smu7_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 		thermal_data->max = data->thermal_temp_setting.temperature_shutdown *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 
+	thermal_data->sw_ctf_threshold = thermal_data->max;
+
 	return 0;
 }
 
drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +10 -0
index 6f5161738bf8..d8cd23438b76 100644
@@ -5242,6 +5242,9 @@ static int vega10_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 {
 	struct vega10_hwmgr *data = hwmgr->backend;
 	PPTable_t *pp_table = &(data->smc_state_table.pp_table);
+	struct phm_ppt_v2_information *pp_table_info =
+		(struct phm_ppt_v2_information *)(hwmgr->pptable);
+	struct phm_tdp_table *tdp_table = pp_table_info->tdp_table;
 
 	memcpy(thermal_data, &SMU7ThermalWithDelayPolicy[0], sizeof(struct PP_TemperatureRange));
 
@@ -5258,6 +5261,13 @@ static int vega10_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 	thermal_data->mem_emergency_max = (pp_table->ThbmLimit + CTF_OFFSET_HBM)*
 		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 
+	if (tdp_table->usSoftwareShutdownTemp > pp_table->ThotspotLimit &&
+	    tdp_table->usSoftwareShutdownTemp < VEGA10_THERMAL_MAXIMUM_ALERT_TEMP)
+		thermal_data->sw_ctf_threshold = tdp_table->usSoftwareShutdownTemp;
+	else
+		thermal_data->sw_ctf_threshold = VEGA10_THERMAL_MAXIMUM_ALERT_TEMP;
+	thermal_data->sw_ctf_threshold *= PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
 	return 0;
 }
 
drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c +4 -0
index 33f31461ea6c..1069eaaae2f8 100644
@@ -2764,6 +2764,8 @@ static int vega12_notify_cac_buffer_info(struct pp_hwmgr *hwmgr,
 static int vega12_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 		struct PP_TemperatureRange *thermal_data)
 {
+	struct phm_ppt_v3_information *pptable_information =
+		(struct phm_ppt_v3_information *)hwmgr->pptable;
 	struct vega12_hwmgr *data =
 			(struct vega12_hwmgr *)(hwmgr->backend);
 	PPTable_t *pp_table = &(data->smc_state_table.pp_table);
@@ -2782,6 +2784,8 @@ static int vega12_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 	thermal_data->mem_emergency_max = (pp_table->ThbmLimit + CTF_OFFSET_HBM)*
 		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	thermal_data->sw_ctf_threshold = pptable_information->us_software_shutdown_temp *
+		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 
 	return 0;
 }
drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c +4 -0
index 2a5abac81b4a..ff77a3683efd 100644
@@ -4207,6 +4207,8 @@ static int vega20_notify_cac_buffer_info(struct pp_hwmgr *hwmgr,
 static int vega20_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 		struct PP_TemperatureRange *thermal_data)
 {
+	struct phm_ppt_v3_information *pptable_information =
+		(struct phm_ppt_v3_information *)hwmgr->pptable;
 	struct vega20_hwmgr *data =
 			(struct vega20_hwmgr *)(hwmgr->backend);
 	PPTable_t *pp_table = &(data->smc_state_table.pp_table);
@@ -4225,6 +4227,8 @@ static int vega20_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
 		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 	thermal_data->mem_emergency_max = (pp_table->ThbmLimit + CTF_OFFSET_HBM)*
 		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	thermal_data->sw_ctf_threshold = pptable_information->us_software_shutdown_temp *
+		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 
 	return 0;
 }
drivers/gpu/drm/amd/pm/powerplay/inc/power_state.h +1 -0
index a5f2227a3971..0ffc2347829d 100644
@@ -131,6 +131,7 @@ struct PP_TemperatureRange {
 	int mem_min;
 	int mem_crit_max;
 	int mem_emergency_max;
+	int sw_ctf_threshold;
 };
 
 struct PP_StateValidationBlock {
Signature · 1 line
2.34.1
author
#1 lore raw

PATCH 2/2 drm/amd/pm: avoid unintentional shutdown due to temperature momentary fluctuation

An intentional delay is added on soft ctf triggered. Then there will be a double check for the GPU temperature before taking further action. This can avoid unintended shutdown due to temperature momentary fluctuation.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h           |  3 ++
 .../gpu/drm/amd/pm/powerplay/amd_powerplay.c  | 48 +++++++++++++++++++
 .../drm/amd/pm/powerplay/hwmgr/smu_helper.c   | 27 ++++-------
 drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h  |  2 +
 drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c     | 34 +++++++++++++
 drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h |  2 +
 .../gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c    |  9 +---
 .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c    |  9 +---
 8 files changed, 102 insertions(+), 32 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +3 -0
index e459381dc759..5ef1f31e703c 100644
@@ -287,6 +287,9 @@ extern int amdgpu_user_partt_mode;
 #define AMDGPU_SMARTSHIFT_MAX_BIAS (100)
 #define AMDGPU_SMARTSHIFT_MIN_BIAS (-100)
 
+/* Extra time delay(in ms) to eliminate the influence of temperature momentary fluctuation */
+#define AMDGPU_SWCTF_EXTRA_DELAY		50
+
 struct amdgpu_xcp_mgr;
 struct amdgpu_device;
 struct amdgpu_irq_src;
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +48 -0

78 lines not on this page · raw message

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +9 -18
index bfe80ac0ad8c..d0b1ab6c4523 100644
@@ -603,21 +603,17 @@ int phm_irq_process(struct amdgpu_device *adev,
 			   struct amdgpu_irq_src *source,
 			   struct amdgpu_iv_entry *entry)
 {
+	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 	uint32_t client_id = entry->client_id;
 	uint32_t src_id = entry->src_id;
 
 	if (client_id == AMDGPU_IRQ_CLIENTID_LEGACY) {
 		if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH) {
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
-		} else if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW)
+			schedule_delayed_work(&hwmgr->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
+		} else if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW) {
 			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected!\n");
-		else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
+		} else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
 			dev_emerg(adev->dev, "ERROR: GPU HW Critical Temperature Fault(aka CTF) detected!\n");
 			/*
 			 * HW CTF just occurred. Shutdown to prevent further damage.
@@ -626,15 +622,10 @@ int phm_irq_process(struct amdgpu_device *adev,
 			orderly_poweroff(true);
 		}
 	} else if (client_id == SOC15_IH_CLIENTID_THM) {
-		if (src_id == 0) {
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
-		} else
+		if (src_id == 0)
+			schedule_delayed_work(&hwmgr->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
+		else
 			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected!\n");
 	} else if (client_id == SOC15_IH_CLIENTID_ROM_SMUIO) {
 		dev_emerg(adev->dev, "ERROR: GPU HW Critical Temperature Fault(aka CTF) detected!\n");
drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h +2 -0
index f1580a26a850..612d66aeaab9 100644
@@ -811,6 +811,8 @@ struct pp_hwmgr {
 	bool gfxoff_state_changed_by_workload;
 	uint32_t pstate_sclk_peak;
 	uint32_t pstate_mclk_peak;
+
+	struct delayed_work swctf_delayed_work;
 };
 
 int hwmgr_early_init(struct pp_hwmgr *hwmgr);
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +34 -0
index 2aecb8faf407..4062b9b46986 100644
@@ -24,6 +24,7 @@
 
 #include <linux/firmware.h>
 #include <linux/pci.h>
+#include <linux/reboot.h>
 
 #include "amdgpu.h"
 #include "amdgpu_smu.h"
@@ -1078,6 +1079,34 @@ static void smu_interrupt_work_fn(struct work_struct *work)
 		smu->ppt_funcs->interrupt_work(smu);
 }
 
+static void smu_swctf_delayed_work_handler(struct work_struct *work)
+{
+	struct smu_context *smu =
+		container_of(work, struct smu_context, swctf_delayed_work.work);
+	struct smu_temperature_range *range =
+				&smu->thermal_range;
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t hotspot_tmp, size;
+
+	/*
+	 * If the hotspot temperature is confirmed as below SW CTF setting point
+	 * after the delay enforced, nothing will be done.
+	 * Otherwise, a graceful shutdown will be performed to prevent further damage.
+	 */
+	if (range->software_shutdown_temp &&
+	    smu->ppt_funcs->read_sensor &&
+	    !smu->ppt_funcs->read_sensor(smu,
+					 AMDGPU_PP_SENSOR_HOTSPOT_TEMP,
+					 &hotspot_tmp,
+					 &size) &&
+	    hotspot_tmp / 1000 < range->software_shutdown_temp)
+		return;
+
+	dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
+	dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
+	orderly_poweroff(true);
+}
+
 static int smu_sw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -1120,6 +1149,9 @@ static int smu_sw_init(void *handle)
 	smu->smu_dpm.dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
 	smu->smu_dpm.requested_dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
 
+	INIT_DELAYED_WORK(&smu->swctf_delayed_work,
+			  smu_swctf_delayed_work_handler);
+
 	ret = smu_smc_table_sw_init(smu);
 	if (ret) {
 		dev_err(adev->dev, "Failed to sw init smc table!\n");
@@ -1600,6 +1632,8 @@ static int smu_smc_hw_cleanup(struct smu_context *smu)
 		return ret;
 	}
 
+	cancel_delayed_work_sync(&smu->swctf_delayed_work);
+
 	ret = smu_disable_dpms(smu);
 	if (ret) {
 		dev_err(adev->dev, "Fail to disable dpm features!\n");
drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +2 -0
index 4ce394903c07..18101ec0ae6e 100644
@@ -573,6 +573,8 @@ struct smu_context
 	u32 debug_param_reg;
 	u32 debug_msg_reg;
 	u32 debug_resp_reg;
+
+	struct delayed_work		swctf_delayed_work;
 };
 
 struct i2c_adapter;
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +2 -7
index e1ef88ee1ed3..aa4a5498a12f 100644
@@ -1412,13 +1412,8 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev,
 	if (client_id == SOC15_IH_CLIENTID_THM) {
 		switch (src_id) {
 		case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
+			schedule_delayed_work(&smu->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
 		break;
 		case THM_11_0__SRCID__THM_DIG_THERM_H2L:
 			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected\n");
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +2 -7
index 9e8414037638..4071bfa38727 100644
@@ -1353,13 +1353,8 @@ static int smu_v13_0_irq_process(struct amdgpu_device *adev,
 	if (client_id == SOC15_IH_CLIENTID_THM) {
 		switch (src_id) {
 		case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
+			schedule_delayed_work(&smu->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
 			break;
 		case THM_11_0__SRCID__THM_DIG_THERM_H2L:
 			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected\n");
Signature · 1 line
2.34.1
Reviewed-by on v1 2/2
#2 lore raw

Re: [PATCH 2/2] drm/amd/pm: avoid unintentional shutdown due to temperature momentary fluctuation

Quoted 22 lines · Evan Quan

An intentional delay is added on soft ctf triggered. Then there will be a double check for the GPU temperature before taking further action. This can avoid unintended shutdown due to temperature momentary fluctuation.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h           |  3 ++
  .../gpu/drm/amd/pm/powerplay/amd_powerplay.c  | 48 +++++++++++++++++++
  .../drm/amd/pm/powerplay/hwmgr/smu_helper.c   | 27 ++++-------
  drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h  |  2 +
  drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c     | 34 +++++++++++++
  drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h |  2 +
  .../gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c    |  9 +---
  .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c    |  9 +---
  8 files changed, 102 insertions(+), 32 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0 -0
index e459381dc759..5ef1f31e703c 100644

drivers/gpu/drm/amd/amdgpu/amdgpu.h extern int amdgpu_user_partt_mode; · line 291

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
@@ -287,6 +287,9 @@ extern int amdgpu_user_partt_mode;
  #define AMDGPU_SMARTSHIFT_MAX_BIAS (100)
  #define AMDGPU_SMARTSHIFT_MIN_BIAS (-100)
 
+/* Extra time delay(in ms) to eliminate the influence of temperature momentary fluctuation */
+#define AMDGPU_SWCTF_EXTRA_DELAY		50

I think a delay of 10-15ms is good enough to filter out any spike.

With that change, the series is

Thanks,
Lijo

Quoted 274 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+
  struct amdgpu_xcp_mgr;
  struct amdgpu_device;
  struct amdgpu_irq_src;
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +48 -0
index 11b7b4cffaae..ff360c699171 100644
@@ -26,6 +26,7 @@
  #include <linux/gfp.h>
  #include <linux/slab.h>
  #include <linux/firmware.h>
+#include <linux/reboot.h>
  #include "amd_shared.h"
  #include "amd_powerplay.h"
  #include "power_state.h"
@@ -91,6 +92,45 @@ static int pp_early_init(void *handle)
  	return 0;
  }
 
+static void pp_swctf_delayed_work_handler(struct work_struct *work)
+{
+	struct pp_hwmgr *hwmgr =
+		container_of(work, struct pp_hwmgr, swctf_delayed_work.work);
+	struct amdgpu_device *adev = hwmgr->adev;
+	struct amdgpu_dpm_thermal *range =
+				&adev->pm.dpm.thermal;
+	uint32_t gpu_temperature, size;
+	int ret;
+
+	/*
+	 * If the hotspot/edge temperature is confirmed as below SW CTF setting point
+	 * after the delay enforced, nothing will be done.
+	 * Otherwise, a graceful shutdown will be performed to prevent further damage.
+	 */
+	if (range->sw_ctf_threshold &&
+	    hwmgr->hwmgr_func->read_sensor) {
+		ret = hwmgr->hwmgr_func->read_sensor(hwmgr,
+						     AMDGPU_PP_SENSOR_HOTSPOT_TEMP,
+						     &gpu_temperature,
+						     &size);
+		/*
+		 * For some legacy ASICs, hotspot temperature retrieving might be not
+		 * supported. Check the edge temperature instead then.
+		 */
+		if (ret == -EOPNOTSUPP)
+			ret = hwmgr->hwmgr_func->read_sensor(hwmgr,
+							     AMDGPU_PP_SENSOR_EDGE_TEMP,
+							     &gpu_temperature,
+							     &size);
+		if (!ret && gpu_temperature / 1000 < range->sw_ctf_threshold)
+			return;
+	}
+
+	dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
+	dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
+	orderly_poweroff(true);
+}
+
  static int pp_sw_init(void *handle)
  {
  	struct amdgpu_device *adev = handle;
@@ -101,6 +141,10 @@ static int pp_sw_init(void *handle)
 
  	pr_debug("powerplay sw init %s\n", ret ? "failed" : "successfully");
 
+	if (!ret)
+		INIT_DELAYED_WORK(&hwmgr->swctf_delayed_work,
+				  pp_swctf_delayed_work_handler);
+
  	return ret;
  }
 
@@ -135,6 +179,8 @@ static int pp_hw_fini(void *handle)
  	struct amdgpu_device *adev = handle;
  	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
+	cancel_delayed_work_sync(&hwmgr->swctf_delayed_work);
+
  	hwmgr_hw_fini(hwmgr);
 
  	return 0;
@@ -221,6 +267,8 @@ static int pp_suspend(void *handle)
  	struct amdgpu_device *adev = handle;
  	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
+	cancel_delayed_work_sync(&hwmgr->swctf_delayed_work);
+
  	return hwmgr_suspend(hwmgr);
  }
 
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +9 -18
index bfe80ac0ad8c..d0b1ab6c4523 100644
@@ -603,21 +603,17 @@ int phm_irq_process(struct amdgpu_device *adev,
  			   struct amdgpu_irq_src *source,
  			   struct amdgpu_iv_entry *entry)
  {
+	struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
  	uint32_t client_id = entry->client_id;
  	uint32_t src_id = entry->src_id;
 
  	if (client_id == AMDGPU_IRQ_CLIENTID_LEGACY) {
  		if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH) {
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
-		} else if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW)
+			schedule_delayed_work(&hwmgr->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
+		} else if (src_id == VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW) {
  			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected!\n");
-		else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
+		} else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
  			dev_emerg(adev->dev, "ERROR: GPU HW Critical Temperature Fault(aka CTF) detected!\n");
  			/*
  			 * HW CTF just occurred. Shutdown to prevent further damage.
@@ -626,15 +622,10 @@ int phm_irq_process(struct amdgpu_device *adev,
  			orderly_poweroff(true);
  		}
  	} else if (client_id == SOC15_IH_CLIENTID_THM) {
-		if (src_id == 0) {
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
-		} else
+		if (src_id == 0)
+			schedule_delayed_work(&hwmgr->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
+		else
  			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected!\n");
  	} else if (client_id == SOC15_IH_CLIENTID_ROM_SMUIO) {
  		dev_emerg(adev->dev, "ERROR: GPU HW Critical Temperature Fault(aka CTF) detected!\n");
drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h +2 -0
index f1580a26a850..612d66aeaab9 100644
@@ -811,6 +811,8 @@ struct pp_hwmgr {
  	bool gfxoff_state_changed_by_workload;
  	uint32_t pstate_sclk_peak;
  	uint32_t pstate_mclk_peak;
+
+	struct delayed_work swctf_delayed_work;
  };
 
  int hwmgr_early_init(struct pp_hwmgr *hwmgr);
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +34 -0
index 2aecb8faf407..4062b9b46986 100644
@@ -24,6 +24,7 @@
 
  #include <linux/firmware.h>
  #include <linux/pci.h>
+#include <linux/reboot.h>
 
  #include "amdgpu.h"
  #include "amdgpu_smu.h"
@@ -1078,6 +1079,34 @@ static void smu_interrupt_work_fn(struct work_struct *work)
  		smu->ppt_funcs->interrupt_work(smu);
  }
 
+static void smu_swctf_delayed_work_handler(struct work_struct *work)
+{
+	struct smu_context *smu =
+		container_of(work, struct smu_context, swctf_delayed_work.work);
+	struct smu_temperature_range *range =
+				&smu->thermal_range;
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t hotspot_tmp, size;
+
+	/*
+	 * If the hotspot temperature is confirmed as below SW CTF setting point
+	 * after the delay enforced, nothing will be done.
+	 * Otherwise, a graceful shutdown will be performed to prevent further damage.
+	 */
+	if (range->software_shutdown_temp &&
+	    smu->ppt_funcs->read_sensor &&
+	    !smu->ppt_funcs->read_sensor(smu,
+					 AMDGPU_PP_SENSOR_HOTSPOT_TEMP,
+					 &hotspot_tmp,
+					 &size) &&
+	    hotspot_tmp / 1000 < range->software_shutdown_temp)
+		return;
+
+	dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
+	dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
+	orderly_poweroff(true);
+}
+
  static int smu_sw_init(void *handle)
  {
  	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -1120,6 +1149,9 @@ static int smu_sw_init(void *handle)
  	smu->smu_dpm.dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
  	smu->smu_dpm.requested_dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
 
+	INIT_DELAYED_WORK(&smu->swctf_delayed_work,
+			  smu_swctf_delayed_work_handler);
+
  	ret = smu_smc_table_sw_init(smu);
  	if (ret) {
  		dev_err(adev->dev, "Failed to sw init smc table!\n");
@@ -1600,6 +1632,8 @@ static int smu_smc_hw_cleanup(struct smu_context *smu)
  		return ret;
  	}
 
+	cancel_delayed_work_sync(&smu->swctf_delayed_work);
+
  	ret = smu_disable_dpms(smu);
  	if (ret) {
  		dev_err(adev->dev, "Fail to disable dpm features!\n");
drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +2 -0
index 4ce394903c07..18101ec0ae6e 100644
@@ -573,6 +573,8 @@ struct smu_context
  	u32 debug_param_reg;
  	u32 debug_msg_reg;
  	u32 debug_resp_reg;
+
+	struct delayed_work		swctf_delayed_work;
  };
 
  struct i2c_adapter;
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +2 -7
index e1ef88ee1ed3..aa4a5498a12f 100644
@@ -1412,13 +1412,8 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev,
  	if (client_id == SOC15_IH_CLIENTID_THM) {
  		switch (src_id) {
  		case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
+			schedule_delayed_work(&smu->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
  		break;
  		case THM_11_0__SRCID__THM_DIG_THERM_H2L:
  			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected\n");
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +2 -7
index 9e8414037638..4071bfa38727 100644
@@ -1353,13 +1353,8 @@ static int smu_v13_0_irq_process(struct amdgpu_device *adev,
  	if (client_id == SOC15_IH_CLIENTID_THM) {
  		switch (src_id) {
  		case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-			dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW CTF) detected!\n");
-			/*
-			 * SW CTF just occurred.
-			 * Try to do a graceful shutdown to prevent further damage.
-			 */
-			dev_emerg(adev->dev, "ERROR: System is going to shutdown due to GPU SW CTF!\n");
-			orderly_poweroff(true);
+			schedule_delayed_work(&smu->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
  			break;
  		case THM_11_0__SRCID__THM_DIG_THERM_H2L:
  			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected\n");
author
#3 lore raw

RE: [PATCH 2/2] drm/amd/pm: avoid unintentional shutdown due to temperature momentary fluctuation

[AMD Official Use Only - General]

Quoted 39 lines

-----Original Message-----
From: Lazar, Lijo <[email protected]>
Sent: Monday, June 26, 2023 7:54 PM
To: Quan, Evan <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: Re: [PATCH 2/2] drm/amd/pm: avoid unintentional shutdown due to temperature momentary fluctuation

Quoted 9 lines · Evan Quan

An intentional delay is added on soft ctf triggered. Then there will be a double check for the GPU temperature before taking further action. This can avoid unintended shutdown due to temperature momentary fluctuation.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h           |  3 ++
  .../gpu/drm/amd/pm/powerplay/amd_powerplay.c  | 48

+++++++++++++++++++

Quoted 3 lines
  .../drm/amd/pm/powerplay/hwmgr/smu_helper.c   | 27 ++++-------
  drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h  |  2 +
  drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c     | 34

+++++++++++++

Quoted 14 lines
  drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h |  2 +
  .../gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c    |  9 +---
  .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c    |  9 +---
  8 files changed, 102 insertions(+), 32 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0 -0
index e459381dc759..5ef1f31e703c 100644
@@ -287,6 +287,9 @@ extern int amdgpu_user_partt_mode;
  #define AMDGPU_SMARTSHIFT_MAX_BIAS (100)
  #define AMDGPU_SMARTSHIFT_MIN_BIAS (-100)
 
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu.h extern int amdgpu_user_partt_mode; · line 290

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+/* Extra time delay(in ms) to eliminate the influence of temperature

momentary fluctuation */

drivers/gpu/drm/amd/amdgpu/amdgpu.h extern int amdgpu_user_partt_mode; · line 291

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+#define AMDGPU_SWCTF_EXTRA_DELAY           50

I think a delay of 10-15ms is good enough to filter out any spike.

50ms is required by our CE team for supporting the customer. It is also aligned with Windows side. Considering we cannot guard that(10-15ms is good), I think it's better to stick to the 50ms recommended setting. How do you think?

Evan

Quoted 342 lines

With that change, the series is

Thanks,
Lijo

Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+
  struct amdgpu_xcp_mgr;
  struct amdgpu_device;
  struct amdgpu_irq_src;
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +1 -0
index 11b7b4cffaae..ff360c699171 100644
@@ -26,6 +26,7 @@
  #include <linux/gfp.h>
  #include <linux/slab.h>
  #include <linux/firmware.h>
+#include <linux/reboot.h>
  #include "amd_shared.h"
  #include "amd_powerplay.h"
  #include "power_state.h"

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 97

Quoted 6 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +3 -0
@@ -91,6 +92,45 @@ static int pp_early_init(void *handle)
    return 0;
  }
 
+static void pp_swctf_delayed_work_handler(struct work_struct *work) {
+   struct pp_hwmgr *hwmgr =
+           container_of(work, struct pp_hwmgr,

swctf_delayed_work.work);

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle)

Quoted 8 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +8 -0
+   struct amdgpu_device *adev = hwmgr->adev;
+   struct amdgpu_dpm_thermal *range =
+                           &adev->pm.dpm.thermal;
+   uint32_t gpu_temperature, size;
+   int ret;
+
+   /*
+    * If the hotspot/edge temperature is confirmed as below SW CTF

setting point

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 107

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +2 -0
+    * after the delay enforced, nothing will be done.
+    * Otherwise, a graceful shutdown will be performed to prevent

further damage.

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 128

Quoted 5 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +5 -0
+    */
+   if (range->sw_ctf_threshold &&
+       hwmgr->hwmgr_func->read_sensor) {
+           ret = hwmgr->hwmgr_func->read_sensor(hwmgr,
+

AMDGPU_PP_SENSOR_HOTSPOT_TEMP,

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle)

Quoted 4 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +4 -0
+                                                &gpu_temperature,
+                                                &size);
+           /*
+            * For some legacy ASICs, hotspot temperature retrieving

might be not

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 128

Quoted 5 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +5 -0
+            * supported. Check the edge temperature instead then.
+            */
+           if (ret == -EOPNOTSUPP)
+                   ret = hwmgr->hwmgr_func->read_sensor(hwmgr,
+

AMDGPU_PP_SENSOR_EDGE_TEMP,

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 122

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +1 -0
+

&gpu_temperature,

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle)

Quoted 7 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +6 -0
+                                                        &size);
+           if (!ret && gpu_temperature / 1000 < range-
 sw_ctf_threshold)
+                   return;
+   }
+
+   dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW

CTF) detected!\n");

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 130

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +1 -0
+   dev_emerg(adev->dev, "ERROR: System is going to shutdown due to

GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device

Quoted 42 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +3 -0
+   orderly_poweroff(true);
+}
+
  static int pp_sw_init(void *handle)
  {
    struct amdgpu_device *adev = handle; @@ -101,6 +141,10 @@ static

int pp_sw_init(void *handle)

pr_debug("powerplay sw init %s\n", ret ? "failed" :
"successfully");

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +8 -0
+   if (!ret)
+           INIT_DELAYED_WORK(&hwmgr->swctf_delayed_work,
+                             pp_swctf_delayed_work_handler);
+
    return ret;
  }
@@ -135,6 +179,8 @@ static int pp_hw_fini(void *handle)
    struct amdgpu_device *adev = handle;
    struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
+   cancel_delayed_work_sync(&hwmgr->swctf_delayed_work);
+
    hwmgr_hw_fini(hwmgr);
 
    return 0;
@@ -221,6 +267,8 @@ static int pp_suspend(void *handle)
    struct amdgpu_device *adev = handle;
    struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
+   cancel_delayed_work_sync(&hwmgr->swctf_delayed_work);
+
    return hwmgr_suspend(hwmgr);
  }
 
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -0
index bfe80ac0ad8c..d0b1ab6c4523 100644
@@ -603,21 +603,17 @@ int phm_irq_process(struct amdgpu_device

*adev,

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 9 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +1 -0
                       struct amdgpu_irq_src *source,
                       struct amdgpu_iv_entry *entry)
  {
+   struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
    uint32_t client_id = entry->client_id;
    uint32_t src_id = entry->src_id;
 
    if (client_id == AMDGPU_IRQ_CLIENTID_LEGACY) {
            if (src_id ==

VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH) {

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 611

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -1
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 616

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 618

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
-                   orderly_poweroff(true);
-           } else if (src_id ==

VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW)

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 614

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +2 -0
+                   schedule_delayed_work(&hwmgr-
 swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 617

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +2 -0
+           } else if (src_id ==
+VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW) {
                    dev_emerg(adev->dev, "ERROR: GPU under

temperature range detected!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +1 -1
-           else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
+           } else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
                    dev_emerg(adev->dev, "ERROR: GPU HW Critical

Temperature Fault(aka CTF) detected!\n");

Quoted 2 lines

/*
* HW CTF just occurred. Shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -0
@@ -626,15 +622,10 @@ int phm_irq_process(struct amdgpu_device

*adev,

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
                    orderly_poweroff(true);
            }
    } else if (client_id == SOC15_IH_CLIENTID_THM) {
-           if (src_id == 0) {
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 635

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 6 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +3 -2
-                   orderly_poweroff(true);
-           } else
+           if (src_id == 0)
+                   schedule_delayed_work(&hwmgr-
 swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 630

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +1 -0
+           else
                    dev_emerg(adev->dev, "ERROR: GPU under

temperature range detected!\n");

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -0
    } else if (client_id == SOC15_IH_CLIENTID_ROM_SMUIO) {
            dev_emerg(adev->dev, "ERROR: GPU HW Critical Temperature

Fault(aka

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct

Quoted 29 lines

CTF) detected!\n"); diff --git
a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
index f1580a26a850..612d66aeaab9 100644

drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h +2 -0
@@ -811,6 +811,8 @@ struct pp_hwmgr {
    bool gfxoff_state_changed_by_workload;
    uint32_t pstate_sclk_peak;
    uint32_t pstate_mclk_peak;
+
+   struct delayed_work swctf_delayed_work;
  };
 
  int hwmgr_early_init(struct pp_hwmgr *hwmgr); diff --git

a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 2aecb8faf407..4062b9b46986 100644

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +1 -0
@@ -24,6 +24,7 @@
 
  #include <linux/firmware.h>
  #include <linux/pci.h>
+#include <linux/reboot.h>
 
  #include "amdgpu.h"
  #include "amdgpu_smu.h"
@@ -1078,6 +1079,34 @@ static void smu_interrupt_work_fn(struct

work_struct *work)

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 7 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +4 -0
            smu->ppt_funcs->interrupt_work(smu);
  }
 
+static void smu_swctf_delayed_work_handler(struct work_struct *work)
+{
+   struct smu_context *smu =
+           container_of(work, struct smu_context,

swctf_delayed_work.work);

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 7 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +7 -0
+   struct smu_temperature_range *range =
+                           &smu->thermal_range;
+   struct amdgpu_device *adev = smu->adev;
+   uint32_t hotspot_tmp, size;
+
+   /*
+    * If the hotspot temperature is confirmed as below SW CTF setting

point

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct · line 1094

Quoted 2 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +2 -0
+    * after the delay enforced, nothing will be done.
+    * Otherwise, a graceful shutdown will be performed to prevent

further damage.

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work) · line 1104

Quoted 5 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +5 -0
+    */
+   if (range->software_shutdown_temp &&
+       smu->ppt_funcs->read_sensor &&
+       !smu->ppt_funcs->read_sensor(smu,
+

AMDGPU_PP_SENSOR_HOTSPOT_TEMP,

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 6 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +6 -0
+                                    &hotspot_tmp,
+                                    &size) &&
+       hotspot_tmp / 1000 < range->software_shutdown_temp)
+           return;
+
+   dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW

CTF) detected!\n");

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct · line 1106

Quoted 1 line
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +1 -0
+   dev_emerg(adev->dev, "ERROR: System is going to shutdown due to

GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 9 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +3 -1
+   orderly_poweroff(true);
+}
+
  static int smu_sw_init(void *handle)
  {
    struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@
-1120,6 +1149,9 @@ static int smu_sw_init(void *handle)
    smu->smu_dpm.dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
    smu->smu_dpm.requested_dpm_level =

AMD_DPM_FORCED_LEVEL_AUTO;

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static int smu_sw_init(void *handle)

Quoted 7 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +3 -0
+   INIT_DELAYED_WORK(&smu->swctf_delayed_work,
+                     smu_swctf_delayed_work_handler);
+
    ret = smu_smc_table_sw_init(smu);
    if (ret) {
            dev_err(adev->dev, "Failed to sw init smc table!\n"); @@ -

1600,6

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct

Quoted 29 lines

+1632,8 @@ static int smu_smc_hw_cleanup(struct smu_context *smu)

            return ret;
    }
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +2 -0
+   cancel_delayed_work_sync(&smu->swctf_delayed_work);
+
    ret = smu_disable_dpms(smu);
    if (ret) {
            dev_err(adev->dev, "Fail to disable dpm features!\n"); diff --git

a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 4ce394903c07..18101ec0ae6e 100644

drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +2 -0
@@ -573,6 +573,8 @@ struct smu_context
    u32 debug_param_reg;
    u32 debug_msg_reg;
    u32 debug_resp_reg;
+
+   struct delayed_work             swctf_delayed_work;
  };
 
  struct i2c_adapter;
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -0
index e1ef88ee1ed3..aa4a5498a12f 100644
@@ -1412,13 +1412,8 @@ static int smu_v11_0_irq_process(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct amdgpu_device *adev,

Quoted 4 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -1
    if (client_id == SOC15_IH_CLIENTID_THM) {
            switch (src_id) {
            case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct · line 1418

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct · line 1420

Quoted 2 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +2 -1
-                   orderly_poweroff(true);
+                   schedule_delayed_work(&smu->swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -0
            break;
            case THM_11_0__SRCID__THM_DIG_THERM_H2L:
                    dev_emerg(adev->dev, "ERROR: GPU under

temperature range

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct

Quoted 7 lines

detected\n"); diff --git
a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index 9e8414037638..4071bfa38727 100644

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -0
@@ -1353,13 +1353,8 @@ static int smu_v13_0_irq_process(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1356

Quoted 4 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -1
    if (client_id == SOC15_IH_CLIENTID_THM) {
            switch (src_id) {
            case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1359

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1361

Quoted 2 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1357

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +2 -1
-                   orderly_poweroff(true);
+                   schedule_delayed_work(&smu->swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1360

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -0
                    break;
            case THM_11_0__SRCID__THM_DIG_THERM_H2L:
                    dev_emerg(adev->dev, "ERROR: GPU under

temperature range

Quoted 1 line

detected\n");

#4 lore raw

Re: [PATCH 2/2] drm/amd/pm: avoid unintentional shutdown due to temperature momentary fluctuation

Quoted 41 lines · Evan

[AMD Official Use Only - General]

Quoted 39 lines

-----Original Message-----
From: Lazar, Lijo <[email protected]>
Sent: Monday, June 26, 2023 7:54 PM
To: Quan, Evan <[email protected]>; [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: Re: [PATCH 2/2] drm/amd/pm: avoid unintentional shutdown due to temperature momentary fluctuation

Quoted 9 lines · Evan Quan

An intentional delay is added on soft ctf triggered. Then there will be a double check for the GPU temperature before taking further action. This can avoid unintended shutdown due to temperature momentary fluctuation.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu.h           |  3 ++
   .../gpu/drm/amd/pm/powerplay/amd_powerplay.c  | 48

+++++++++++++++++++

Quoted 3 lines
   .../drm/amd/pm/powerplay/hwmgr/smu_helper.c   | 27 ++++-------
   drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h  |  2 +
   drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c     | 34

+++++++++++++

Quoted 14 lines
   drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h |  2 +
   .../gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c    |  9 +---
   .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c    |  9 +---
   8 files changed, 102 insertions(+), 32 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0 -0
index e459381dc759..5ef1f31e703c 100644
@@ -287,6 +287,9 @@ extern int amdgpu_user_partt_mode;
   #define AMDGPU_SMARTSHIFT_MAX_BIAS (100)
   #define AMDGPU_SMARTSHIFT_MIN_BIAS (-100)
 
Quoted 9 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu.h extern int amdgpu_user_partt_mode; · line 290

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+/* Extra time delay(in ms) to eliminate the influence of temperature

momentary fluctuation */

drivers/gpu/drm/amd/amdgpu/amdgpu.h extern int amdgpu_user_partt_mode; · line 291

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+#define AMDGPU_SWCTF_EXTRA_DELAY           50

I think a delay of 10-15ms is good enough to filter out any spike.

50ms is required by our CE team for supporting the customer. It is also aligned with Windows side. Considering we cannot guard that(10-15ms is good), I think it's better to stick to the 50ms recommended setting. How do you think?

IMO, a temperature reading consistenly remaining high for 10-15 ms shouldn't be considered a spike since thermal controller (given its clock) would have taken multiple readings by that time for the same sensor.

I'm fine if you want to align with Windows side.

Thanks,
Lijo

Quoted 343 lines

Evan

Quoted 342 lines

With that change, the series is

Thanks,
Lijo

Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
+
   struct amdgpu_xcp_mgr;
   struct amdgpu_device;
   struct amdgpu_irq_src;
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +1 -0
index 11b7b4cffaae..ff360c699171 100644
@@ -26,6 +26,7 @@
   #include <linux/gfp.h>
   #include <linux/slab.h>
   #include <linux/firmware.h>
+#include <linux/reboot.h>
   #include "amd_shared.h"
   #include "amd_powerplay.h"
   #include "power_state.h"

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 97

Quoted 6 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +3 -0
@@ -91,6 +92,45 @@ static int pp_early_init(void *handle)
     return 0;
   }
 
+static void pp_swctf_delayed_work_handler(struct work_struct *work) {
+   struct pp_hwmgr *hwmgr =
+           container_of(work, struct pp_hwmgr,

swctf_delayed_work.work);

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle)

Quoted 8 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +8 -0
+   struct amdgpu_device *adev = hwmgr->adev;
+   struct amdgpu_dpm_thermal *range =
+                           &adev->pm.dpm.thermal;
+   uint32_t gpu_temperature, size;
+   int ret;
+
+   /*
+    * If the hotspot/edge temperature is confirmed as below SW CTF

setting point

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 107

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +2 -0
+    * after the delay enforced, nothing will be done.
+    * Otherwise, a graceful shutdown will be performed to prevent

further damage.

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 128

Quoted 5 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +5 -0
+    */
+   if (range->sw_ctf_threshold &&
+       hwmgr->hwmgr_func->read_sensor) {
+           ret = hwmgr->hwmgr_func->read_sensor(hwmgr,
+

AMDGPU_PP_SENSOR_HOTSPOT_TEMP,

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle)

Quoted 4 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +4 -0
+                                                &gpu_temperature,
+                                                &size);
+           /*
+            * For some legacy ASICs, hotspot temperature retrieving

might be not

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 128

Quoted 5 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +5 -0
+            * supported. Check the edge temperature instead then.
+            */
+           if (ret == -EOPNOTSUPP)
+                   ret = hwmgr->hwmgr_func->read_sensor(hwmgr,
+

AMDGPU_PP_SENSOR_EDGE_TEMP,

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 122

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +1 -0
+

&gpu_temperature,

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle)

Quoted 7 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +6 -0
+                                                        &size);
+           if (!ret && gpu_temperature / 1000 < range-
 sw_ctf_threshold)
+                   return;
+   }
+
+   dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW

CTF) detected!\n");

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c static int pp_early_init(void *handle) · line 130

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +1 -0
+   dev_emerg(adev->dev, "ERROR: System is going to shutdown due to

GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device

Quoted 42 lines
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +3 -0
+   orderly_poweroff(true);
+}
+
   static int pp_sw_init(void *handle)
   {
     struct amdgpu_device *adev = handle; @@ -101,6 +141,10 @@ static

int pp_sw_init(void *handle)

pr_debug("powerplay sw init %s\n", ret ? "failed" :
"successfully");

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +8 -0
+   if (!ret)
+           INIT_DELAYED_WORK(&hwmgr->swctf_delayed_work,
+                             pp_swctf_delayed_work_handler);
+
     return ret;
   }
@@ -135,6 +179,8 @@ static int pp_hw_fini(void *handle)
     struct amdgpu_device *adev = handle;
     struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
+   cancel_delayed_work_sync(&hwmgr->swctf_delayed_work);
+
     hwmgr_hw_fini(hwmgr);
 
     return 0;
@@ -221,6 +267,8 @@ static int pp_suspend(void *handle)
     struct amdgpu_device *adev = handle;
     struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
 
+   cancel_delayed_work_sync(&hwmgr->swctf_delayed_work);
+
     return hwmgr_suspend(hwmgr);
   }
 
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -0
index bfe80ac0ad8c..d0b1ab6c4523 100644
@@ -603,21 +603,17 @@ int phm_irq_process(struct amdgpu_device

*adev,

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 9 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +1 -0
                        struct amdgpu_irq_src *source,
                        struct amdgpu_iv_entry *entry)
   {
+   struct pp_hwmgr *hwmgr = adev->powerplay.pp_handle;
     uint32_t client_id = entry->client_id;
     uint32_t src_id = entry->src_id;
 
     if (client_id == AMDGPU_IRQ_CLIENTID_LEGACY) {
             if (src_id ==

VISLANDS30_IV_SRCID_CG_TSS_THERMAL_LOW_TO_HIGH) {

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 611

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -1
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 616

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 618

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
-                   orderly_poweroff(true);
-           } else if (src_id ==

VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW)

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 614

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +2 -0
+                   schedule_delayed_work(&hwmgr-
 swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 617

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +2 -0
+           } else if (src_id ==
+VISLANDS30_IV_SRCID_CG_TSS_THERMAL_HIGH_TO_LOW) {
                     dev_emerg(adev->dev, "ERROR: GPU under

temperature range detected!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +1 -1
-           else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
+           } else if (src_id == VISLANDS30_IV_SRCID_GPIO_19) {
                     dev_emerg(adev->dev, "ERROR: GPU HW Critical

Temperature Fault(aka CTF) detected!\n");

Quoted 2 lines

/*
* HW CTF just occurred. Shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device

Quoted 1 line
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -0
@@ -626,15 +622,10 @@ int phm_irq_process(struct amdgpu_device

*adev,

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
                     orderly_poweroff(true);
             }
     } else if (client_id == SOC15_IH_CLIENTID_THM) {
-           if (src_id == 0) {
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 635

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device *adev,

Quoted 6 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +3 -2
-                   orderly_poweroff(true);
-           } else
+           if (src_id == 0)
+                   schedule_delayed_work(&hwmgr-
 swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c int phm_irq_process(struct amdgpu_device · line 630

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +1 -0
+           else
                     dev_emerg(adev->dev, "ERROR: GPU under

temperature range detected!\n");

Quoted 2 lines
drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu_helper.c +0 -0
     } else if (client_id == SOC15_IH_CLIENTID_ROM_SMUIO) {
             dev_emerg(adev->dev, "ERROR: GPU HW Critical Temperature

Fault(aka

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct

Quoted 29 lines

CTF) detected!\n"); diff --git
a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
index f1580a26a850..612d66aeaab9 100644

drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h +2 -0
@@ -811,6 +811,8 @@ struct pp_hwmgr {
     bool gfxoff_state_changed_by_workload;
     uint32_t pstate_sclk_peak;
     uint32_t pstate_mclk_peak;
+
+   struct delayed_work swctf_delayed_work;
   };
 
   int hwmgr_early_init(struct pp_hwmgr *hwmgr); diff --git

a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 2aecb8faf407..4062b9b46986 100644

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +1 -0
@@ -24,6 +24,7 @@
 
   #include <linux/firmware.h>
   #include <linux/pci.h>
+#include <linux/reboot.h>
 
   #include "amdgpu.h"
   #include "amdgpu_smu.h"
@@ -1078,6 +1079,34 @@ static void smu_interrupt_work_fn(struct

work_struct *work)

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 7 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +4 -0
             smu->ppt_funcs->interrupt_work(smu);
   }
 
+static void smu_swctf_delayed_work_handler(struct work_struct *work)
+{
+   struct smu_context *smu =
+           container_of(work, struct smu_context,

swctf_delayed_work.work);

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 7 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +7 -0
+   struct smu_temperature_range *range =
+                           &smu->thermal_range;
+   struct amdgpu_device *adev = smu->adev;
+   uint32_t hotspot_tmp, size;
+
+   /*
+    * If the hotspot temperature is confirmed as below SW CTF setting

point

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct · line 1094

Quoted 2 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +2 -0
+    * after the delay enforced, nothing will be done.
+    * Otherwise, a graceful shutdown will be performed to prevent

further damage.

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work) · line 1104

Quoted 5 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +5 -0
+    */
+   if (range->software_shutdown_temp &&
+       smu->ppt_funcs->read_sensor &&
+       !smu->ppt_funcs->read_sensor(smu,
+

AMDGPU_PP_SENSOR_HOTSPOT_TEMP,

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 6 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +6 -0
+                                    &hotspot_tmp,
+                                    &size) &&
+       hotspot_tmp / 1000 < range->software_shutdown_temp)
+           return;
+
+   dev_emerg(adev->dev, "ERROR: GPU over temperature range(SW

CTF) detected!\n");

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct · line 1106

Quoted 1 line
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +1 -0
+   dev_emerg(adev->dev, "ERROR: System is going to shutdown due to

GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static void smu_interrupt_work_fn(struct work_struct *work)

Quoted 9 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +3 -1
+   orderly_poweroff(true);
+}
+
   static int smu_sw_init(void *handle)
   {
     struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@
-1120,6 +1149,9 @@ static int smu_sw_init(void *handle)
     smu->smu_dpm.dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
     smu->smu_dpm.requested_dpm_level =

AMD_DPM_FORCED_LEVEL_AUTO;

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c static int smu_sw_init(void *handle)

Quoted 7 lines
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +3 -0
+   INIT_DELAYED_WORK(&smu->swctf_delayed_work,
+                     smu_swctf_delayed_work_handler);
+
     ret = smu_smc_table_sw_init(smu);
     if (ret) {
             dev_err(adev->dev, "Failed to sw init smc table!\n"); @@ -

1600,6

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct

Quoted 29 lines

+1632,8 @@ static int smu_smc_hw_cleanup(struct smu_context *smu)

             return ret;
     }
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +2 -0
+   cancel_delayed_work_sync(&smu->swctf_delayed_work);
+
     ret = smu_disable_dpms(smu);
     if (ret) {
             dev_err(adev->dev, "Fail to disable dpm features!\n"); diff --git

a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 4ce394903c07..18101ec0ae6e 100644

drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +2 -0
@@ -573,6 +573,8 @@ struct smu_context
     u32 debug_param_reg;
     u32 debug_msg_reg;
     u32 debug_resp_reg;
+
+   struct delayed_work             swctf_delayed_work;
   };
 
   struct i2c_adapter;
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -0
index e1ef88ee1ed3..aa4a5498a12f 100644
@@ -1412,13 +1412,8 @@ static int smu_v11_0_irq_process(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct amdgpu_device *adev,

Quoted 4 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -1
     if (client_id == SOC15_IH_CLIENTID_THM) {
             switch (src_id) {
             case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct · line 1418

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct · line 1420

Quoted 2 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +2 -1
-                   orderly_poweroff(true);
+                   schedule_delayed_work(&smu->swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c static int smu_v11_0_irq_process(struct amdgpu_device *adev,

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +0 -0
             break;
             case THM_11_0__SRCID__THM_DIG_THERM_H2L:
                     dev_emerg(adev->dev, "ERROR: GPU under

temperature range

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct

Quoted 7 lines

detected\n"); diff --git
a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index 9e8414037638..4071bfa38727 100644

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -0
@@ -1353,13 +1353,8 @@ static int smu_v13_0_irq_process(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1356

Quoted 4 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -1
     if (client_id == SOC15_IH_CLIENTID_THM) {
             switch (src_id) {
             case THM_11_0__SRCID__THM_DIG_THERM_L2H:
-                   dev_emerg(adev->dev, "ERROR: GPU over

temperature range(SW CTF) detected!\n");

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1359

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -3
-                   /*
-                    * SW CTF just occurred.
-                    * Try to do a graceful shutdown to prevent further

damage.

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1361

Quoted 2 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -2
-                    */
-                   dev_emerg(adev->dev, "ERROR: System is going to

shutdown due to GPU SW CTF!\n");

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1357

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +2 -1
-                   orderly_poweroff(true);
+                   schedule_delayed_work(&smu->swctf_delayed_work,
+

msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));

drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c static int smu_v13_0_irq_process(struct · line 1360

Quoted 3 lines
drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +0 -0
                     break;
             case THM_11_0__SRCID__THM_DIG_THERM_H2L:
                     dev_emerg(adev->dev, "ERROR: GPU under

temperature range

Quoted 1 line

detected\n");