drm/amdgpu: fix possible pstate switch race condition
Subsystems (2)
PATCH 1/2 drm/amdgpu: fix possible pstate switch race condition
Added lock protection so that the p-state switch will be guarded to be sequential. Also update the hive pstate only all device from the hive are in the same state.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 34 ++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +3 -0
@@ -1041,6 +1041,9 @@ struct amdgpu_device {uint64_t unique_id;uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS ]; + +/* device pstate */ +int pstate; };static inline struct amdgpu_device *amdgpu_ttm_adev (struct ttm_bo_device *bdev)
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +32 -2
@@ -274,12 +274,18 @@ int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) {int ret =0 ;struct amdgpu_hive_info *hive =amdgpu_get_xgmi_hive (adev,0 ); +struct amdgpu_device *tmp_adev; +bool update_hive_pstate = true;if (!hive)return 0 ; -if (hive->pstate == pstate) +mutex_lock (&hive->hive_lock); + +if (hive->pstate == pstate) { +mutex_unlock (&hive->hive_lock);return 0 ; + }dev_dbg (adev->dev,"Set xgmi pstate %d.\n" , pstate);
@@ -290,11 +296,32 @@ int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) ret = adev->powerplay.pp_funcs->set_xgmi_pstate (adev->powerplay.pp_handle, pstate); -if (ret) +if (ret) {dev_err (adev->dev,"XGMI: Set pstate failure on device %llx, hive %llx, ret %d" , adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id, ret); + goto out; + } + +/* Update device pstate */ + adev->pstate = pstate; + +/* +* Update the hive pstate only all devices of the hive +* are in the same pstate +*/ +list_for_each_entry (tmp_adev, &hive->device_list, gmc.xgmi.head) { +if (tmp_adev->pstate != adev->pstate) { + update_hive_pstate = false; +break ; + } + } +if (update_hive_pstate) + hive->pstate = pstate; + +out: +mutex_unlock (&hive->hive_lock);return ret; }
@@ -369,6 +396,9 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) goto exit; } +/* Set default device pstate */ + adev->pstate =-1 ; + top_info = &adev->psp.xgmi_context.top_info;list_add_tail (&adev->gmc.xgmi.head, &hive->device_list);
Signature · 6 lines
2.23.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 2/2 drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +35 -12
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r;for (i =0 ; i < adev->num_ip_blocks; i++) {
@@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)if (r)DRM_ERROR ("enable mgpu fan boost failed (%d).\n" , r); + +if (adev->gmc.xgmi.num_physical_nodes >1 ) { +mutex_lock (&mgpu_info.mutex); + +/* +* Reset device p-state to low as this was booted with high. +* +* This should be performed only after all devices from the same +* hive get initialized. +* +* However, it's unknown how many device in the hive in advance. +* As this is counted one by one during devices initializations. +* +* So, we wait for all XGMI interlinked devices initialized. +* This may bring some delays as those devices may come from +* different hives. But that should be OK. +*/ +if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { +for (i =0 ; i < mgpu_info.num_gpu; i++) { + gpu_instance = &(mgpu_info.gpu_ins[i]); +if (gpu_instance->adev->flags &AMD_IS_APU ) +continue ; + + r =amdgpu_xgmi_set_pstate (gpu_instance->adev,0 ); +if (r) { +DRM_ERROR ("pstate setting failed (%d).\n" , r); +break ; + } + } + } + +mutex_unlock (&mgpu_info.mutex); + } +return 0 ; }
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) r =amdgpu_ib_ring_tests (adev);if (r)DRM_ERROR ("ib ring test failed (%d).\n" , r); - -/* -* set to low pstate by default -* This should be performed after all devices from -* XGMI finish their initializations. Thus it's moved -* to here. -* The time delay is 2S. TODO: confirm whether that -* is enough for all possible XGMI setups. -*/ - r =amdgpu_xgmi_set_pstate (adev,0 ); -if (r) -DRM_ERROR ("pstate setting failed (%d).\n" , r); }static void amdgpu_device_delay_enable_gfx_off (struct work_struct *work)
Signature · 6 lines
2.23.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]
Cc: Kim, Jonathan <[email protected]>; Strawbridge, Michael <[email protected]>; Quan, Evan <[email protected]>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high.
+ *
+ * This should be performed only after all devices from the same
+ * hive get initialized.
+ *
+ * However, it's unknown how many device in the hive in advance.
+ * As this is counted one by one during devices initializations.
+ *
+ * So, we wait for all XGMI interlinked devices initialized.
+ * This may bring some delays as those devices may come from
+ * different hives. But that should be OK.
+ */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */
- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CH2PR12MB3831CEABDD4CBA2FEB8DEFEB857E0-rV3HgkLYx2Wb1l0FfFgDpQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
Re: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Hi Jon,
Does that mean we can simply use this instead?
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) {
Thanks,
Michael
________________________________
From: Kim, Jonathan <[email protected]>
Sent: 05 November 2019 11:32 AM
To: Quan, Evan <[email protected]>; [email protected] <[email protected]>
Cc: Strawbridge, Michael <[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]
Cc: Kim, Jonathan <[email protected]>; Strawbridge, Michael <[email protected]>; Quan, Evan <[email protected]>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high. + * + * This should be performed only after all devices from the same + * hive get initialized. + * + * However, it's unknown how many device in the hive in advance. + * As this is counted one by one during devices initializations. + * + * So, we wait for all XGMI interlinked devices initialized. + * This may bring some delays as those devices may come from + * different hives. But that should be OK. + */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): DM5PR12MB248696FDB2D0E563221E7CDFE87E0-2J9CzHegvk8hnZ7s9K7NnwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Yes that's correct. This should fix the issue.
Jon
From: Strawbridge, Michael <[email protected]>
Sent: Tuesday, November 5, 2019 11:40 AM
To: Kim, Jonathan <[email protected]>; Quan, Evan <[email protected]>; [email protected]
Subject: Re: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Hi Jon,
Does that mean we can simply use this instead?
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) {
Thanks,
Michael
________________________________
From: Kim, Jonathan <[email protected]<mailto:[email protected]>>
Sent: 05 November 2019 11:32 AM
To: Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>
Cc: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]<mailto:[email protected]>
Cc: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high. + * + * This should be performed only after all devices from the same + * hive get initialized. + * + * However, it's unknown how many device in the hive in advance. + * As this is counted one by one during devices initializations. + * + * So, we wait for all XGMI interlinked devices initialized. + * This may bring some delays as those devices may come from + * different hives. But that should be OK. + */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CH2PR12MB3831D16D352FA009002800CC857E0-rV3HgkLYx2Wb1l0FfFgDpQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
Also while num_physical_nodes should be used here instead. It doesn't make sense to have a pre-condition check i.e. (if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) ) against the total number of nodes while the loop checks for the current num gpus from mgpu_info. This will fail to set the last node to low pstate for the same 0-indexing issue once the pre-condition passes.
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 11:42 AM
To: Strawbridge, Michael <[email protected]>; Quan, Evan <[email protected]>; [email protected]
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Yes that's correct. This should fix the issue.
Jon
From: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 11:40 AM
To: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: Re: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Hi Jon,
Does that mean we can simply use this instead?
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) {
Thanks,
Michael
________________________________
From: Kim, Jonathan <[email protected]<mailto:[email protected]>>
Sent: 05 November 2019 11:32 AM
To: Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>
Cc: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]<mailto:[email protected]>
Cc: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high. + * + * This should be performed only after all devices from the same + * hive get initialized. + * + * However, it's unknown how many device in the hive in advance. + * As this is counted one by one during devices initializations. + * + * So, we wait for all XGMI interlinked devices initialized. + * This may bring some delays as those devices may come from + * different hives. But that should be OK. + */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CH2PR12MB3831868F2A7B4A8BCCBA24F7857E0-rV3HgkLYx2Wb1l0FfFgDpQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Taking a look at this again, it's not an indexing issue, it's a placement problem. I don't think this solution will work if we expect to pstate switch all gpus.
Quoted 1 line
From
amdgpu_kms.c,amdgpu_register_gpu_instancecomes afteramdgpu_device_init. This means we'll never reach mgpu_info.num_gpu ==adev->gmc.xgmi.num_physical_nodesuntil in this code path.
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 12:07 PM
To: Strawbridge, Michael <[email protected]>; Quan, Evan <[email protected]>; [email protected]
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
Also while num_physical_nodes should be used here instead. It doesn't make sense to have a pre-condition check i.e. (if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) ) against the total number of nodes while the loop checks for the current num gpus from mgpu_info. This will fail to set the last node to low pstate for the same 0-indexing issue once the pre-condition passes.
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 11:42 AM
To: Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Yes that's correct. This should fix the issue.
Jon
From: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 11:40 AM
To: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: Re: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Hi Jon,
Does that mean we can simply use this instead?
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) {
Thanks,
Michael
________________________________
From: Kim, Jonathan <[email protected]<mailto:[email protected]>>
Sent: 05 November 2019 11:32 AM
To: Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>
Cc: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]<mailto:[email protected]>
Cc: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high. + * + * This should be performed only after all devices from the same + * hive get initialized. + * + * However, it's unknown how many device in the hive in advance. + * As this is counted one by one during devices initializations. + * + * So, we wait for all XGMI interlinked devices initialized. + * This may bring some delays as those devices may come from + * different hives. But that should be OK. + */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CH2PR12MB383150DB7FA13B7E5472A37085790-rV3HgkLYx2Wb1l0FfFgDpQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Thanks Jon.
Per discussed in another mail thread, this should be applied after the patch below
https://lists.freedesktop.org/archives/amd-gfx/2019-November/042160.html
Sorry for missing this important information.
Regards,
Evan
From: Kim, Jonathan <[email protected]>
Sent: Wednesday, November 6, 2019 10:18 AM
To: Strawbridge, Michael <[email protected]>; Quan, Evan <[email protected]>; [email protected]
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Taking a look at this again, it's not an indexing issue, it's a placement problem. I don't think this solution will work if we expect to pstate switch all gpus.
Quoted 1 line
From
amdgpu_kms.c,amdgpu_register_gpu_instancecomes afteramdgpu_device_init. This means we'll never reach mgpu_info.num_gpu ==adev->gmc.xgmi.num_physical_nodesuntil in this code path.
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 12:07 PM
To: Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
Also while num_physical_nodes should be used here instead. It doesn't make sense to have a pre-condition check i.e. (if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) ) against the total number of nodes while the loop checks for the current num gpus from mgpu_info. This will fail to set the last node to low pstate for the same 0-indexing issue once the pre-condition passes.
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 11:42 AM
To: Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Yes that's correct. This should fix the issue.
Jon
From: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 11:40 AM
To: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: Re: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Hi Jon,
Does that mean we can simply use this instead?
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) {
Thanks,
Michael
________________________________
From: Kim, Jonathan <[email protected]<mailto:[email protected]>>
Sent: 05 November 2019 11:32 AM
To: Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>
Cc: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]<mailto:[email protected]>
Cc: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high. + * + * This should be performed only after all devices from the same + * hive get initialized. + * + * However, it's unknown how many device in the hive in advance. + * As this is counted one by one during devices initializations. + * + * So, we wait for all XGMI interlinked devices initialized. + * This may bring some delays as those devices may come from + * different hives. But that should be OK. + */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): MN2PR12MB3344BF9A70FE63F2D5E4AE63E4790-rweVpJHSKToDMgCC8P//[email protected]
Kim, Jonathan
Reviewed-byRE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
No problem. With the incoming patch referenced below, this patch should be ok since it doesn't break the build.
From: Quan, Evan <[email protected]>
Sent: Tuesday, November 5, 2019 10:11 PM
To: Kim, Jonathan <[email protected]>; Strawbridge, Michael <[email protected]>; [email protected]
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Thanks Jon.
Per discussed in another mail thread, this should be applied after the patch below
https://lists.freedesktop.org/archives/amd-gfx/2019-November/042160.html
Sorry for missing this important information.
Regards,
Evan
From: Kim, Jonathan <[email protected]<mailto:[email protected]>>
Sent: Wednesday, November 6, 2019 10:18 AM
To: Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Taking a look at this again, it's not an indexing issue, it's a placement problem. I don't think this solution will work if we expect to pstate switch all gpus.
Quoted 1 line
From
amdgpu_kms.c,amdgpu_register_gpu_instancecomes afteramdgpu_device_init. This means we'll never reach mgpu_info.num_gpu ==adev->gmc.xgmi.num_physical_nodesuntil in this code path.
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 12:07 PM
To: Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
Also while num_physical_nodes should be used here instead. It doesn't make sense to have a pre-condition check i.e. (if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) ) against the total number of nodes while the loop checks for the current num gpus from mgpu_info. This will fail to set the last node to low pstate for the same 0-indexing issue once the pre-condition passes.
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
From: Kim, Jonathan
Sent: Tuesday, November 5, 2019 11:42 AM
To: Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Yes that's correct. This should fix the issue.
Jon
From: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 11:40 AM
To: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]>
Subject: Re: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Hi Jon,
Does that mean we can simply use this instead?
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes - 1) {
Thanks,
Michael
________________________________
From: Kim, Jonathan <[email protected]<mailto:[email protected]>>
Sent: 05 November 2019 11:32 AM
To: Quan, Evan <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>
Cc: Strawbridge, Michael <[email protected]<mailto:[email protected]>>
Subject: RE: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
Please see inline.
Jon
-----Original Message-----
From: Quan, Evan <[email protected]<mailto:[email protected]>>
Sent: Tuesday, November 5, 2019 5:24 AM
To: [email protected]<mailto:[email protected]>
Cc: Kim, Jonathan <[email protected]<mailto:[email protected]>>; Strawbridge, Michael <[email protected]<mailto:[email protected]>>; Quan, Evan <[email protected]<mailto:[email protected]>>
Subject: [PATCH 2/2] drm/amdgpu: perform p-state switch after the whole hive initialized
P-state switch should be performed after all devices from the hive get initialized.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 47 ++++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -0
@@ -2090,6 +2090,7 @@ static int amdgpu_device_enable_mgpu_fan_boost(void)*/ static int amdgpu_device_ip_late_init (struct amdgpu_device *adev) { +struct amdgpu_gpu_instance *gpu_instance;int i =0 , r; for (i = 0; i < adev->num_ip_blocks; i++) { @@ -2115,6 +2116,40 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) if (r)
DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);+
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * Reset device p-state to low as this was booted with high. + * + * This should be performed only after all devices from the same + * hive get initialized. + * + * However, it's unknown how many device in the hive in advance. + * As this is counted one by one during devices initializations. + * + * So, we wait for all XGMI interlinked devices initialized. + * This may bring some delays as those devices may come from + * different hives. But that should be OK. + */
+ if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
[JK] This condition will never succeed. mgpu_info.num_dgpu is 0-indexed while num_physical_nodes give total nodes.
+ for (i = 0; i < mgpu_info.num_gpu; i++) {
+ gpu_instance = &(mgpu_info.gpu_ins[i]);
+ if (gpu_instance->adev->flags & AMD_IS_APU)
+ continue;
+
+ r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 0);
+ if (r) {
+ DRM_ERROR("pstate setting failed (%d).\n", r);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&mgpu_info.mutex);
+ }
+
return 0;
}
@@ -2227,18 +2262,6 @@ static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- /*
- * set to low pstate by default
- * This should be performed after all devices from
- * XGMI finish their initializations. Thus it's moved
- * to here.
- * The time delay is 2S. TODO: confirm whether that
- * is enough for all possible XGMI setups.
- */- r = amdgpu_xgmi_set_pstate(adev, 0);
- if (r)
- DRM_ERROR("pstate setting failed (%d).\n", r);
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Series is Reviewed-by: Feifei Xu <[email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Quan, Evan
Sent: 2019年11月5日 18:24
To: [email protected]
Cc: Strawbridge, Michael <[email protected]>; Kim, Jonathan <[email protected]>; Quan, Evan <[email protected]>
Subject: [PATCH 1/2] drm/amdgpu: fix possible pstate switch race condition
Added lock protection so that the p-state switch will be guarded to be sequential. Also update the hive pstate only all device from the hive are in the same state.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 34 ++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +3 -0
@@ -1041,6 +1041,9 @@ struct amdgpu_device {uint64_t unique_id;uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS ]; + +/* device pstate */ +int pstate; }; static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 167d9fbd2c4f..de20a9a1c444 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +32 -2
@@ -274,12 +274,18 @@ int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) {int ret =0 ;struct amdgpu_hive_info *hive =amdgpu_get_xgmi_hive (adev,0 ); +struct amdgpu_device *tmp_adev; +bool update_hive_pstate = true;if (!hive)return 0 ; -if (hive->pstate == pstate) +mutex_lock (&hive->hive_lock); + +if (hive->pstate == pstate) { +mutex_unlock (&hive->hive_lock);return 0 ; + }dev_dbg (adev->dev,"Set xgmi pstate %d.\n" , pstate);
@@ -290,11 +296,32 @@ int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) ret = adev->powerplay.pp_funcs->set_xgmi_pstate (adev->powerplay.pp_handle, pstate); -if (ret) +if (ret) {dev_err (adev->dev,"XGMI: Set pstate failure on device %llx, hive %llx, ret %d" , adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id, ret); + goto out; + } + +/* Update device pstate */ + adev->pstate = pstate; + +/* +* Update the hive pstate only all devices of the hive +* are in the same pstate +*/ +list_for_each_entry (tmp_adev, &hive->device_list, gmc.xgmi.head) { +if (tmp_adev->pstate != adev->pstate) { + update_hive_pstate = false; +break ; + } + } +if (update_hive_pstate) + hive->pstate = pstate; + +out: +mutex_unlock (&hive->hive_lock);return ret; }
@@ -369,6 +396,9 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev) goto exit; } +/* Set default device pstate */ + adev->pstate =-1 ; + top_info = &adev->psp.xgmi_context.top_info;list_add_tail (&adev->gmc.xgmi.head, &hive->device_list);
--
2.23.0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx