add missing mutex lock to amdgpu_get_xgmi_hive() (v2)
Subsystems (2)
PATCH 1/1 add missing mutex lock to amdgpu_get_xgmi_hive() (v2)
v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +23 -13
@@ -40,26 +40,39 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)return &hive->device_list; } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev,int lock) {int i;struct amdgpu_hive_info *tmp;if (!adev->gmc.xgmi.hive_id)return NULL ; + +mutex_lock (&xgmi_mutex); +for (i =0 ; i < hive_count; ++i) { tmp = &xgmi_hives[i]; -if (tmp->hive_id == adev->gmc.xgmi.hive_id) +if (tmp->hive_id == adev->gmc.xgmi.hive_id) { +if (lock) +mutex_lock (&tmp->hive_lock); +mutex_unlock (&xgmi_mutex);return tmp; + } } -if (i >=AMDGPU_MAX_XGMI_HIVE ) +if (i >=AMDGPU_MAX_XGMI_HIVE ) { +mutex_unlock (&xgmi_mutex);return NULL ; + }/* initialize new hive if not exist */ tmp = &xgmi_hives[hive_count++]; tmp->hive_id = adev->gmc.xgmi.hive_id;INIT_LIST_HEAD (&tmp->device_list);mutex_init (&tmp->hive_lock); +if (lock) +mutex_lock (&tmp->hive_lock); + +mutex_unlock (&xgmi_mutex);return tmp; }
@@ -111,8 +124,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)return ret; } -mutex_lock (&xgmi_mutex); - hive =amdgpu_get_xgmi_hive (adev); +/* find hive and take lock */ + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) goto exit;
@@ -142,8 +155,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)break ; } +mutex_unlock (&hive->hive_lock); exit: -mutex_unlock (&xgmi_mutex);return ret; }
@@ -154,15 +167,12 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)if (!adev->gmc.xgmi.supported)return ; -mutex_lock (&xgmi_mutex); - - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) - goto exit; +return ;if (!(hive->number_devices--))mutex_destroy (&hive->hive_lock); - -exit: -mutex_unlock (&xgmi_mutex); +else +mutex_unlock (&hive->hive_lock); }
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +1 -1
@@ -32,7 +32,7 @@ struct amdgpu_hive_info {struct mutex hive_lock; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev,int lock);int amdgpu_xgmi_update_topology (struct amdgpu_hive_info *hive,struct amdgpu_device *adev);int amdgpu_xgmi_add_device (struct amdgpu_device *adev);void amdgpu_xgmi_remove_device (struct amdgpu_device *adev);
Signature · 6 lines
2.17.2 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock to amdgpu_get_xgmi_hive() (v2)
v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +17 -3
@@ -40,26 +40,39 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)return &hive->device_list; } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock) {int i;struct amdgpu_hive_info *tmp;if (!adev->gmc.xgmi.hive_id)return NULL ; + +mutex_lock (&xgmi_mutex); +for (i =0 ; i < hive_count; ++i) { tmp = &xgmi_hives[i]; -if (tmp->hive_id == adev->gmc.xgmi.hive_id) +if (tmp->hive_id == adev->gmc.xgmi.hive_id) { +if (lock) +mutex_lock (&tmp->hive_lock); +mutex_unlock (&xgmi_mutex);return tmp; + } } -if (i >=AMDGPU_MAX_XGMI_HIVE ) +if (i >=AMDGPU_MAX_XGMI_HIVE ) { +mutex_unlock (&xgmi_mutex);return NULL ; + }/* initialize new hive if not exist */ tmp = &xgmi_hives[hive_count++]; tmp->hive_id = adev->gmc.xgmi.hive_id;INIT_LIST_HEAD (&tmp->device_list);mutex_init (&tmp->hive_lock); +if (lock) +mutex_lock (&tmp->hive_lock); + +mutex_unlock (&xgmi_mutex);return tmp;
}
@@ -111,8 +124,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
return ret;
}
- mutex_lock(&xgmi_mutex);
- hive = amdgpu_get_xgmi_hive(adev);
+ /* find hive and take lock */
+ hive = amdgpu_get_xgmi_hive(adev, 1);
if (!hive)
goto exit;
@@ -142,8 +155,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
break;
}
+ mutex_unlock(&hive->hive_lock);
exit:
- mutex_unlock(&xgmi_mutex);
return ret;
}
@@ -154,15 +167,12 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
if (!adev->gmc.xgmi.supported)
return;
- mutex_lock(&xgmi_mutex);
-
- hive = amdgpu_get_xgmi_hive(adev);
+ hive = amdgpu_get_xgmi_hive(adev, 1);
if (!hive)
- goto exit;
+ return;
if (!(hive->number_devices--))
mutex_destroy(&hive->hive_lock);
-
-exit:
- mutex_unlock(&xgmi_mutex);
+ else
+ mutex_unlock(&hive->hive_lock);
}
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2 -2
@@ -32,7 +32,7 @@ struct amdgpu_hive_info {struct mutex hive_lock; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock);int amdgpu_xgmi_update_topology (struct amdgpu_hive_info *hive,struct amdgpu_device *adev);int amdgpu_xgmi_add_device (struct amdgpu_device *adev);void amdgpu_xgmi_remove_device (struct amdgpu_device *adev); --
2.17.2
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): DM6PR12MB324181E29144A5FBA327AADBF4890-lmeGfMZKVrEShx2hYn8pVQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;
Let's say i have device 0 in hive A and it just got a gpu reset and at the same time device 1 is being added to same have though amdgpu_xgmi_add_device, hive->hive_lock is acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take the hive->hive_lock inside amdgpu_get_xgmi_hive but release is still outside of the function.
Andrey
Quoted 104 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +24
-13@@ -40,26 +40,39 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)return &hive->device_list; } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock) {int i;struct amdgpu_hive_info *tmp;if (!adev->gmc.xgmi.hive_id)return NULL ; + +mutex_lock (&xgmi_mutex); +for (i =0 ; i < hive_count; ++i) { tmp = &xgmi_hives[i]; -if (tmp->hive_id == adev->gmc.xgmi.hive_id) +if (tmp->hive_id == adev->gmc.xgmi.hive_id) { +if (lock) +mutex_lock (&tmp->hive_lock); +mutex_unlock (&xgmi_mutex);return tmp; + } } -if (i >=AMDGPU_MAX_XGMI_HIVE ) +if (i >=AMDGPU_MAX_XGMI_HIVE ) { +mutex_unlock (&xgmi_mutex);return NULL ; + }/* initialize new hive if not exist */ tmp = &xgmi_hives[hive_count++]; tmp->hive_id = adev->gmc.xgmi.hive_id;INIT_LIST_HEAD (&tmp->device_list);mutex_init (&tmp->hive_lock); +if (lock) +mutex_lock (&tmp->hive_lock); + +mutex_unlock (&xgmi_mutex);return tmp; }@@ -111,8 +124,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)return ret; } -mutex_lock (&xgmi_mutex); - hive =amdgpu_get_xgmi_hive (adev); +/* find hive and take lock */ + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) goto exit;@@ -142,8 +155,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)break ; } +mutex_unlock (&hive->hive_lock); exit: -mutex_unlock (&xgmi_mutex);return ret; }@@ -154,15 +167,12 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)if (!adev->gmc.xgmi.supported)return ; -mutex_lock (&xgmi_mutex); - - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) - goto exit; +return ;if (!(hive->number_devices--))mutex_destroy (&hive->hive_lock); - -exit: -mutex_unlock (&xgmi_mutex); +else +mutex_unlock (&hive->hive_lock); }drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2
-2@@ -32,7 +32,7 @@ struct amdgpu_hive_info {struct mutex hive_lock; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock);int amdgpu_xgmi_update_topology (struct amdgpu_hive_info *hive,struct amdgpu_device *adev);int amdgpu_xgmi_add_device (struct amdgpu_device *adev);void amdgpu_xgmi_remove_device (struct amdgpu_device *adev); --2.17.2
Quoted 8 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 967dc160-e50b-0f35-b11e-6d1c9951c6aa-5C7GfCeVMHo@public.gmane.org
Quoted 37 lines · Andrey
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;Let's say i have device 0 in hive A and it just got a gpu reset and at
Quoted 8 lines
the same time device 1 is being added to same have though
amdgpu_xgmi_add_device,hive->hive_lockis acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take thehive->hive_lockinsideamdgpu_get_xgmi_hivebut release is still outside of the function.
Is adding a device while resetting a device even a valid operation anyways?
I think this means more so that the reset logic is broken. Instead there should be a per-hive reset lock that is taken and that is tested instead.
Tom
Quoted 107 lines
Andrey
Quoted 104 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +24
-13@@ -40,26 +40,39 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)return &hive->device_list; } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock) {int i;struct amdgpu_hive_info *tmp;if (!adev->gmc.xgmi.hive_id)return NULL ; + +mutex_lock (&xgmi_mutex); +for (i =0 ; i < hive_count; ++i) { tmp = &xgmi_hives[i]; -if (tmp->hive_id == adev->gmc.xgmi.hive_id) +if (tmp->hive_id == adev->gmc.xgmi.hive_id) { +if (lock) +mutex_lock (&tmp->hive_lock); +mutex_unlock (&xgmi_mutex);return tmp; + } } -if (i >=AMDGPU_MAX_XGMI_HIVE ) +if (i >=AMDGPU_MAX_XGMI_HIVE ) { +mutex_unlock (&xgmi_mutex);return NULL ; + }/* initialize new hive if not exist */ tmp = &xgmi_hives[hive_count++]; tmp->hive_id = adev->gmc.xgmi.hive_id;INIT_LIST_HEAD (&tmp->device_list);mutex_init (&tmp->hive_lock); +if (lock) +mutex_lock (&tmp->hive_lock); + +mutex_unlock (&xgmi_mutex);return tmp; }@@ -111,8 +124,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)return ret; } -mutex_lock (&xgmi_mutex); - hive =amdgpu_get_xgmi_hive (adev); +/* find hive and take lock */ + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) goto exit;@@ -142,8 +155,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)break ; } +mutex_unlock (&hive->hive_lock); exit: -mutex_unlock (&xgmi_mutex);return ret; }@@ -154,15 +167,12 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)if (!adev->gmc.xgmi.supported)return ; -mutex_lock (&xgmi_mutex); - - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) - goto exit; +return ;if (!(hive->number_devices--))mutex_destroy (&hive->hive_lock); - -exit: -mutex_unlock (&xgmi_mutex); +else +mutex_unlock (&hive->hive_lock); }drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2
-2@@ -32,7 +32,7 @@ struct amdgpu_hive_info {struct mutex hive_lock; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock);int amdgpu_xgmi_update_topology (struct amdgpu_hive_info *hive,struct amdgpu_device *adev);int amdgpu_xgmi_add_device (struct amdgpu_device *adev);void amdgpu_xgmi_remove_device (struct amdgpu_device *adev); --2.17.2
Quoted 9 lines
Quoted 8 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): e7aa0543-b54f-665a-b4b4-d6db58201923-5C7GfCeVMHo@public.gmane.org
Quoted 44 lines · Tom
Quoted 35 lines · Andrey
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;Let's say i have device 0 in hive A and it just got a gpu reset and at
Quoted 8 lines
the same time device 1 is being added to same have though
amdgpu_xgmi_add_device,hive->hive_lockis acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take thehive->hive_lockinsideamdgpu_get_xgmi_hivebut release is still outside of the function.
Quoted 2 lines
Is adding a device while resetting a device even a valid operation anyways?
In theory it's valid if you have hot pluggable devices
Quoted 6 lines
I think this means more so that the reset logic is broken. Instead there should be a per-hive reset lock that is taken and that is tested instead.
Tom
The hive->hive_lock was added exactly for this purpose and used only for that purpose. Maybe the naming i gave it wasn't reflective of it's purpose :)
Andrey
Quoted 108 lines
Quoted 108 lines
Andrey
Quoted 104 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +24
-13@@ -40,26 +40,39 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)return &hive->device_list; } -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev) +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock) {int i;struct amdgpu_hive_info *tmp;if (!adev->gmc.xgmi.hive_id)return NULL ; + +mutex_lock (&xgmi_mutex); +for (i =0 ; i < hive_count; ++i) { tmp = &xgmi_hives[i]; -if (tmp->hive_id == adev->gmc.xgmi.hive_id) +if (tmp->hive_id == adev->gmc.xgmi.hive_id) { +if (lock) +mutex_lock (&tmp->hive_lock); +mutex_unlock (&xgmi_mutex);return tmp; + } } -if (i >=AMDGPU_MAX_XGMI_HIVE ) +if (i >=AMDGPU_MAX_XGMI_HIVE ) { +mutex_unlock (&xgmi_mutex);return NULL ; + }/* initialize new hive if not exist */ tmp = &xgmi_hives[hive_count++]; tmp->hive_id = adev->gmc.xgmi.hive_id;INIT_LIST_HEAD (&tmp->device_list);mutex_init (&tmp->hive_lock); +if (lock) +mutex_lock (&tmp->hive_lock); + +mutex_unlock (&xgmi_mutex);return tmp; }@@ -111,8 +124,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)return ret; } -mutex_lock (&xgmi_mutex); - hive =amdgpu_get_xgmi_hive (adev); +/* find hive and take lock */ + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) goto exit;@@ -142,8 +155,8 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)break ; } +mutex_unlock (&hive->hive_lock); exit: -mutex_unlock (&xgmi_mutex);return ret; }@@ -154,15 +167,12 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)if (!adev->gmc.xgmi.supported)return ; -mutex_lock (&xgmi_mutex); - - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,1 );if (!hive) - goto exit; +return ;if (!(hive->number_devices--))mutex_destroy (&hive->hive_lock); - -exit: -mutex_unlock (&xgmi_mutex); +else +mutex_unlock (&hive->hive_lock); }drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2
-2@@ -32,7 +32,7 @@ struct amdgpu_hive_info {struct mutex hive_lock; }; -struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device *adev); +struct amdgpu_hive_info *amdgpu_get_xgmi_hive (struct amdgpu_device +*adev,int lock);int amdgpu_xgmi_update_topology (struct amdgpu_hive_info *hive,struct amdgpu_device *adev);int amdgpu_xgmi_add_device (struct amdgpu_device *adev);void amdgpu_xgmi_remove_device (struct amdgpu_device *adev); --2.17.2
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 15dbfb42-df6b-74f3-877b-0680993c23b8-5C7GfCeVMHo@public.gmane.org
Quoted 56 lines · Andrey
Quoted 44 lines · Tom
Quoted 35 lines · Andrey
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;Let's say i have device 0 in hive A and it just got a gpu reset and at
Quoted 8 lines
the same time device 1 is being added to same have though
amdgpu_xgmi_add_device,hive->hive_lockis acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take thehive->hive_lockinsideamdgpu_get_xgmi_hivebut release is still outside of the function.Quoted 2 lines
Is adding a device while resetting a device even a valid operation anyways?
In theory it's valid if you have hot pluggable devices
Quoted 5 lines
I think this means more so that the reset logic is broken. Instead there should be a per-hive reset lock that is taken and that is tested instead.
Quoted 5 lines
Quoted 1 line
Tom
The
hive->hive_lockwas added exactly for this purpose and used only for that purpose. Maybe the naming i gave it wasn't reflective of it's purpose :)
But the add/remove should use per-hive locks not the global lock... :-)
(I'm honestly not trying to bike shed I just thought the get_hive function looked wrong :-)).
Tom
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): d1c4662b-2e8c-bf38-dd14-de919f10488e-5C7GfCeVMHo@public.gmane.org
Quoted 58 lines · Tom
Quoted 48 lines · Andrey
Quoted 43 lines · Tom
Quoted 34 lines · Andrey
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;Let's say i have device 0 in hive A and it just got a gpu reset and at
Quoted 8 lines
the same time device 1 is being added to same have though
amdgpu_xgmi_add_device,hive->hive_lockis acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take thehive->hive_lockinsideamdgpu_get_xgmi_hivebut release is still outside of the function.Quoted 2 lines
Is adding a device while resetting a device even a valid operation anyways?
In theory it's valid if you have hot pluggable devices
Quoted 8 lines
Quoted 5 lines
I think this means more so that the reset logic is broken. Instead there should be a per-hive reset lock that is taken and that is tested instead.
Tom
The
hive->hive_lockwas added exactly for this purpose and used only for that purpose. Maybe the naming i gave it wasn't reflective of it's purpose :)
Quoted 6 lines
But the add/remove should use per-hive locks not the global lock... :-)
(I'm honestly not trying to bike shed I just thought the
get_hivefunction looked wrong :-)).Tom
Totally agree with you, if Shayun (who origianlly added the global xgmi_mutex) is fine with switching to per hive mutex then me too, I just point out the problem with gpu reset and as you said we then need to rename the existing hive_lock into reset_lock and then and another per hive lock to do what you propose. Also - is there a way to not take the hive lock inside amdgpu_get_xgmi_hive but release it outside ? AFAIK it's an opening for problems where people use it but forget to call release.
Andrey
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 133f4e17-aa94-16cc-9bea-2eb16140e1de-5C7GfCeVMHo@public.gmane.org
Quoted 68 lines · Andrey
Quoted 58 lines · Tom
Quoted 48 lines · Andrey
Quoted 43 lines · Tom
Quoted 34 lines · Andrey
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;Let's say i have device 0 in hive A and it just got a gpu reset and at
Quoted 8 lines
the same time device 1 is being added to same have though
amdgpu_xgmi_add_device,hive->hive_lockis acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take thehive->hive_lockinsideamdgpu_get_xgmi_hivebut release is still outside of the function.Quoted 2 lines
Is adding a device while resetting a device even a valid operation anyways?
In theory it's valid if you have hot pluggable devices
Quoted 8 lines
Quoted 5 lines
I think this means more so that the reset logic is broken. Instead there should be a per-hive reset lock that is taken and that is tested instead.
Tom
The
hive->hive_lockwas added exactly for this purpose and used only for that purpose. Maybe the naming i gave it wasn't reflective of it's purpose :)Quoted 6 lines
But the add/remove should use per-hive locks not the global lock... :-)
(I'm honestly not trying to bike shed I just thought the
get_hivefunction looked wrong :-)).Tom
Quoted 8 lines
Totally agree with you, if Shayun (who origianlly added the global
xgmi_mutex) is fine with switching to per hive mutex then me too, I just point out the problem with gpu reset and as you said we then need to rename the existinghive_lockintoreset_lockand then and another per hive lock to do what you propose. Also - is there a way to not take the hive lock insideamdgpu_get_xgmi_hivebut release it outside ? AFAIK it's an opening for problems where people use it but forget to call release.
I wanted to take the per-hive lock inside get_hive because it also takes the global lock so that add/remove couldn't happen in parallel.
For instance, deleting the last node while adding a new node means the per-hive mutex could be in limbo (because remove will delete the lock).
Adding a per-hive reset lock would fix the remaining issues no?
Tom
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 136b5b59-4f10-2eed-7295-bdeee06025ac-5C7GfCeVMHo@public.gmane.org
Quoted 76 lines · Tom
Quoted 64 lines · Andrey
Quoted 56 lines · Tom
Quoted 47 lines · Andrey
Quoted 43 lines · Tom
Quoted 34 lines · Andrey
Quoted 24 lines · Shaoyun
I think it's reasonable to use the hive specific lock for hive specific functions. The changes is acked-by Shaoyun.liu < [email protected]>
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of StDenis, Tom
Sent: Monday, January 7, 2019 10:16 AM
To: [email protected]
Cc: StDenis, Tom <[email protected]>
Subject: [PATCH] add missing mutex lock toamdgpu_get_xgmi_hive()(v2)v2: Move locks around in other functions so that this function can stand on its own. Also only hold the hive specific lock for add/remove device instead of the driver global lock so you can't add/remove devices in parallel from one hive.
Signed-off-by Tom St Denis <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 36 ++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · line 3531
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-1@@ -3525,7 +3525,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,* by different nodes. No point also since the one node already executing * reset will also reset all the other nodes in the hive. */ - hive =amdgpu_get_xgmi_hive (adev); + hive =amdgpu_get_xgmi_hive (adev,0 );if (hive && adev->gmc.xgmi.num_physical_nodes >1 && !mutex_trylock (&hive->hive_lock))return 0 ;Let's say i have device 0 in hive A and it just got a gpu reset and at
Quoted 8 lines
the same time device 1 is being added to same have though
amdgpu_xgmi_add_device,hive->hive_lockis acquired by this new device being added and so gpu reset for device 0 bails out on '!mutex_trylock(&hive->hive_lock))' without completing the reset. Also in general i feel a bit uncomfortable about the confusing acquisition scheme in the function and the fact that you take thehive->hive_lockinsideamdgpu_get_xgmi_hivebut release is still outside of the function.Quoted 2 lines
Is adding a device while resetting a device even a valid operation anyways?
In theory it's valid if you have hot pluggable devices
Quoted 8 lines
Quoted 5 lines
I think this means more so that the reset logic is broken. Instead there should be a per-hive reset lock that is taken and that is tested instead.
Tom
The
hive->hive_lockwas added exactly for this purpose and used only for that purpose. Maybe the naming i gave it wasn't reflective of it's purpose :)Quoted 6 lines
But the add/remove should use per-hive locks not the global lock... :-)
(I'm honestly not trying to bike shed I just thought the
get_hivefunction looked wrong :-)).Tom
Quoted 8 lines
Totally agree with you, if Shayun (who origianlly added the global
xgmi_mutex) is fine with switching to per hive mutex then me too, I just point out the problem with gpu reset and as you said we then need to rename the existinghive_lockintoreset_lockand then and another per hive lock to do what you propose. Also - is there a way to not take the hive lock insideamdgpu_get_xgmi_hivebut release it outside ? AFAIK it's an opening for problems where people use it but forget to call release.I wanted to take the per-hive lock inside
get_hivebecause it also takes the global lock so that add/remove couldn't happen in parallel.
Quoted 6 lines
For instance, deleting the last node while adding a new node means the per-hive mutex could be in limbo (because remove will delete the lock).
Adding a per-hive reset lock would fix the remaining issues no?
Tom
Looks like it.
Andrey
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx