drm/amdkfd: Add queue to MES if it becomes active
Subsystems (3)
PATCH 1/2 drm/amdkfd: Add queue to MES if it becomes active
We remove the user queue from MES scheduler to update queue properties. If the queue becomes active after updating, add the user queue to MES scheduler, to be able to handle command packet submission.
--- drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4 -2
@@ -866,8 +866,10 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q,* dqm->active_queue_count to determine whether a new runlist must be * uploaded. */ -if (q->properties.is_active && !prev_active) { -increment_queue_count (dqm, &pdd->qpd, q); +if (q->properties.is_active) { + add_queue = true; +if (!prev_active) +increment_queue_count (dqm, &pdd->qpd, q); }else if (!q->properties.is_active && prev_active) {decrement_queue_count (dqm, &pdd->qpd, q); }else if (q->gws && !q->properties.is_gws) {
Signature · 1 line
2.35.1
PATCH 2/2 drm/amdkfd: Free queue after unmap queue success
After queue unmap or remove from MES successfully, free queue sysfs entries, doorbell and remove from queue list. Otherwise, application may destroy queue again, cause below kernel warning or crash backtrace.
For outstanding queues, either application forget to destroy or failed to destroy, kfd_process_notifier_release will remove queue sysfs entries, kfd_process_wq_release will free queue doorbell.
refcount_t: underflow; use-after-free.
WARNING: CPU: 7 PID: 3053 at lib/refcount.c:28 Call Trace: kobject_put+0xd6/0x1a0 kfd_procfs_del_queue+0x27/0x30 [amdgpu] pqm_destroy_queue+0xeb/0x240 [amdgpu] kfd_ioctl_destroy_queue+0x32/0x70 [amdgpu] kfd_ioctl+0x27d/0x500 [amdgpu] do_syscall_64+0x35/0x80 WARNING: CPU: 2 PID: 3053 at drivers/gpu/drm/amd/amdgpu/../amdkfd/kfd_device_queue_manager.c:400 Call Trace: deallocate_doorbell.isra.0+0x39/0x40 [amdgpu] destroy_queue_cpsch+0xb3/0x270 [amdgpu] pqm_destroy_queue+0x108/0x240 [amdgpu] kfd_ioctl_destroy_queue+0x32/0x70 [amdgpu] kfd_ioctl+0x27d/0x500 [amdgpu] general protection fault, probably for non-canonical address 0xdead000000000108: Call Trace: pqm_destroy_queue+0xf0/0x200 [amdgpu] kfd_ioctl_destroy_queue+0x2f/0x60 [amdgpu] kfd_ioctl+0x19b/0x600 [amdgpu]
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 27 +++++++++++-------- .../amd/amdkfd/kfd_process_queue_manager.c | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +16 -11
@@ -1875,6 +1875,21 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, } +if (q->properties.is_active) { +if (!dqm->dev->shared_resources.enable_mes) { +decrement_queue_count (dqm, qpd, q); + retval =execute_queues_cpsch (dqm, +KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES ,0 ); +if (retval == -ETIME ) + qpd->reset_wavefronts = true; + }else { + retval =remove_queue_mes (dqm, q, qpd); + } + +if (retval) + goto failed_unmap_queue; + } + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type ( q->properties.type)];
@@ -1888,17 +1903,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,list_del (&q->list); qpd->queue_count--; -if (q->properties.is_active) { -if (!dqm->dev->shared_resources.enable_mes) { -decrement_queue_count (dqm, qpd, q); - retval =execute_queues_cpsch (dqm, -KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES ,0 ); -if (retval == -ETIME ) - qpd->reset_wavefronts = true; - }else { - retval =remove_queue_mes (dqm, q, qpd); - } - }/* * Unconditionally decrement this counter, regardless of the queue's
@@ -1915,6 +1919,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,return retval; +failed_unmap_queue: failed_try_destroy_debugged_queue:dqm_unlock (dqm);
drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +1 -1
@@ -419,7 +419,6 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) }if (pqn->q) { -kfd_procfs_del_queue (pqn->q); dqm = pqn->q->device->dqm; retval = dqm->ops.destroy_queue (dqm, &pdd->qpd, pqn->q);if (retval) {
@@ -439,6 +438,7 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)if (dev->shared_resources.enable_mes)amdgpu_amdkfd_free_gtt_mem (dev->adev, pqn->q->gang_ctx_bo); +kfd_procfs_del_queue (pqn->q);uninit_queue (pqn->q); }
Signature · 1 line
2.35.1
[Public]
Quoted 40 lines
-----Original Message-----
From: Yang, Philip <[email protected]>
Sent: Wednesday, June 15, 2022 5:57 PM
To: [email protected]
Cc: Sider, Graham <[email protected]>; Yang, Philip <[email protected]>
Subject: [PATCH 1/2] drm/amdkfd: Add queue to MES if it becomes activeWe remove the user queue from MES scheduler to update queue properties.
If the queue becomes active after updating, add the user queue to MES scheduler, to be able to handle command packet submission.Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-0@@ -866,8 +866,10 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q, * dqm->active_queue_count to determine whether a new runlistmust be
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4
-3* uploaded. */ -if (q->properties.is_active && !prev_active) { -increment_queue_count (dqm, &pdd->qpd, q); +if (q->properties.is_active) { + add_queue = true; +if (!prev_active) +increment_queue_count (dqm, &pdd->qpd, q); }else if (!q->properties.is_active && prev_active) {decrement_queue_count (dqm, &pdd->qpd, q); }else if (q->gws && !q->properties.is_gws) { --2.35.1
Does it break the case of q->gws with q->properties.is_active == true?
Regards,
Eric
Quoted 26 lines · Philip Yang
We remove the user queue from MES scheduler to update queue properties. If the queue becomes active after updating, add the user queue to MES scheduler, to be able to handle command packet submission.
Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4
-2@@ -866,8 +866,10 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q,* dqm->active_queue_count to determine whether a new runlist must be * uploaded. */ -if (q->properties.is_active && !prev_active) { -increment_queue_count (dqm, &pdd->qpd, q); +if (q->properties.is_active) { + add_queue = true; +if (!prev_active) +increment_queue_count (dqm, &pdd->qpd, q); }else if (!q->properties.is_active && prev_active) {decrement_queue_count (dqm, &pdd->qpd, q); }else if (q->gws && !q->properties.is_gws) {
On 2022-06-16 10:49, Eric Huang wrote:
Does it break the case of q->gws with q->properties.is_active == true?
Thanks for catching this, it breaks pqm_set_gws, I will revert this and send v2 for review.
Regards,
Philip
Regards,
Eric
On 2022-06-15 17:56, Philip Yang wrote:
We remove the user queue from MES scheduler to update queue properties. If the queue becomes active after updating, add the user queue to MES scheduler, to be able to handle command packet submission.
---
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0 -0
@@ -866,8 +866,10 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q,
* dqm->active_queue_count to determine whether a new runlist must be
* uploaded.
*/
- if (q->properties.is_active && !prev_active) {
- increment_queue_count(dqm, &pdd->qpd, q);
+ if (q->properties.is_active) {
+ add_queue = true;
+ if (!prev_active)
+ increment_queue_count(dqm, &pdd->qpd, q);
} else if (!q->properties.is_active && prev_active) { decrement_queue_count(dqm, &pdd->qpd, q);
} else if (q->gws && !q->properties.is_gws) {