Skip to contentSkip to conversation

drm/amdkfd: don't use dqm lock during device reset/suspend/resume

PATCH In review Yang, Philip 3 participants 9 messages Oct 22, 2019 7 messages not in the archive

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 drm/amdkfd: don't use dqm lock during device reset/suspend/resume

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
 drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
 .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
 4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
 		return -EPERM;
 	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
 	process = kfd_create_process(filep);
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
 	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
 		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
 		return 0;
 	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
 	kfd_signal_reset_event(kfd);
 	return 0;
 }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
 	if (!kfd->init_complete)
 		return 0;
 
-	dqm_unlock(kfd->dqm);
-
 	ret = kfd_resume(kfd);
 	if (ret)
 		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5

139 lines not on this page · raw message

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
index 2eaea6b04cbe..a8c37e6da027 100644
@@ -201,6 +201,7 @@ struct device_queue_manager {
 	bool			is_hws_hang;
 	struct work_struct	hw_exception_work;
 	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
 };
 
 void device_queue_manager_init_cik(
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

#2 lore raw
Quoted 11 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

Is there a chance of race condition here where dqm->device_stopped returns true for some operation (e.g.map_queues_cpsch) but just as it proceeds GPU reset starts  ?

Andrey

Quoted 257 lines

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
  drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
  drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
  .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
  .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
  4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
  		return -EPERM;
  	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
  	process = kfd_create_process(filep);
  	if (IS_ERR(process))
  		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
  	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
  		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
  		return 0;
  	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
  	kfd_signal_reset_event(kfd);
  	return 0;
  }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
  	if (!kfd->init_complete)
  		return 0;
 
-	dqm_unlock(kfd->dqm);
-
  	ret = kfd_resume(kfd);
  	if (ret)
  		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
  	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
  				&q->gart_mqd_addr, &q->properties);
  	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
  		if (WARN(q->process->mm != current->mm,
  					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
  			goto out_free_mqd;
  	}
 
+add_queue_to_list:
  	list_add(&q->list, &qpd->queues_list);
  	qpd->queue_count++;
  	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
  	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
  	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
  				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
  				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
  		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
  		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
  		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
  		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
  				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
  				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
  		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
  				q->properties.type)];
  		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;
+
  		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
  				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
  				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
  			 * maintain a consistent eviction state
  			 */
  			ret = retval;
-		dqm->queue_count--;
  	}
 
  out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
  		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
  				q->properties.type)];
  		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;
+
  		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
  				       q->queue, &q->properties, mm);
  		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
  			 * maintain a consistent eviction state
  			 */
  			ret = retval;
-		dqm->queue_count++;
  	}
  	qpd->evicted = 0;
  out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
  	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
  		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
  	return 0;
  }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
  {
  	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
  		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
  	return 0;
  }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
  	dqm_lock(dqm);
  	/* clear hang status when driver try to start the hw scheduler */
  	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
  	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
  	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
  {
  	dqm_lock(dqm);
  	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
  	dqm_unlock(dqm);
 
  	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
  {
  	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
  	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
  		return 0;
-
  	if (dqm->active_runlist)
  		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
  {
  	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
  	if (dqm->is_hws_hang)
  		return -EIO;
  	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
  	int pipe, queue;
  	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
  	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
  					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
  					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
index 2eaea6b04cbe..a8c37e6da027 100644

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
@@ -201,6 +201,7 @@ struct device_queue_manager {
  	bool			is_hws_hang;
  	struct work_struct	hw_exception_work;
  	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
  };
 
  void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 83bfc853-cb0e-4285-d310-232683130751-5C7GfCeVMHo@public.gmane.org

#4 lore raw
Quoted 10 lines · Andrey
Quoted 9 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

Quoted 8 lines
Quoted 2 lines

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

Is there a chance of race condition here where dqm->device_stopped returns true for some operation (e.g.map_queues_cpsch) but just as it proceeds GPU reset starts  ?

Andrey

Correction -

dqm->device_stopped returns FALSE

Andrey

Quoted 264 lines
Quoted 264 lines

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
   drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
   drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
   .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
   .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
   4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
   		return -EPERM;
   	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
   	process = kfd_create_process(filep);
   	if (IS_ERR(process))
   		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
   	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
   		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
   		return 0;
   	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
   	kfd_signal_reset_event(kfd);
   	return 0;
   }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
   	if (!kfd->init_complete)
   		return 0;
 
-	dqm_unlock(kfd->dqm);
-
   	ret = kfd_resume(kfd);
   	if (ret)
   		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
   	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
   				&q->gart_mqd_addr, &q->properties);
   	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
   		if (WARN(q->process->mm != current->mm,
   					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
   			goto out_free_mqd;
   	}
 
+add_queue_to_list:
   	list_add(&q->list, &qpd->queues_list);
   	qpd->queue_count++;
   	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
   	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
   	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
   				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
   				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
   		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
   		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
   		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
   		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
   				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
   				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
   		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
   				q->properties.type)];
   		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;
+
   		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
   				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
   				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
   			 * maintain a consistent eviction state
   			 */
   			ret = retval;
-		dqm->queue_count--;
   	}
 
   out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
   		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
   				q->properties.type)];
   		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;
+
   		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
   				       q->queue, &q->properties, mm);
   		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
   			 * maintain a consistent eviction state
   			 */
   			ret = retval;
-		dqm->queue_count++;
   	}
   	qpd->evicted = 0;
   out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
   	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
   		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
   	return 0;
   }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
   {
   	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
   		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
   	return 0;
   }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
   	dqm_lock(dqm);
   	/* clear hang status when driver try to start the hw scheduler */
   	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
   	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
   	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
   {
   	dqm_lock(dqm);
   	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
   	dqm_unlock(dqm);
 
   	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
   {
   	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
   	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
   		return 0;
-
   	if (dqm->active_runlist)
   		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
   {
   	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
   	if (dqm->is_hws_hang)
   		return -EIO;
   	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
   	int pipe, queue;
   	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
   	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
   					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
   					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
index 2eaea6b04cbe..a8c37e6da027 100644
@@ -201,6 +201,7 @@ struct device_queue_manager {
   	bool			is_hws_hang;
   	struct work_struct	hw_exception_work;
   	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
   };
 
Quoted 5 lines

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
   void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): b7c3ad13-0b36-5fbb-5079-4a72a8dd853a-5C7GfCeVMHo@public.gmane.org

#6 lore raw
Quoted 19 lines · Andrey
Quoted 17 lines · Andrey
Quoted 11 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

Is there a chance of race condition here where dqm->device_stopped returns true for some operation (e.g.map_queues_cpsch) but just as it proceeds GPU reset starts  ?

Quoted 7 lines
Quoted 1 line

Andrey

Correction -

dqm->device_stopped returns FALSE

No race condition here, dqm->device_stopped is set to FALSE in kgd2kfd_post_reset -> dqm->ops.start(), which the last step of amdgpu_device_gpu_recover, so it's safe to do map_queue_cpsch.

Regards,
Philip

Quoted 271 lines

Andrey

Quoted 269 lines
Quoted 258 lines

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
    drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
    drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
    .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
    .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
    4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
    		return -EPERM;
    	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
    	process = kfd_create_process(filep);
    	if (IS_ERR(process))
    		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
    	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
    		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
    		return 0;
    	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
    	kfd_signal_reset_event(kfd);
    	return 0;
    }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
    	if (!kfd->init_complete)
    		return 0;
 
-	dqm_unlock(kfd->dqm);
-
    	ret = kfd_resume(kfd);
    	if (ret)
    		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
    	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
    				&q->gart_mqd_addr, &q->properties);
    	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
    		if (WARN(q->process->mm != current->mm,
    					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
    			goto out_free_mqd;
    	}
 
+add_queue_to_list:
    	list_add(&q->list, &qpd->queues_list);
    	qpd->queue_count++;
    	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
    	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
    	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
    				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
    				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
    		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
    		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
    		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
    		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
    				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
    				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
    		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
    				q->properties.type)];
    		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;
+
    		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
    				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
    				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
    			 * maintain a consistent eviction state
    			 */
    			ret = retval;
-		dqm->queue_count--;
    	}
 
    out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
    		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
    				q->properties.type)];
    		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;
+
    		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
    				       q->queue, &q->properties, mm);
    		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
    			 * maintain a consistent eviction state
    			 */
    			ret = retval;
-		dqm->queue_count++;
    	}
    	qpd->evicted = 0;
    out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
    	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
    		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
    	return 0;
    }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
    {
    	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
    		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
    	return 0;
    }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
    	dqm_lock(dqm);
    	/* clear hang status when driver try to start the hw scheduler */
    	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
    	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
    	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
    {
    	dqm_lock(dqm);
    	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
    	dqm_unlock(dqm);
 
    	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
    {
    	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
    	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
    		return 0;
-
    	if (dqm->active_runlist)
    		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
    {
    	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
    	if (dqm->is_hws_hang)
    		return -EIO;
    	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
    	int pipe, queue;
    	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
    	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
    					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
    					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
index 2eaea6b04cbe..a8c37e6da027 100644

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
@@ -201,6 +201,7 @@ struct device_queue_manager {
    	bool			is_hws_hang;
    	struct work_struct	hw_exception_work;
    	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
    };
 
    void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 92e6ce2b-5a2c-62ef-9c03-51c3ea28be1e-5C7GfCeVMHo@public.gmane.org

#8 lore raw
Quoted 23 lines · Philip
Quoted 21 lines · Andrey
Quoted 10 lines · Andrey
Quoted 9 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

Quoted 7 lines
Quoted 2 lines

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

Is there a chance of race condition here where dqm->device_stopped returns true for some operation (e.g.map_queues_cpsch) but just as it proceeds GPU reset starts  ?

Andrey

Correction -

Quoted 8 lines
Quoted 2 lines

dqm->device_stopped returns FALSE

No race condition here, dqm->device_stopped is set to FALSE in kgd2kfd_post_reset -> dqm->ops.start(), which the last step of amdgpu_device_gpu_recover, so it's safe to do map_queue_cpsch.

Regards,
Philip

Sorry - i was confused by the commit description vs. body - in the description it's called device_stopped flag while in the body it's sched_running - probably the description needs to be fixed.

So i mean the switch true->false when GPU reset just begins - you get an IOCTL to map a queue (if i understood KFD code correctly), check dqm->sched_running == true and continue, what if right then GPU reset started due to some issue like job timeout or user triggered reset - dqm->sched_running becomes false but you already past that checkpoint in the IOCTL, no ?

Andrey

Quoted 266 lines
Quoted 266 lines

Andrey

Quoted 263 lines
Quoted 263 lines

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
     drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
     drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
     .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
     .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
     4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
     		return -EPERM;
     	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
     	process = kfd_create_process(filep);
     	if (IS_ERR(process))
     		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
     	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
     		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
     		return 0;
     	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
     	kfd_signal_reset_event(kfd);
     	return 0;
     }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
     	if (!kfd->init_complete)
     		return 0;
 
-	dqm_unlock(kfd->dqm);
-
     	ret = kfd_resume(kfd);
     	if (ret)
     		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
     	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
     				&q->gart_mqd_addr, &q->properties);
     	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
     		if (WARN(q->process->mm != current->mm,
     					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
     			goto out_free_mqd;
     	}
 
+add_queue_to_list:
     	list_add(&q->list, &qpd->queues_list);
     	qpd->queue_count++;
     	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
     	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
     	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
     				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
     				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
     		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
     		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
     		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
     		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
     				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
     				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
     		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
     				q->properties.type)];
     		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;
+
     		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
     				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
     				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
     			 * maintain a consistent eviction state
     			 */
     			ret = retval;
-		dqm->queue_count--;
     	}
 
     out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
     		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
     				q->properties.type)];
     		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;
+
     		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
     				       q->queue, &q->properties, mm);
     		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
     			 * maintain a consistent eviction state
     			 */
     			ret = retval;
-		dqm->queue_count++;
     	}
     	qpd->evicted = 0;
     out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
     	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
     		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
     	return 0;
     }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
     {
     	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
     		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
     	return 0;
     }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
     	dqm_lock(dqm);
     	/* clear hang status when driver try to start the hw scheduler */
     	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
     	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
     	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
     {
     	dqm_lock(dqm);
     	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
     	dqm_unlock(dqm);
 
     	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
     {
     	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
     	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
     		return 0;
-
     	if (dqm->active_runlist)
     		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
     {
     	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
     	if (dqm->is_hws_hang)
     		return -EIO;
     	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
     	int pipe, queue;
     	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
     	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
     					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
     					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
index 2eaea6b04cbe..a8c37e6da027 100644
@@ -201,6 +201,7 @@ struct device_queue_manager {
     	bool			is_hws_hang;
     	struct work_struct	hw_exception_work;
     	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
     };
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
     void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): b51add2e-ded9-0de0-962b-96cf458340be-5C7GfCeVMHo@public.gmane.org

#10 lore raw
Quoted 31 lines · Andrey
Quoted 29 lines · Philip
Quoted 17 lines · Andrey
Quoted 16 lines · Andrey
Quoted 11 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

Is there a chance of race condition here where dqm->device_stopped returns true for some operation (e.g.map_queues_cpsch) but just as it proceeds GPU reset starts  ?

Quoted 6 lines
Quoted 1 line

Andrey

Correction -

dqm->device_stopped returns FALSE

No race condition here, dqm->device_stopped is set to FALSE in kgd2kfd_post_reset -> dqm->ops.start(), which the last step of amdgpu_device_gpu_recover, so it's safe to do map_queue_cpsch.

Quoted 7 lines
Quoted 2 lines

Regards,
Philip

Sorry - i was confused by the commit description vs. body - in the description it's called device_stopped flag while in the body it's sched_running - probably the description needs to be fixed.

Thanks, commit description is updated.

Quoted 7 lines

So i mean the switch true->false when GPU reset just begins - you get an IOCTL to map a queue (if i understood KFD code correctly), check dqm->sched_running == true and continue, what if right then GPU reset started due to some issue like job timeout or user triggered reset - dqm->sched_running becomes false but you already past that checkpoint in the IOCTL, no ?

map a queue is done under dqm lock, to send the updated runlist to HWS, then relase dqm lock. GPU reset start pre_reset will unmap all queues first under dqm lock, then set dqm->sched_running to false, release dqm lock, and then continue to reset HW. dqm lock guarantee the two operations are serialized.

Philip

Quoted 273 lines

Andrey

Quoted 271 lines
Quoted 271 lines

Andrey

Quoted 268 lines
Quoted 257 lines

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
      drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
      drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
      .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
      .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
      4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
      		return -EPERM;
      	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
      	process = kfd_create_process(filep);
      	if (IS_ERR(process))
      		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
      	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
      		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
      		return 0;
      	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
      	kfd_signal_reset_event(kfd);
      	return 0;
      }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
      	if (!kfd->init_complete)
      		return 0;
 
-	dqm_unlock(kfd->dqm);
-
      	ret = kfd_resume(kfd);
      	if (ret)
      		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
      	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
      				&q->gart_mqd_addr, &q->properties);
      	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
      		if (WARN(q->process->mm != current->mm,
      					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
      			goto out_free_mqd;
      	}
 
+add_queue_to_list:
      	list_add(&q->list, &qpd->queues_list);
      	qpd->queue_count++;
      	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
      	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
      	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
      				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
      				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
      		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
      		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
      		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
      		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
      				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
      				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
      		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
      				q->properties.type)];
      		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;
+
      		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
      				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
      				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
      			 * maintain a consistent eviction state
      			 */
      			ret = retval;
-		dqm->queue_count--;
      	}
 
      out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
      		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
      				q->properties.type)];
      		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;
+
      		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
      				       q->queue, &q->properties, mm);
      		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
      			 * maintain a consistent eviction state
      			 */
      			ret = retval;
-		dqm->queue_count++;
      	}
      	qpd->evicted = 0;
      out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
      	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
      		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
      	return 0;
      }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
      {
      	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
      		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
      	return 0;
      }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
      	dqm_lock(dqm);
      	/* clear hang status when driver try to start the hw scheduler */
      	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
      	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
      	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
      {
      	dqm_lock(dqm);
      	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
      	dqm_unlock(dqm);
 
      	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
      {
      	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
      	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
      		return 0;
-
      	if (dqm->active_runlist)
      		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
      {
      	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
      	if (dqm->is_hws_hang)
      		return -EIO;
      	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
      	int pipe, queue;
      	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
      	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
      					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
      					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
index 2eaea6b04cbe..a8c37e6da027 100644

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
@@ -201,6 +201,7 @@ struct device_queue_manager {
      	bool			is_hws_hang;
      	struct work_struct	hw_exception_work;
      	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
      };
 
      void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): a400a3cf-b8e2-18d7-7460-14c304a9c7d4-5C7GfCeVMHo@public.gmane.org

#12 lore raw
Quoted 45 lines · Philip
Quoted 28 lines · Andrey
Quoted 27 lines · Philip
Quoted 13 lines · Andrey
Quoted 12 lines · Andrey
Quoted 11 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

Quoted 9 lines
Quoted 5 lines

Is there a chance of race condition here where dqm->device_stopped returns true for some operation (e.g.map_queues_cpsch) but just as it proceeds GPU reset starts  ?

Andrey

Correction -

dqm->device_stopped returns FALSE

No race condition here, dqm->device_stopped is set to FALSE in kgd2kfd_post_reset -> dqm->ops.start(), which the last step of amdgpu_device_gpu_recover, so it's safe to do map_queue_cpsch.

Quoted 6 lines
Quoted 2 lines

Regards,
Philip

Sorry - i was confused by the commit description vs. body - in the description it's called device_stopped flag while in the body it's sched_running - probably the description needs to be fixed.

Thanks, commit description is updated.

Quoted 7 lines

So i mean the switch true->false when GPU reset just begins - you get an IOCTL to map a queue (if i understood KFD code correctly), check dqm->sched_running == true and continue, what if right then GPU reset started due to some issue like job timeout or user triggered reset - dqm->sched_running becomes false but you already past that checkpoint in the IOCTL, no ?

Quoted 7 lines

map a queue is done under dqm lock, to send the updated runlist to HWS, then relase dqm lock. GPU reset start pre_reset will unmap all queues first under dqm lock, then set dqm->sched_running to false, release dqm lock, and then continue to reset HW. dqm lock guarantee the two operations are serialized.

Philip

I see now that it's ok.

Andrey

Quoted 267 lines
Quoted 267 lines

Andrey

Quoted 264 lines
Quoted 264 lines

Andrey

Quoted 262 lines
Quoted 262 lines

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
---
       drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
       drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
       .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
       .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
       4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
       		return -EPERM;
       	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
       	process = kfd_create_process(filep);
       	if (IS_ERR(process))
       		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-
       	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
       		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
       		return 0;
       	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
       	kfd_signal_reset_event(kfd);
       	return 0;
       }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
       	if (!kfd->init_complete)
       		return 0;
 
-	dqm_unlock(kfd->dqm);
-
       	ret = kfd_resume(kfd);
       	if (ret)
       		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +42 -5
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
       	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
       				&q->gart_mqd_addr, &q->properties);
       	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
       		if (WARN(q->process->mm != current->mm,
       					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
       			goto out_free_mqd;
       	}
 
+add_queue_to_list:
       	list_add(&q->list, &qpd->queues_list);
       	qpd->queue_count++;
       	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
       	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
       	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
       				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
       				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
       		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
       		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
       		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
       		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
       				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
       				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
       		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
       				q->properties.type)];
       		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;
+
       		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
       				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
       				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
       			 * maintain a consistent eviction state
       			 */
       			ret = retval;
-		dqm->queue_count--;
       	}
 
       out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
       		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
       				q->properties.type)];
       		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;
+
       		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
       				       q->queue, &q->properties, mm);
       		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
       			 * maintain a consistent eviction state
       			 */
       			ret = retval;
-		dqm->queue_count++;
       	}
       	qpd->evicted = 0;
       out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
       	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
       		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
       	return 0;
       }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
       {
       	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
       		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
       	return 0;
       }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
       	dqm_lock(dqm);
       	/* clear hang status when driver try to start the hw scheduler */
       	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
       	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
       	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
       {
       	dqm_lock(dqm);
       	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
       	dqm_unlock(dqm);
 
       	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
       {
       	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
       	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
       		return 0;
-
       	if (dqm->active_runlist)
       		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
       {
       	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
       	if (dqm->is_hws_hang)
       		return -EIO;
       	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
       	int pipe, queue;
       	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
       	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
       					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
       					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
index 2eaea6b04cbe..a8c37e6da027 100644
@@ -201,6 +201,7 @@ struct device_queue_manager {
       	bool			is_hws_hang;
       	struct work_struct	hw_exception_work;
       	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
       };
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
       void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Reviewed-by Felix Kuehling
#13 lore raw
In reply to a message not in the archive
Quoted 52 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
Quoted 2 lines

Three more comments inline. With those comments addressed, this patch is

Quoted 15 lines
---
  drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
  drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
  .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
  .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
  4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +0 -0
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
  		return -EPERM;
  	}
 

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c static int kfd_open(struct inode *inode, struct file *filep) · line 129

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -3
+	if (kfd_is_locked())
+		return -EAGAIN;
+
  	process = kfd_create_process(filep);
  	if (IS_ERR(process))
  		return PTR_ERR(process);
 
-	if (kfd_is_locked())
-		return -EAGAIN;
-

Is this part of the change still needed? I remember that this sequence was a bit tricky with some potential race condition when Shaoyun was working on it. This may have unintended side effects.

Quoted 75 lines
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +0 -0
  	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
  		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
  		return 0;
  	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
  	kfd_signal_reset_event(kfd);
  	return 0;
  }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
  	if (!kfd->init_complete)
  		return 0;
 
-	dqm_unlock(kfd->dqm);
-
  	ret = kfd_resume(kfd);
  	if (ret)
  		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +16 -0
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
  	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
  				&q->gart_mqd_addr, &q->properties);
  	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
  		if (WARN(q->process->mm != current->mm,
  					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
  			goto out_free_mqd;
  	}
 
+add_queue_to_list:
  	list_add(&q->list, &qpd->queues_list);
  	qpd->queue_count++;
  	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
  	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
  	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
  				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
  				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
  		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
  		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
  		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
  		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
  				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
  				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, · line 624

Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4 -0
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
  		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
  				q->properties.type)];
  		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;

Add a WARN here as well. When the scheduler is not running the queues should already be evicted.

Quoted 13 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1 -1
+
  		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
  				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
  				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
  			 * maintain a consistent eviction state
  			 */
  			ret = retval;
-		dqm->queue_count--;
  	}
 
  out:

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · line 737

Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4 -0
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
  		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
  				q->properties.type)];
  		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;

Add a WARN here as well. This should not happen while we're suspended.

Regards,
  Felix

Quoted 87 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +17 -4
+
  		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
  				       q->queue, &q->properties, mm);
  		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
  			 * maintain a consistent eviction state
  			 */
  			ret = retval;
-		dqm->queue_count++;
  	}
  	qpd->evicted = 0;
  out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
  	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
  		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
  	return 0;
  }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
  {
  	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
  		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
  	return 0;
  }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
  	dqm_lock(dqm);
  	/* clear hang status when driver try to start the hw scheduler */
  	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
  	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
  	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
  {
  	dqm_lock(dqm);
  	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
  	dqm_unlock(dqm);
 
  	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
  {
  	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
  	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
  		return 0;
-
  	if (dqm->active_runlist)
  		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
  {
  	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
  	if (dqm->is_hws_hang)
  		return -EIO;
  	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
  	int pipe, queue;
  	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
  	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
  					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
  					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
index 2eaea6b04cbe..a8c37e6da027 100644

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
@@ -201,6 +201,7 @@ struct device_queue_manager {
  	bool			is_hws_hang;
  	struct work_struct	hw_exception_work;
  	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
  };
 
  void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 558b2dd3-8095-81e0-547b-788b74c9b329-5C7GfCeVMHo@public.gmane.org

#15 lore raw
Quoted 83 lines · Felix
Quoted 52 lines · Philip

If device reset/suspend/resume failed for some reason, dqm lock is hold forever and this causes deadlock. Below is a kernel backtrace when application open kfd after suspend/resume failed.

Instead of holding dqm lock in pre_reset and releasing dqm lock in post_reset, add dqm->device_stopped flag which is modified in dqm->ops.start and dqm->ops.stop. The flag doesn't need lock protection because write/read are all inside dqm lock.

For HWS case, map_queues_cpsch and unmap_queues_cpsch checks device_stopped flag before sending the updated runlist.

v2: For no-HWS case, when device is stopped, don't call load/destroy_mqd for eviction, restore and create queue, and avoid debugfs dump hdqs.

Backtrace of dqm lock deadlock:

[Thu Oct 17 16:43:37 2019] INFO: task rocminfo:3024 blocked for more
than 120 seconds.
[Thu Oct 17 16:43:37 2019]       Not tainted
5.0.0-rc1-kfd-compute-rocm-dkms-no-npi-1131 #1
[Thu Oct 17 16:43:37 2019] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
[Thu Oct 17 16:43:37 2019] rocminfo        D    0  3024   2947
0x80000000
[Thu Oct 17 16:43:37 2019] Call Trace:
[Thu Oct 17 16:43:37 2019]  ? __schedule+0x3d9/0x8a0
[Thu Oct 17 16:43:37 2019]  schedule+0x32/0x70
[Thu Oct 17 16:43:37 2019]  schedule_preempt_disabled+0xa/0x10
[Thu Oct 17 16:43:37 2019]  __mutex_lock.isra.9+0x1e3/0x4e0
[Thu Oct 17 16:43:37 2019]  ? __call_srcu+0x264/0x3b0
[Thu Oct 17 16:43:37 2019]  ? process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]  process_termination_cpsch+0x24/0x2f0
[amdgpu]
[Thu Oct 17 16:43:37 2019]
kfd_process_dequeue_from_all_devices+0x42/0x60 [amdgpu]
[Thu Oct 17 16:43:37 2019]  kfd_process_notifier_release+0x1be/0x220
[amdgpu]
[Thu Oct 17 16:43:37 2019]  __mmu_notifier_release+0x3e/0xc0
[Thu Oct 17 16:43:37 2019]  exit_mmap+0x160/0x1a0
[Thu Oct 17 16:43:37 2019]  ? __handle_mm_fault+0xba3/0x1200
[Thu Oct 17 16:43:37 2019]  ? exit_robust_list+0x5a/0x110
[Thu Oct 17 16:43:37 2019]  mmput+0x4a/0x120
[Thu Oct 17 16:43:37 2019]  do_exit+0x284/0xb20
[Thu Oct 17 16:43:37 2019]  ? handle_mm_fault+0xfa/0x200
[Thu Oct 17 16:43:37 2019]  do_group_exit+0x3a/0xa0
[Thu Oct 17 16:43:37 2019]  __x64_sys_exit_group+0x14/0x20
[Thu Oct 17 16:43:37 2019]  do_syscall_64+0x4f/0x100
[Thu Oct 17 16:43:37 2019]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
Quoted 2 lines

Three more comments inline. With those comments addressed, this patch is

Quoted 22 lines
---
   drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |  6 +--
   drivers/gpu/drm/amd/amdkfd/kfd_device.c       |  5 --
   .../drm/amd/amdkfd/kfd_device_queue_manager.c | 47 +++++++++++++++++--
   .../drm/amd/amdkfd/kfd_device_queue_manager.h |  1 +
   4 files changed, 46 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -0
index d9e36dbf13d5..40d75c39f08e 100644
@@ -120,13 +120,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
   		return -EPERM;
   	}
 
+	if (kfd_is_locked())
+		return -EAGAIN;
+
   	process = kfd_create_process(filep);
   	if (IS_ERR(process))
   		return PTR_ERR(process);
 
Quoted 9 lines

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c static int kfd_open(struct inode *inode, struct file *filep) · line 129

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +0 -3
-	if (kfd_is_locked())
-		return -EAGAIN;
-

Is this part of the change still needed? I remember that this sequence was a bit tricky with some potential race condition when Shaoyun was working on it. This may have unintended side effects.

Revert this change to be safe, it is not needed and not clear if there are side effects.

Quoted 77 lines
Quoted 77 lines
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +0 -0
   	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
   		process->pasid, process->is_32bit_user_mode);
 
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -5
index 8f4b24e84964..4fa8834ce7cb 100644
@@ -730,9 +730,6 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
   		return 0;
   	kgd2kfd_suspend(kfd);
 
-	/* hold dqm->lock to prevent further execution*/
-	dqm_lock(kfd->dqm);
-
   	kfd_signal_reset_event(kfd);
   	return 0;
   }
@@ -750,8 +747,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
   	if (!kfd->init_complete)
   		return 0;
 
-	dqm_unlock(kfd->dqm);
-
   	ret = kfd_resume(kfd);
   	if (ret)
   		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +16 -0
index 81fb545cf42c..82e1c6280d13 100644
@@ -340,6 +340,10 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
   	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
   				&q->gart_mqd_addr, &q->properties);
   	if (q->properties.is_active) {
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
+			goto add_queue_to_list;
+		}
 
   		if (WARN(q->process->mm != current->mm,
   					"should only run in user thread"))
@@ -351,6 +355,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
   			goto out_free_mqd;
   	}
 
+add_queue_to_list:
   	list_add(&q->list, &qpd->queues_list);
   	qpd->queue_count++;
   	if (q->properties.is_active)
@@ -458,6 +463,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 
   	deallocate_doorbell(qpd, q);
 
+	if (!dqm->sched_running) {
+		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
+		return 0;
+	}
+
   	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
   				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
   				KFD_UNMAP_LATENCY_MS,
@@ -533,6 +543,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
   		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
   		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
   		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
+
+		if (!dqm->sched_running) {
+			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
+			goto out_unlock;
+		}
+
   		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
   				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
   				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -602,6 +618,11 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
   		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
   				q->properties.type)];
Quoted 9 lines

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, · line 624

Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4 -0
   		q->properties.is_active = false;
+		dqm->queue_count--;
+
+		if (!dqm->sched_running)
+			continue;

Add a WARN here as well. When the scheduler is not running the queues should already be evicted.

Done

Quoted 12 lines
Quoted 12 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1 -1
+
   		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
   				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
   				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -610,7 +631,6 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
   			 * maintain a consistent eviction state
   			 */
   			ret = retval;
-		dqm->queue_count--;
   	}
 
Quoted 12 lines

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · line 737

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4 -0
   out:
@@ -711,6 +731,11 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
   		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
   				q->properties.type)];
   		q->properties.is_active = true;
+		dqm->queue_count++;
+
+		if (!dqm->sched_running)
+			continue;

Add a WARN here as well. This should not happen while we're suspended.

Done.

Quoted 96 lines

Regards,
  Felix

Quoted 93 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +17 -4
+
   		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
   				       q->queue, &q->properties, mm);
   		if (retval && !ret)
@@ -718,7 +743,6 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
   			 * maintain a consistent eviction state
   			 */
   			ret = retval;
-		dqm->queue_count++;
   	}
   	qpd->evicted = 0;
   out:
@@ -915,7 +939,8 @@ static int start_nocpsch(struct device_queue_manager *dqm)
 
   	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
   		return pm_init(&dqm->packets, dqm);
-	
+	dqm->sched_running = true;
+
   	return 0;
   }
 
@@ -923,7 +948,8 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
   {
   	if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
   		pm_uninit(&dqm->packets);
-	
+	dqm->sched_running = false;
+
   	return 0;
   }
 
@@ -1074,6 +1100,7 @@ static int start_cpsch(struct device_queue_manager *dqm)
   	dqm_lock(dqm);
   	/* clear hang status when driver try to start the hw scheduler */
   	dqm->is_hws_hang = false;
+	dqm->sched_running = true;
   	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
   	dqm_unlock(dqm);
 
@@ -1089,6 +1116,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)
   {
   	dqm_lock(dqm);
   	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm->sched_running = false;
   	dqm_unlock(dqm);
 
   	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
@@ -1275,9 +1303,10 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
   {
   	int retval;
 
+	if (!dqm->sched_running)
+		return 0;
   	if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
   		return 0;
-
   	if (dqm->active_runlist)
   		return 0;
 
@@ -1299,6 +1328,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
   {
   	int retval = 0;
 
+	if (!dqm->sched_running)
+		return 0;
   	if (dqm->is_hws_hang)
   		return -EIO;
   	if (!dqm->active_runlist)
@@ -1903,6 +1934,12 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
   	int pipe, queue;
   	int r = 0;
 
+	if (!dqm->sched_running) {
+		seq_printf(m, " Device is stopped\n");
+
+		return 0;
+	}
+
   	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
   					KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
   					&dump, &n_regs);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +1 -0
index 2eaea6b04cbe..a8c37e6da027 100644
@@ -201,6 +201,7 @@ struct device_queue_manager {
   	bool			is_hws_hang;
   	struct work_struct	hw_exception_work;
   	struct kfd_mem_obj	hiq_sdma_mqd;
+	bool			sched_running;
   };
 
Quoted 1 line

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h struct device_queue_manager { · line 207

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +0 -0
   void device_queue_manager_init_cik(

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx