drm/amdkfd: support per-queue reset on gfx9
Subsystems (3)
PATCH 1/2 drm/amdkfd: support per-queue reset on gfx9
Support per-queue reset for GFX9. The recommendation is for the driver to target reset the HW queue via a SPI MMIO register write.
Since this requires pipe and HW queue info and MEC FW is limited to doorbell reports of hung queues after an unmap failure, scan the HW queue slots defined by SET_RESOURCES first to identify the user queue candidates to reset.
Only signal reset events to processes that have had a queue reset.
If queue reset fails, fall back to GPU reset.
--- .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 1 + .../drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c | 9 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h | 6 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 11 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 56 +++++ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 6 + .../drm/amd/amdkfd/kfd_device_queue_manager.c | 195 ++++++++++++++++-- .../drm/amd/amdkfd/kfd_device_queue_manager.h | 12 ++ drivers/gpu/drm/amd/amdkfd/kfd_events.c | 15 ++ .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 6 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 2 + .../gpu/drm/amd/include/kgd_kfd_interface.h | 4 + 16 files changed, 310 insertions(+), 26 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +1 -0
@@ -191,4 +191,5 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +2 -1
@@ -418,5 +418,6 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, - .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings + .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c +2 -1
@@ -541,5 +541,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { kgd_gfx_v9_4_3_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_aldebaran_set_wave_launch_mode, .set_address_watch = kgd_gfx_v9_4_3_set_address_watch, - .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch + .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +9 -0
@@ -1070,6 +1070,14 @@ static void program_trap_handler_settings(struct amdgpu_device *adev,unlock_srbm (adev); } +uint64_t kgd_gfx_v10_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only) +{ +return 0 ; +} +const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .program_sh_mem_settings = kgd_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
@@ -1097,4 +1105,5 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info, .program_trap_handler_settings = program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +6 -0
@@ -56,3 +56,9 @@ void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v10_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +2 -1
@@ -680,5 +680,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .set_wave_launch_trap_override = kgd_gfx_v10_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_gfx_v10_set_wave_launch_mode, .set_address_watch = kgd_gfx_v10_set_address_watch, - .clear_address_watch = kgd_gfx_v10_clear_address_watch + .clear_address_watch = kgd_gfx_v10_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +10 -1
@@ -786,6 +786,14 @@ static uint32_t kgd_gfx_v11_clear_address_watch(struct amdgpu_device *adev,return 0 ; } +static uint64_t kgd_gfx_v11_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only) +{ +return 0 ; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,
@@ -808,5 +816,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .set_wave_launch_trap_override = kgd_gfx_v11_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_gfx_v11_set_wave_launch_mode, .set_address_watch = kgd_gfx_v11_set_address_watch, - .clear_address_watch = kgd_gfx_v11_clear_address_watch + .clear_address_watch = kgd_gfx_v11_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v11_hqd_detect_and_reset };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +56 -0
66 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +6 -0
@@ -101,3 +101,9 @@ void kgd_gfx_v9_build_grace_period_packet_info(struct amdgpu_device *adev,uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +174 -21
243 lines not on this page · raw message
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +12 -0
@@ -210,6 +210,13 @@ struct device_queue_manager_asic_ops { struct kfd_node *dev); }; +struct dqm_detect_hang_info { +int pipe_id; +int queue_id; +int xcc_id; +uint64_t queue_address; +}; +/** * struct device_queue_manager *
@@ -264,6 +271,11 @@ struct device_queue_manager {uint32_t wait_times;wait_queue_head_t destroy_wait; + +/* for per-queue reset support */ +struct dqm_detect_hang_info *detect_hang_info; +size_t detect_hang_info_size; +int detect_hang_count; }; void device_queue_manager_init_cik(
drivers/gpu/drm/amd/amdkfd/kfd_events.c +15 -0
@@ -31,6 +31,7 @@#include <linux/memory.h> #include "kfd_priv.h" #include "kfd_events.h" +#include "kfd_device_queue_manager.h" #include <linux/device.h> /*
@@ -1250,6 +1251,20 @@ void kfd_signal_reset_event(struct kfd_node *dev)continue ; } +if (dev->dqm->detect_hang_count && !p->has_reset_queue) +continue ; + +if (dev->dqm->detect_hang_count) { +struct amdgpu_task_info *ti; + + ti =amdgpu_vm_get_task_info_pasid (dev->adev, p->pasid); +if (ti) { +DRM_ERROR ("Process info: process %s tid %d thread %s pid %d\n" , + ti->process_name, ti->tgid, ti->task_name, ti->pid); +amdgpu_vm_put_task_info (ti); + } + } +rcu_read_lock (); id =KFD_FIRST_NONSIGNAL_EVENT_ID ;
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +5 -1
@@ -321,8 +321,11 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,static bool check_preemption_failed (struct mqd_manager *mm,void *mqd) {struct v9_mqd *m = (struct v9_mqd *)mqd; +uint32_t doorbell_id = m->queue_doorbell_id0; -return kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0,0 ); + m->queue_doorbell_id0 =0 ; + +return kfd_check_hiq_mqd_doorbell_id (mm->dev, doorbell_id,0 ); } static int get_wave_state(struct mqd_manager *mm, void *mqd,
@@ -624,6 +627,7 @@ static bool check_preemption_failed_v9_4_3(struct mqd_manager *mm, void *mqd) m =get_mqd (mqd + hiq_mqd_size * inst); ret |=kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0, inst); + m->queue_doorbell_id0 =0 ; ++inst; }
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4 -0
@@ -504,6 +504,7 @@ struct queue_properties {bool is_being_destroyed;bool is_active;bool is_gws; +bool is_reset;uint32_t pm4_target_xcc;bool is_dbg_wa;bool is_user_cu_masked;
@@ -982,6 +983,9 @@ struct kfd_process {struct semaphore runtime_enable_sema;bool is_runtime_retry;struct kfd_runtime_info runtime_info; + +/* Tracks queue reset status */ +bool has_reset_queue; };#define KFD_PROCESS_TABLE_SIZE 5/* bits: 32 entries */
drivers/gpu/drm/amd/amdkfd/kfd_process.c +2 -0
@@ -1851,6 +1851,8 @@ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger) goto fail; } n_evicted++; + + pdd->dev->dqm->is_hws_hang = false; }return r;
drivers/gpu/drm/amd/include/kgd_kfd_interface.h +4 -0
@@ -318,6 +318,10 @@ struct kfd2kgd_calls {void (*program_trap_handler_settings)(struct amdgpu_device *adev,uint32_t vmid,uint64_t tba_addr,uint64_t tma_addr,uint32_t inst); +uint64_t (*hqd_detect_and_reset)(struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, + bool detect_only); };#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
Signature · 1 line
2.34.1
PATCH 2/2 drm/amdkfd: support the debugger during per-queue reset
In order to allow ROCm GDB to handle reset queues, raise an EC_QUEUE_RESET exception so that the debugger can subscribe and query this exception.
Reset queues should still be considered suspendable with a status flag of KFD_DBG_QUEUE_RESET_MASK.
However they should not be resumable since user space will no longer be able to access reset queues.
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 30 +++++++++++++++---- include/uapi/linux/kfd_ioctl.h | 4 +++ 2 files changed, 29 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +25 -5
@@ -163,6 +163,10 @@ static void kfd_hws_hang(struct device_queue_manager *dqm)list_for_each_entry (q, &qpd->queues_list, list) { q->properties.is_reset = true; q->process->has_reset_queue = true; + +kfd_dbg_ev_raise (KFD_EC_MASK (EC_QUEUE_RESET ), + q->process, q->device, q->doorbell_id, + false,NULL ,0 ); } }
@@ -987,7 +991,7 @@ static int suspend_single_queue(struct device_queue_manager *dqm, {bool is_new; -if (q->properties.is_suspended) +if (q->properties.is_suspended || q->properties.is_reset)return 0 ; pr_debug("Suspending PASID %u queue [%i]\n",
@@ -1008,6 +1012,9 @@ static int suspend_single_queue(struct device_queue_manager *dqm,if (dqm->dev->kfd->shared_resources.enable_mes) {int r =remove_queue_mes (dqm, q, &pdd->qpd); +if (q->properties.is_reset) +return 0 ; +if (r)return r; }
@@ -1971,6 +1978,9 @@ static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q q->properties.is_active = false;decrement_queue_count (dqm, qpd, q); } + +kfd_dbg_ev_raise (KFD_EC_MASK (EC_QUEUE_RESET ), q->process, q->device, + q->doorbell_id, false,NULL ,0 ); }static int detect_queue_hang (struct device_queue_manager *dqm)
@@ -3036,7 +3046,8 @@ int resume_queues(struct kfd_process *p, queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK ; }else { - queue_ids[q_idx] |= + queue_ids[q_idx] |= q->properties.is_reset ? +KFD_DBG_QUEUE_RESET_MASK :KFD_DBG_QUEUE_ERROR_MASK ;break ; }
@@ -3071,7 +3082,7 @@ int resume_queues(struct kfd_process *p, queue_ids);/* mask queue as error on resume fail */ -if (q_idx !=QUEUE_NOT_FOUND ) +if (q_idx !=QUEUE_NOT_FOUND && !q->properties.is_reset) queue_ids[q_idx] |=KFD_DBG_QUEUE_ERROR_MASK ; }
@@ -3118,6 +3129,7 @@ int suspend_queues(struct kfd_process *p,struct qcm_process_device *qpd = &pdd->qpd;struct queue *q;int r, per_device_suspended =0 ; +bool has_queue_reset_fail = false;mutex_lock (&p->event_mutex);dqm_lock (dqm);
@@ -3134,6 +3146,9 @@ int suspend_queues(struct kfd_process *p,if (!err) { queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK ; +if (q->properties.is_reset) + queue_ids[q_idx] |=KFD_DBG_QUEUE_RESET_MASK ; +if (exception_clear_mask && is_mes) q->properties.exception_status &= ~exception_clear_mask;
@@ -3175,13 +3190,18 @@ int suspend_queues(struct kfd_process *p,continue ;/* mask queue as error on suspend fail */ -if (r) +if (r && !q->properties.is_reset) { + has_queue_reset_fail = true; queue_ids[q_idx] |=KFD_DBG_QUEUE_ERROR_MASK ; -else if (exception_clear_mask) + }else if (exception_clear_mask) { q->properties.exception_status &= ~exception_clear_mask; + } } +if (!has_queue_reset_fail) + total_suspended += per_device_suspended; +dqm_unlock (dqm);mutex_unlock (&p->event_mutex);amdgpu_device_flush_hdp (dqm->dev->adev,NULL );
include/uapi/linux/kfd_ioctl.h +4 -0
@@ -875,6 +875,7 @@ enum kfd_dbg_trap_exception_code {EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID =21 ,EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID =22 ,EC_QUEUE_PACKET_VENDOR_UNSUPPORTED =23 , +EC_QUEUE_RESET =29 ,EC_QUEUE_PREEMPTION_ERROR =30 , EC_QUEUE_NEW = 31,/* per device */
@@ -907,6 +908,7 @@ enum kfd_dbg_trap_exception_code { KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID) | \ KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID) | \ KFD_EC_MASK(EC_QUEUE_PACKET_VENDOR_UNSUPPORTED) | \ + KFD_EC_MASK(EC_QUEUE_RESET) | \ KFD_EC_MASK(EC_QUEUE_PREEMPTION_ERROR) | \ KFD_EC_MASK(EC_QUEUE_NEW))#define KFD_EC_MASK_DEVICE (KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE) | \
@@ -997,8 +999,10 @@ struct kfd_queue_snapshot_entry { };/* Queue status return for suspend/resume */ +#define KFD_DBG_QUEUE_RESET_BIT 29#define KFD_DBG_QUEUE_ERROR_BIT 30#define KFD_DBG_QUEUE_INVALID_BIT 31 +#define KFD_DBG_QUEUE_RESET_MASK (1 << KFD_DBG_QUEUE_RESET_BIT)#define KFD_DBG_QUEUE_ERROR_MASK (1 << KFD_DBG_QUEUE_ERROR_BIT)#define KFD_DBG_QUEUE_INVALID_MASK (1 << KFD_DBG_QUEUE_INVALID_BIT)
Signature · 1 line
2.34.1
Felix Kuehling
maintainerQuoted 149 lines · Jonathan Kim
Support per-queue reset for GFX9. The recommendation is for the driver to target reset the HW queue via a SPI MMIO register write.
Since this requires pipe and HW queue info and MEC FW is limited to doorbell reports of hung queues after an unmap failure, scan the HW queue slots defined by
SET_RESOURCESfirst to identify the user queue candidates to reset.Only signal reset events to processes that have had a queue reset.
If queue reset fails, fall back to GPU reset.
Signed-off-by Jonathan Kim <[email protected]>--- .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 1 + .../drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c | 9 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h | 6 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 11 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 56 +++++ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 6 + .../drm/amd/amdkfd/kfd_device_queue_manager.c | 195 ++++++++++++++++-- .../drm/amd/amdkfd/kfd_device_queue_manager.h | 12 ++ drivers/gpu/drm/amd/amdkfd/kfd_events.c | 15 ++ .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 6 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 2 + .../gpu/drm/amd/include/kgd_kfd_interface.h | 4 + 16 files changed, 310 insertions(+), 26 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +1
-0@@ -191,4 +191,5 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +2
-1@@ -418,5 +418,6 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, - .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings + .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c +2
-1@@ -541,5 +541,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { kgd_gfx_v9_4_3_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_aldebaran_set_wave_launch_mode, .set_address_watch = kgd_gfx_v9_4_3_set_address_watch, - .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch + .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +9
-0@@ -1070,6 +1070,14 @@ static void program_trap_handler_settings(struct amdgpu_device *adev,unlock_srbm (adev); } +uint64_t kgd_gfx_v10_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only) +{ +return 0 ; +} +const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .program_sh_mem_settings = kgd_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,@@ -1097,4 +1105,5 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info, .program_trap_handler_settings = program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +6
-0@@ -56,3 +56,9 @@ void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v10_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +2
-1@@ -680,5 +680,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .set_wave_launch_trap_override = kgd_gfx_v10_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_gfx_v10_set_wave_launch_mode, .set_address_watch = kgd_gfx_v10_set_address_watch, - .clear_address_watch = kgd_gfx_v10_clear_address_watch + .clear_address_watch = kgd_gfx_v10_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +10
-1@@ -786,6 +786,14 @@ static uint32_t kgd_gfx_v11_clear_address_watch(struct amdgpu_device *adev,return 0 ; } +static uint64_t kgd_gfx_v11_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only) +{ +return 0 ; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,@@ -808,5 +816,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .set_wave_launch_trap_override = kgd_gfx_v11_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_gfx_v11_set_wave_launch_mode, .set_address_watch = kgd_gfx_v11_set_address_watch, - .clear_address_watch = kgd_gfx_v11_clear_address_watch + .clear_address_watch = kgd_gfx_v11_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v11_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev, · line 1150
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +4
-0@@ -1144,6 +1144,61 @@ void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,kgd_gfx_v9_unlock_srbm (adev, inst); } +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only)
This is a bit misleading. This function doesn't detect anything. It just returns a queue address and optionally resets the queue. And the way it's called is like "detect and reset, but don't reset", except that last part is not obvious if you don't know what the last parameter means. It would make the code easier to read if you split this into two functions: hqd_get_pq_addr and hqd_reset.
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev, · line 1161
Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +11
-0+{ +uint32_t low, high, temp; +unsigned long end_jiffies; +uint64_t queue_addr =0 ; + +kgd_gfx_v9_acquire_queue (adev, pipe_id, queue_id, inst); +if (!RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_ACTIVE)) + goto unlock_out; + + low =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_PQ_BASE); + high =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_PQ_BASE_HI);
You may need to disable GFXOFF before accessing these registers. Maybe you're assuming that the GPU is not idle when you're getting here, and maybe that's a good assumption, so I'm not sure.
Quoted 71 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +41
-0+ +/* only concerned with user queues. */ +if (!high) + goto unlock_out; + + queue_addr = (((queue_addr | high) <<32 ) | low) <<8 ; + +if (detect_only) + goto unlock_out; + +pr_debug ("Attempting queue reset on XCC %i pipe id %i queue id %i\n" , + inst, pipe_id, queue_id); + +/* assume previous dequeue request issued will take affect after reset */ +WREG32_SOC15 (GC ,GET_INST (GC , inst), mmSPI_COMPUTE_QUEUE_RESET,0x1 ); + + end_jiffies = (utimeout *HZ /1000 ) + jiffies; +while (true) { + temp =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_ACTIVE); + +if (!(temp &CP_HQD_ACTIVE__ACTIVE_MASK )) +break ; + +if (time_after (jiffies, end_jiffies)) { + queue_addr =0 ; +break ; + } + +usleep_range (500 ,1000 ); + } + +unlock_out: +if (!detect_only) +pr_debug ("queue reset on XCC %i pipe id %i queue id %i %s\n" , + inst, pipe_id, queue_id, !!queue_addr ?"succeeded!" :"failed!" ); +kgd_gfx_v9_release_queue (adev, inst); + +return queue_addr; +} +const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping,@@ -1172,4 +1227,5 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +6
-0@@ -101,3 +101,9 @@ void kgd_gfx_v9_build_grace_period_packet_info(struct amdgpu_device *adev,uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-0@@ -153,6 +153,19 @@ void program_sh_mem_settings(struct device_queue_manager *dqm,static void kfd_hws_hang (struct device_queue_manager *dqm) {
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c void program_sh_mem_settings(struct device_queue_manager *dqm, · line 165
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+struct device_process_node *cur; +struct qcm_process_device *qpd; +struct queue *q; + +/* Mark all device queues as reset. */ +list_for_each_entry (cur, &dqm->queues, list) { + qpd = cur->qpd; +list_for_each_entry (q, &qpd->queues_list, list) { + q->properties.is_reset = true; + q->process->has_reset_queue = true;
Both seem to track the same information, but one does it per queue, the other does it per process. Why do you need to track this per queue at all? The only place where I see q->properties->is_reset used is in update_queue. I think you can use the per-process tracking there.
That said, I think it would be better to track has_reset_queue in the pdd or qpd because GPU resets are per GPU, and these fields are accessed under the per-GPU DQM lock and reset events sent to user mode are per GPU as well.
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int update_queue(struct device_queue_manager *dqm, struct queue *q, · line 897
Quoted 12 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +5
-0+ } + } +/* * Issue a GPU reset if HWS is unresponsive */ @@ -880,6 +893,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q,else if (prev_active) retval =remove_queue_mes (dqm, q, &pdd->qpd); +/* queue is reset so inaccessable */ +if (q->properties.is_reset) {
You could use pdd->process->has_reset_queue (or pdd->has_reset_queue) here and get rid of is_reset.
Quoted 123 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +94
-1+ retval = -EACCES ; + goto out_unlock; + } +if (retval) {dev_err (dev,"unmap queue failed\n" ); goto out_unlock;@@ -1629,7 +1648,7 @@ static int initialize_cpsch(struct device_queue_manager *dqm)static int start_cpsch (struct device_queue_manager *dqm) {struct device *dev = dqm->dev->adev->dev; -int retval; +int retval, num_hw_queue_slots; retval =0 ;@@ -1682,6 +1701,14 @@ static int start_cpsch(struct device_queue_manager *dqm) &dqm->wait_times); } +/* setup per-queue reset detection buffer */ + num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * + dqm->dev->kfd->shared_resources.num_pipe_per_mec * +NUM_XCC (dqm->dev->xcc_mask); + + dqm->detect_hang_info_size = num_hw_queue_slots *sizeof (struct dqm_detect_hang_info ); + dqm->detect_hang_info =kzalloc (dqm->detect_hang_info_size,GFP_KERNEL ); +dqm_unlock (dqm);return 0 ;@@ -1715,6 +1742,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)kfd_gtt_sa_free (dqm->dev, dqm->fence_mem);if (!dqm->dev->kfd->shared_resources.enable_mes)pm_uninit (&dqm->packet_mgr); +kfree (dqm->detect_hang_info);dqm_unlock (dqm);return 0 ;@@ -1931,6 +1959,143 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)return retval; } +static void set_queue_as_reset (struct device_queue_manager *dqm,struct queue *q, +struct qcm_process_device *qpd) +{ +pr_err ("queue id 0x%0x at pasid 0x%0x is reset\n" , + q->properties.queue_id, q->process->pasid); + + q->properties.is_reset = true; + q->process->has_reset_queue = true; +if (q->properties.is_active) { + q->properties.is_active = false; +decrement_queue_count (dqm, qpd, q); + } +} + +static int detect_queue_hang (struct device_queue_manager *dqm) +{ +int i; + +memset (dqm->detect_hang_info,0 , dqm->detect_hang_info_size); + +for (i =0 ; i <AMDGPU_MAX_QUEUES ; ++i) { +uint32_t mec, pipe, queue; +int xcc_id; + + mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) + / dqm->dev->kfd->shared_resources.num_pipe_per_mec; + +if (mec || !test_bit (i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) +continue ; + +amdgpu_queue_mask_bit_to_mec_queue (dqm->dev->adev, i, &mec, &pipe, &queue); + +for_each_inst (xcc_id, dqm->dev->xcc_mask) { +uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_detect_and_reset ( + dqm->dev->adev, pipe, queue, xcc_id, +0 , true); +struct dqm_detect_hang_info hang_info; + +if (!queue_addr) +continue ; + + hang_info.pipe_id = pipe; + hang_info.queue_id = queue; + hang_info.xcc_id = xcc_id; + hang_info.queue_address = queue_addr; + + dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; + dqm->detect_hang_count++; + } + } + +return dqm->detect_hang_count; +} + +static struct queue *find_queue_by_address (struct device_queue_manager *dqm,uint64_t queue_address) +{ +struct device_process_node *cur; +struct qcm_process_device *qpd; +struct queue *q; + +list_for_each_entry (cur, &dqm->queues, list) { + qpd = cur->qpd; +list_for_each_entry (q, &qpd->queues_list, list) { +if (queue_address == q->properties.queue_address) +return q; + } + } + +return NULL ; +} + +/* only for compute queue */ +static int reset_queues_cpsch (struct device_queue_manager *dqm,uint16_t pasid) +{ +int retval =0 , reset_count =0 , i; + +if (pasid) { +dqm_lock (dqm); + + retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) · line 2051
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+ pasid, USE_DEFAULT_GRACE_PERIOD, true); + +dqm_unlock (dqm); +return retval; + } + +if (!dqm->detect_hang_info || dqm->is_hws_hang) +return -EIO ; + +/* assume dqm locked. */
This is weird, the same function does different things and has different assumptions about its locking context depending where it's called from. Maybe keep the original function and and make this a new function with a distinct name.
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) · line 2062
Quoted 11 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +11
-0+if (!detect_queue_hang (dqm)) +return -ENOTRECOVERABLE ; + +for (i =0 ; i < dqm->detect_hang_count; i++) { +struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; +struct queue *q =find_queue_by_address (dqm, hang_info.queue_address); +struct kfd_process_device *pdd; +uint64_t queue_addr =0 ; +int pipe = hang_info.pipe_id; +int queue = hang_info.queue_id; +int xcc_id = hang_info.xcc_id;
Why do you need these local variables. hang_info is already a local variable. Now you have two copies of this on your stack.
Regards,
Felix
Quoted 219 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +44
-20+ +if (!q) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + + pdd =kfd_get_process_device_data (dqm->dev, q->process); +if (!pdd) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + + queue_addr = dqm->dev->kfd2kgd->hqd_detect_and_reset ( + dqm->dev->adev, pipe, queue, xcc_id, +KFD_UNMAP_LATENCY_MS , false); + +if (queue_addr != q->properties.queue_address) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + +set_queue_as_reset (dqm, q, &pdd->qpd); + reset_count++; + } + +if (reset_count == dqm->detect_hang_count) +kfd_signal_reset_event (dqm->dev); +else + retval = -ENOTRECOVERABLE ; + +reset_fail: + dqm->detect_hang_count =0 ; + +return retval; +} +/* dqm->lock mutex has to be locked before calling this function */ static int unmap_queues_cpsch(struct device_queue_manager *dqm, enum kfd_unmap_queues_filter filter,@@ -1981,11 +2146,14 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,*/ mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ ];if (mqd_mgr->check_preemption_failed (mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd)) { -while (halt_if_hws_hang) -schedule (); -kfd_hws_hang (dqm); - retval = -ETIME ; - goto out; +if (reset_queues_cpsch (dqm,0 )) { +while (halt_if_hws_hang) +schedule (); + dqm->is_hws_hang = true; +kfd_hws_hang (dqm); + retval = -ETIME ; + goto out; + } }/* We need to reset the grace period value for this device */ @@ -2003,21 +2171,6 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,return retval; } -/* only for compute queue */ -static int reset_queues_cpsch (struct device_queue_manager *dqm, -uint16_t pasid) -{ -int retval; - -dqm_lock (dqm); - - retval =unmap_queues_cpsch (dqm,KFD_UNMAP_QUEUES_FILTER_BY_PASID , - pasid,USE_DEFAULT_GRACE_PERIOD , true); - -dqm_unlock (dqm); -return retval; -} -/* dqm->lock mutex has to be locked before calling this function */ static int execute_queues_cpsch(struct device_queue_manager *dqm, enum kfd_unmap_queues_filter filter,drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +12
-0@@ -210,6 +210,13 @@ struct device_queue_manager_asic_ops { struct kfd_node *dev); }; +struct dqm_detect_hang_info { +int pipe_id; +int queue_id; +int xcc_id; +uint64_t queue_address; +}; +/** * struct device_queue_manager * @@ -264,6 +271,11 @@ struct device_queue_manager {uint32_t wait_times;wait_queue_head_t destroy_wait; + +/* for per-queue reset support */ +struct dqm_detect_hang_info *detect_hang_info; +size_t detect_hang_info_size; +int detect_hang_count; }; void device_queue_manager_init_cik(drivers/gpu/drm/amd/amdkfd/kfd_events.c +15
-0@@ -31,6 +31,7 @@#include <linux/memory.h> #include "kfd_priv.h" #include "kfd_events.h" +#include "kfd_device_queue_manager.h" #include <linux/device.h> /* @@ -1250,6 +1251,20 @@ void kfd_signal_reset_event(struct kfd_node *dev)continue ; } +if (dev->dqm->detect_hang_count && !p->has_reset_queue) +continue ; + +if (dev->dqm->detect_hang_count) { +struct amdgpu_task_info *ti; + + ti =amdgpu_vm_get_task_info_pasid (dev->adev, p->pasid); +if (ti) { +DRM_ERROR ("Process info: process %s tid %d thread %s pid %d\n" , + ti->process_name, ti->tgid, ti->task_name, ti->pid); +amdgpu_vm_put_task_info (ti); + } + } +rcu_read_lock (); id =KFD_FIRST_NONSIGNAL_EVENT_ID ;drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +5
-1@@ -321,8 +321,11 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,static bool check_preemption_failed (struct mqd_manager *mm,void *mqd) {struct v9_mqd *m = (struct v9_mqd *)mqd; +uint32_t doorbell_id = m->queue_doorbell_id0; -return kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0,0 ); + m->queue_doorbell_id0 =0 ; + +return kfd_check_hiq_mqd_doorbell_id (mm->dev, doorbell_id,0 ); } static int get_wave_state(struct mqd_manager *mm, void *mqd,@@ -624,6 +627,7 @@ static bool check_preemption_failed_v9_4_3(struct mqd_manager *mm, void *mqd) m =get_mqd (mqd + hiq_mqd_size * inst); ret |=kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0, inst); + m->queue_doorbell_id0 =0 ; ++inst; }drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4
-0@@ -504,6 +504,7 @@ struct queue_properties {bool is_being_destroyed;bool is_active;bool is_gws; +bool is_reset;uint32_t pm4_target_xcc;bool is_dbg_wa;bool is_user_cu_masked;@@ -982,6 +983,9 @@ struct kfd_process {struct semaphore runtime_enable_sema;bool is_runtime_retry;struct kfd_runtime_info runtime_info; + +/* Tracks queue reset status */ +bool has_reset_queue; };#define KFD_PROCESS_TABLE_SIZE 5/* bits: 32 entries */ drivers/gpu/drm/amd/amdkfd/kfd_process.c +2
-0@@ -1851,6 +1851,8 @@ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger) goto fail; } n_evicted++; + + pdd->dev->dqm->is_hws_hang = false; }return r;drivers/gpu/drm/amd/include/kgd_kfd_interface.h +4
-0@@ -318,6 +318,10 @@ struct kfd2kgd_calls {void (*program_trap_handler_settings)(struct amdgpu_device *adev,uint32_t vmid,uint64_t tba_addr,uint64_t tma_addr,uint32_t inst); +uint64_t (*hqd_detect_and_reset)(struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, + bool detect_only); };#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
Quoted 168 lines · Felix Kuehling
Quoted 149 lines · Jonathan Kim
Support per-queue reset for GFX9. The recommendation is for the driver to target reset the HW queue via a SPI MMIO register write.
Since this requires pipe and HW queue info and MEC FW is limited to doorbell reports of hung queues after an unmap failure, scan the HW queue slots defined by
SET_RESOURCESfirst to identify the user queue candidates to reset.Only signal reset events to processes that have had a queue reset.
If queue reset fails, fall back to GPU reset.
Signed-off-by Jonathan Kim <[email protected]>--- .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 1 + .../drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c | 9 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h | 6 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 11 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 56 +++++ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 6 + .../drm/amd/amdkfd/kfd_device_queue_manager.c | 195 ++++++++++++++++-- .../drm/amd/amdkfd/kfd_device_queue_manager.h | 12 ++ drivers/gpu/drm/amd/amdkfd/kfd_events.c | 15 ++ .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 6 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 2 + .../gpu/drm/amd/include/kgd_kfd_interface.h | 4 + 16 files changed, 310 insertions(+), 26 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +1
-0@@ -191,4 +191,5 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +2
-1@@ -418,5 +418,6 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, - .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings + .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c +2
-1@@ -541,5 +541,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { kgd_gfx_v9_4_3_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_aldebaran_set_wave_launch_mode, .set_address_watch = kgd_gfx_v9_4_3_set_address_watch, - .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch + .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +9
-0@@ -1070,6 +1070,14 @@ static void program_trap_handler_settings(struct amdgpu_device *adev,unlock_srbm (adev); } +uint64_t kgd_gfx_v10_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only) +{ +return 0 ; +} +const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .program_sh_mem_settings = kgd_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,@@ -1097,4 +1105,5 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times, .build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info, .program_trap_handler_settings = program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +6
-0@@ -56,3 +56,9 @@ void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v10_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +2
-1@@ -680,5 +680,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .set_wave_launch_trap_override = kgd_gfx_v10_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_gfx_v10_set_wave_launch_mode, .set_address_watch = kgd_gfx_v10_set_address_watch, - .clear_address_watch = kgd_gfx_v10_clear_address_watch + .clear_address_watch = kgd_gfx_v10_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +10
-1@@ -786,6 +786,14 @@ static uint32_t kgd_gfx_v11_clear_address_watch(struct amdgpu_device *adev,return 0 ; } +static uint64_t kgd_gfx_v11_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only) +{ +return 0 ; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,@@ -808,5 +816,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .set_wave_launch_trap_override = kgd_gfx_v11_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_gfx_v11_set_wave_launch_mode, .set_address_watch = kgd_gfx_v11_set_address_watch, - .clear_address_watch = kgd_gfx_v11_clear_address_watch + .clear_address_watch = kgd_gfx_v11_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v11_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev, · line 1150
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +4
-0@@ -1144,6 +1144,61 @@ void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,kgd_gfx_v9_unlock_srbm (adev, inst); } +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only)This is a bit misleading. This function doesn't detect anything. It just returns a queue address and optionally resets the queue. And the way it's called is like "detect and reset, but don't reset", except that last part is not obvious if you don't know what the last parameter means. It would make the code easier to read if you split this into two
functions:hqd_get_pq_addrandhqd_reset.
Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev, · line 1161
Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +11
-0+{ +uint32_t low, high, temp; +unsigned long end_jiffies; +uint64_t queue_addr =0 ; + +kgd_gfx_v9_acquire_queue (adev, pipe_id, queue_id, inst); +if (!RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_ACTIVE)) + goto unlock_out; + + low =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_PQ_BASE); + high =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_PQ_BASE_HI);You may need to disable GFXOFF before accessing these registers. Maybe you're assuming that the GPU is not idle when you're getting here, and maybe that's a good assumption, so I'm not sure.
I think you should wrap any reset MMIO in amdgpu_gfx_rlc_enter_safe_mode() and amdgpu_gfx_rlc_exit_safe_mode().
Alex
Quoted 490 lines
Quoted 73 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +41
-0+ +/* only concerned with user queues. */ +if (!high) + goto unlock_out; + + queue_addr = (((queue_addr | high) <<32 ) | low) <<8 ; + +if (detect_only) + goto unlock_out; + +pr_debug ("Attempting queue reset on XCC %i pipe id %i queue id %i\n" , + inst, pipe_id, queue_id); + +/* assume previous dequeue request issued will take affect after reset */ +WREG32_SOC15 (GC ,GET_INST (GC , inst), mmSPI_COMPUTE_QUEUE_RESET,0x1 ); + + end_jiffies = (utimeout *HZ /1000 ) + jiffies; +while (true) { + temp =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_ACTIVE); + +if (!(temp &CP_HQD_ACTIVE__ACTIVE_MASK )) +break ; + +if (time_after (jiffies, end_jiffies)) { + queue_addr =0 ; +break ; + } + +usleep_range (500 ,1000 ); + } + +unlock_out: +if (!detect_only) +pr_debug ("queue reset on XCC %i pipe id %i queue id %i %s\n" , + inst, pipe_id, queue_id, !!queue_addr ?"succeeded!" :"failed!" ); +kgd_gfx_v9_release_queue (adev, inst); + +return queue_addr; +} +const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping,@@ -1172,4 +1227,5 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info, .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +6
-0@@ -101,3 +101,9 @@ void kgd_gfx_v9_build_grace_period_packet_info(struct amdgpu_device *adev,uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-0@@ -153,6 +153,19 @@ void program_sh_mem_settings(struct device_queue_manager *dqm,static void kfd_hws_hang (struct device_queue_manager *dqm) {drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c void program_sh_mem_settings(struct device_queue_manager *dqm, · line 165
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+struct device_process_node *cur; +struct qcm_process_device *qpd; +struct queue *q; + +/* Mark all device queues as reset. */ +list_for_each_entry (cur, &dqm->queues, list) { + qpd = cur->qpd; +list_for_each_entry (q, &qpd->queues_list, list) { + q->properties.is_reset = true; + q->process->has_reset_queue = true;Both seem to track the same information, but one does it per queue, the other does it per process. Why do you need to track this per queue at all? The only place where I see
q->properties->is_resetused is inupdate_queue. I think you can use the per-process tracking there.That said, I think it would be better to track
has_reset_queuein the pdd or qpd because GPU resets are per GPU, and these fields are accessed under the per-GPU DQM lock and reset events sent to user mode are per GPU as well.drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int update_queue(struct device_queue_manager *dqm, struct queue *q, · line 897
Quoted 12 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +5
-0+ } + } +/* * Issue a GPU reset if HWS is unresponsive */ @@ -880,6 +893,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q,else if (prev_active) retval =remove_queue_mes (dqm, q, &pdd->qpd); +/* queue is reset so inaccessable */ +if (q->properties.is_reset) {You could use
pdd->process->has_reset_queue(orpdd->has_reset_queue) here and get rid ofis_reset.Quoted 123 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +94
-1+ retval = -EACCES ; + goto out_unlock; + } +if (retval) {dev_err (dev,"unmap queue failed\n" ); goto out_unlock;@@ -1629,7 +1648,7 @@ static int initialize_cpsch(struct device_queue_manager *dqm)static int start_cpsch (struct device_queue_manager *dqm) {struct device *dev = dqm->dev->adev->dev; -int retval; +int retval, num_hw_queue_slots; retval =0 ;@@ -1682,6 +1701,14 @@ static int start_cpsch(struct device_queue_manager *dqm) &dqm->wait_times); } +/* setup per-queue reset detection buffer */ + num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe * + dqm->dev->kfd->shared_resources.num_pipe_per_mec * +NUM_XCC (dqm->dev->xcc_mask); + + dqm->detect_hang_info_size = num_hw_queue_slots *sizeof (struct dqm_detect_hang_info ); + dqm->detect_hang_info =kzalloc (dqm->detect_hang_info_size,GFP_KERNEL ); +dqm_unlock (dqm);return 0 ;@@ -1715,6 +1742,7 @@ static int stop_cpsch(struct device_queue_manager *dqm)kfd_gtt_sa_free (dqm->dev, dqm->fence_mem);if (!dqm->dev->kfd->shared_resources.enable_mes)pm_uninit (&dqm->packet_mgr); +kfree (dqm->detect_hang_info);dqm_unlock (dqm);return 0 ;@@ -1931,6 +1959,143 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)return retval; } +static void set_queue_as_reset (struct device_queue_manager *dqm,struct queue *q, +struct qcm_process_device *qpd) +{ +pr_err ("queue id 0x%0x at pasid 0x%0x is reset\n" , + q->properties.queue_id, q->process->pasid); + + q->properties.is_reset = true; + q->process->has_reset_queue = true; +if (q->properties.is_active) { + q->properties.is_active = false; +decrement_queue_count (dqm, qpd, q); + } +} + +static int detect_queue_hang (struct device_queue_manager *dqm) +{ +int i; + +memset (dqm->detect_hang_info,0 , dqm->detect_hang_info_size); + +for (i =0 ; i <AMDGPU_MAX_QUEUES ; ++i) { +uint32_t mec, pipe, queue; +int xcc_id; + + mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) + / dqm->dev->kfd->shared_resources.num_pipe_per_mec; + +if (mec || !test_bit (i, dqm->dev->kfd->shared_resources.cp_queue_bitmap)) +continue ; + +amdgpu_queue_mask_bit_to_mec_queue (dqm->dev->adev, i, &mec, &pipe, &queue); + +for_each_inst (xcc_id, dqm->dev->xcc_mask) { +uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_detect_and_reset ( + dqm->dev->adev, pipe, queue, xcc_id, +0 , true); +struct dqm_detect_hang_info hang_info; + +if (!queue_addr) +continue ; + + hang_info.pipe_id = pipe; + hang_info.queue_id = queue; + hang_info.xcc_id = xcc_id; + hang_info.queue_address = queue_addr; + + dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; + dqm->detect_hang_count++; + } + } + +return dqm->detect_hang_count; +} + +static struct queue *find_queue_by_address (struct device_queue_manager *dqm,uint64_t queue_address) +{ +struct device_process_node *cur; +struct qcm_process_device *qpd; +struct queue *q; + +list_for_each_entry (cur, &dqm->queues, list) { + qpd = cur->qpd; +list_for_each_entry (q, &qpd->queues_list, list) { +if (queue_address == q->properties.queue_address) +return q; + } + } + +return NULL ; +} + +/* only for compute queue */ +static int reset_queues_cpsch (struct device_queue_manager *dqm,uint16_t pasid) +{ +int retval =0 , reset_count =0 , i; + +if (pasid) { +dqm_lock (dqm); + + retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) · line 2051
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+ pasid, USE_DEFAULT_GRACE_PERIOD, true); + +dqm_unlock (dqm); +return retval; + } + +if (!dqm->detect_hang_info || dqm->is_hws_hang) +return -EIO ; + +/* assume dqm locked. */ This is weird, the same function does different things and has different assumptions about its locking context depending where it's called from. Maybe keep the original function and and make this a new function with a distinct name.
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) · line 2062
Quoted 11 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +11
-0+if (!detect_queue_hang (dqm)) +return -ENOTRECOVERABLE ; + +for (i =0 ; i < dqm->detect_hang_count; i++) { +struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i]; +struct queue *q =find_queue_by_address (dqm, hang_info.queue_address); +struct kfd_process_device *pdd; +uint64_t queue_addr =0 ; +int pipe = hang_info.pipe_id; +int queue = hang_info.queue_id; +int xcc_id = hang_info.xcc_id;Why do you need these local variables.
hang_infois already a local variable. Now you have two copies of this on your stack.Regards,
FelixQuoted 219 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +44
-20+ +if (!q) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + + pdd =kfd_get_process_device_data (dqm->dev, q->process); +if (!pdd) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + + queue_addr = dqm->dev->kfd2kgd->hqd_detect_and_reset ( + dqm->dev->adev, pipe, queue, xcc_id, +KFD_UNMAP_LATENCY_MS , false); + +if (queue_addr != q->properties.queue_address) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + +set_queue_as_reset (dqm, q, &pdd->qpd); + reset_count++; + } + +if (reset_count == dqm->detect_hang_count) +kfd_signal_reset_event (dqm->dev); +else + retval = -ENOTRECOVERABLE ; + +reset_fail: + dqm->detect_hang_count =0 ; + +return retval; +} +/* dqm->lock mutex has to be locked before calling this function */ static int unmap_queues_cpsch(struct device_queue_manager *dqm, enum kfd_unmap_queues_filter filter,@@ -1981,11 +2146,14 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,*/ mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ ];if (mqd_mgr->check_preemption_failed (mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd)) { -while (halt_if_hws_hang) -schedule (); -kfd_hws_hang (dqm); - retval = -ETIME ; - goto out; +if (reset_queues_cpsch (dqm,0 )) { +while (halt_if_hws_hang) +schedule (); + dqm->is_hws_hang = true; +kfd_hws_hang (dqm); + retval = -ETIME ; + goto out; + } }/* We need to reset the grace period value for this device */ @@ -2003,21 +2171,6 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,return retval; } -/* only for compute queue */ -static int reset_queues_cpsch (struct device_queue_manager *dqm, -uint16_t pasid) -{ -int retval; - -dqm_lock (dqm); - - retval =unmap_queues_cpsch (dqm,KFD_UNMAP_QUEUES_FILTER_BY_PASID , - pasid,USE_DEFAULT_GRACE_PERIOD , true); - -dqm_unlock (dqm); -return retval; -} -/* dqm->lock mutex has to be locked before calling this function */ static int execute_queues_cpsch(struct device_queue_manager *dqm, enum kfd_unmap_queues_filter filter,drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +12
-0@@ -210,6 +210,13 @@ struct device_queue_manager_asic_ops { struct kfd_node *dev); }; +struct dqm_detect_hang_info { +int pipe_id; +int queue_id; +int xcc_id; +uint64_t queue_address; +}; +/** * struct device_queue_manager * @@ -264,6 +271,11 @@ struct device_queue_manager {uint32_t wait_times;wait_queue_head_t destroy_wait; + +/* for per-queue reset support */ +struct dqm_detect_hang_info *detect_hang_info; +size_t detect_hang_info_size; +int detect_hang_count; }; void device_queue_manager_init_cik(drivers/gpu/drm/amd/amdkfd/kfd_events.c +15
-0@@ -31,6 +31,7 @@#include <linux/memory.h> #include "kfd_priv.h" #include "kfd_events.h" +#include "kfd_device_queue_manager.h" #include <linux/device.h> /* @@ -1250,6 +1251,20 @@ void kfd_signal_reset_event(struct kfd_node *dev)continue ; } +if (dev->dqm->detect_hang_count && !p->has_reset_queue) +continue ; + +if (dev->dqm->detect_hang_count) { +struct amdgpu_task_info *ti; + + ti =amdgpu_vm_get_task_info_pasid (dev->adev, p->pasid); +if (ti) { +DRM_ERROR ("Process info: process %s tid %d thread %s pid %d\n" , + ti->process_name, ti->tgid, ti->task_name, ti->pid); +amdgpu_vm_put_task_info (ti); + } + } +rcu_read_lock (); id =KFD_FIRST_NONSIGNAL_EVENT_ID ;drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +5
-1@@ -321,8 +321,11 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,static bool check_preemption_failed (struct mqd_manager *mm,void *mqd) {struct v9_mqd *m = (struct v9_mqd *)mqd; +uint32_t doorbell_id = m->queue_doorbell_id0; -return kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0,0 ); + m->queue_doorbell_id0 =0 ; + +return kfd_check_hiq_mqd_doorbell_id (mm->dev, doorbell_id,0 ); } static int get_wave_state(struct mqd_manager *mm, void *mqd,@@ -624,6 +627,7 @@ static bool check_preemption_failed_v9_4_3(struct mqd_manager *mm, void *mqd) m =get_mqd (mqd + hiq_mqd_size * inst); ret |=kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0, inst); + m->queue_doorbell_id0 =0 ; ++inst; }drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4
-0@@ -504,6 +504,7 @@ struct queue_properties {bool is_being_destroyed;bool is_active;bool is_gws; +bool is_reset;uint32_t pm4_target_xcc;bool is_dbg_wa;bool is_user_cu_masked;@@ -982,6 +983,9 @@ struct kfd_process {struct semaphore runtime_enable_sema;bool is_runtime_retry;struct kfd_runtime_info runtime_info; + +/* Tracks queue reset status */ +bool has_reset_queue; };#define KFD_PROCESS_TABLE_SIZE 5/* bits: 32 entries */ drivers/gpu/drm/amd/amdkfd/kfd_process.c +2
-0@@ -1851,6 +1851,8 @@ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger) goto fail; } n_evicted++; + + pdd->dev->dqm->is_hws_hang = false; }return r;drivers/gpu/drm/amd/include/kgd_kfd_interface.h +4
-0@@ -318,6 +318,10 @@ struct kfd2kgd_calls {void (*program_trap_handler_settings)(struct amdgpu_device *adev,uint32_t vmid,uint64_t tba_addr,uint64_t tma_addr,uint32_t inst); +uint64_t (*hqd_detect_and_reset)(struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, + bool detect_only); };#endif /* KGD_KFD_INTERFACE_H_INCLUDED */
[Public]
Quoted 193 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Felix Kuehling
Sent: Wednesday, July 24, 2024 4:35 PM
To: Kim, Jonathan <[email protected]>; [email protected]
Subject: Re: [PATCH 1/2] drm/amdkfd: support per-queue reset on gfx9Caution: This message originated from an External Source. Use proper caution when opening attachments, clicking links, or responding.
Quoted 13 lines · Jonathan Kim
Support per-queue reset for GFX9. The recommendation is for the driver to target reset the HW queue via a SPI MMIO register write.
Since this requires pipe and HW queue info and MEC FW is limited to doorbell reports of hung queues after an unmap failure, scan the HW queue slots defined by
SET_RESOURCESfirst to identify the user queue candidates to reset.Only signal reset events to processes that have had a queue reset.
If queue reset fails, fall back to GPU reset.
Signed-off-by Jonathan Kim <[email protected]>Quoted 11 lines
--- .../drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c | 1 + .../drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c | 9 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h | 6 + .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c | 3 +- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 11 +- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 56 +++++ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 6 + .../drm/amd/amdkfd/kfd_device_queue_manager.c | 195++++++++++++++++--
Review
Quoted 9 lines
.../drm/amd/amdkfd/kfd_device_queue_manager.h | 12 ++ drivers/gpu/drm/amd/amdkfd/kfd_events.c | 15 ++ .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 6 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 2 + .../gpu/drm/amd/include/kgd_kfd_interface.h | 4 + 16 files changed, 310 insertions(+), 26 deletions(-)(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.cdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c const struct kfd2kgd_calls aldebaran_kfd2kgd = { · line 192
Quoted 6 lines
index
aff08321e976..1dc601e4518a100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +0
-0@@ -191,4 +191,5 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info =
kgd_gfx_v9_build_grace_period_packet_info,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c const struct kfd2kgd_calls aldebaran_kfd2kgd = { · line 193
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +0
-0.program_trap_handler_settings =
kgd_gfx_v9_program_trap_handler_settings,Review
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c +1
-0+ .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.cdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c const struct kfd2kgd_calls arcturus_kfd2kgd = { · line 419
Quoted 6 lines
index
3a3f3ce09f00..534975c722df100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +0
-0@@ -418,5 +418,6 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times, .build_grace_period_packet_info =
kgd_gfx_v9_build_grace_period_packet_info,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c const struct kfd2kgd_calls arcturus_kfd2kgd = { · line 421
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +0
-1.get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, - .program_trap_handler_settings =
kgd_gfx_v9_program_trap_handler_settingsdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c const struct kfd2kgd_calls arcturus_kfd2kgd = { · line 421
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +1
-0+ .program_trap_handler_settings =
kgd_gfx_v9_program_trap_handler_settings,Review
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +1
-0+ .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.cReview
Quoted 12 lines
index
a5c7259cf2a3..b53c1cfa34de100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c +2
-1@@ -541,5 +541,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { kgd_gfx_v9_4_3_set_wave_launch_trap_override, .set_wave_launch_mode = kgd_aldebaran_set_wave_launch_mode, .set_address_watch = kgd_gfx_v9_4_3_set_address_watch, - .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch + .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.cdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c static void
Quoted 4 lines
index
3ab6c3aa0ad1..dd449a0caba8100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +0
-0@@ -1070,6 +1070,14 @@ static voidprogram_trap_handler_settings(struct
amdgpu_device*adev,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c static void program_trap_handler_settings(struct amdgpu_device *adev,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +1
-0unlock_srbm (adev); } +uint64_t kgd_gfx_v10_hqd_detect_and_reset(struct amdgpu_device*adev,
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c const struct kfd2kgd_calls gfx_v10_kfd2kgd = { · line 1106
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +7
-0+ uint32_t pipe_id, uint32_t queue_id, + uint32_t inst, unsigned int utimeout, + bool detect_only) +{ + return 0; +} +const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .program_sh_mem_settings = kgd_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,@@ -1097,4 +1105,5 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times, .build_grace_period_packet_info =
kgd_gfx_v10_build_grace_period_packet_info,Review
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +1
-0.program_trap_handler_settings = program_trap_handler_settings, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.hdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h void
Quoted 4 lines
index
67bcaa3d4226..9eeda8808a3c100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +0
-0@@ -56,3 +56,9 @@ voidkgd_gfx_v10_build_grace_period_packet_info(struct
amdgpu_device*adev,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +1
-0uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v10_hqd_detect_and_reset(struct amdgpu_device*adev,
Review
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h +5
-0+uint32_t pipe_id, + uint32_t queue_id, + uint32_t inst, + unsigned int utimeout, + bool detect_only);(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.cdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { · line 680
Quoted 5 lines
index
8c8437a4383f..9872bca4156b100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +0
-0@@ -680,5 +680,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .set_wave_launch_trap_override =
kgd_gfx_v10_set_wave_launch_trap_override,Review
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +2
-1.set_wave_launch_mode = kgd_gfx_v10_set_wave_launch_mode, .set_address_watch = kgd_gfx_v10_set_address_watch, - .clear_address_watch = kgd_gfx_v10_clear_address_watch + .clear_address_watch = kgd_gfx_v10_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v10_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.cdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c static uint32_t
Quoted 4 lines
index
b61a32d6af4b..483937c571ab100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +0
-0@@ -786,6 +786,14 @@ static uint32_tkgd_gfx_v11_clear_address_watch(struct
amdgpu_device*adev,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c static uint32_t kgd_gfx_v11_clear_address_watch(struct amdgpu_device *adev,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +1
-0return 0 ; } +static uint64_t kgd_gfx_v11_hqd_detect_and_reset(struct amdgpu_device*adev,
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c const struct kfd2kgd_calls gfx_v11_kfd2kgd = { · line 816
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +7
-0+ uint32_t pipe_id, uint32_t queue_id, + uint32_t inst, unsigned int utimeout, + bool detect_only) +{ + return 0; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,@@ -808,5 +816,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .set_wave_launch_trap_override =
kgd_gfx_v11_set_wave_launch_trap_override,Review
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +2
-1.set_wave_launch_mode = kgd_gfx_v11_set_wave_launch_mode, .set_address_watch = kgd_gfx_v11_set_address_watch, - .clear_address_watch = kgd_gfx_v11_clear_address_watch + .clear_address_watch = kgd_gfx_v11_clear_address_watch, + .hqd_detect_and_reset = kgd_gfx_v11_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.cdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void
Quoted 4 lines
index
5a35a8ca8922..9428e98ac12b100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +0
-0@@ -1144,6 +1144,61 @@ voidkgd_gfx_v9_program_trap_handler_settings(struct
amdgpu_device*adev,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev, · line 1150
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +4
-0kgd_gfx_v9_unlock_srbm (adev, inst); } +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, +bool detect_only)
Quoted 6 lines
This is a bit misleading. This function doesn't detect anything. It just returns a queue address and optionally resets the queue. And the way it's called is like "detect and reset, but don't reset", except that last part is not obvious if you don't know what the last parameter means. It would make the code easier to read if you split this into two
functions:hqd_get_pq_addrandhqd_reset.
Ok. Agreed. That seems cleaner and hqd_get_pq_addr could be useful outside the reset scope in the future.
Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +11
-0+{ +uint32_t low, high, temp; +unsigned long end_jiffies; +uint64_t queue_addr =0 ; + +kgd_gfx_v9_acquire_queue (adev, pipe_id, queue_id, inst); +if (!RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_ACTIVE)) + goto unlock_out; + + low =RREG32_SOC15 (GC ,GET_INST (GC , inst), mmCP_HQD_PQ_BASE); + high = RREG32_SOC15(GC, GET_INST(GC, inst),
mmCP_HQD_PQ_BASE_HI);
Quoted 3 lines
You may need to disable GFXOFF before accessing these registers. Maybe you're assuming that the GPU is not idle when you're getting here, and maybe that's a good assumption, so I'm not sure.
These registers were assumed to be accessed HWS post-hang so the GPU can't be idle.
But it's probably not harmful to have these accessed in RLC safe mode based on Alex's suggestion if anyone wants to use this function outside the KFD reset scope in the future.
Quoted 92 lines
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +5
-0+ +/* only concerned with user queues. */ +if (!high) + goto unlock_out; +drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +10
-0+ queue_addr = (((queue_addr | high) <<32 ) | low) <<8 ; + +if (detect_only) + goto unlock_out; + +pr_debug ("Attempting queue reset on XCC %i pipe id %i queue id %i\n" , + inst, pipe_id, queue_id); + +/* assume previous dequeue request issued will take affect after reset */ + WREG32_SOC15(GC, GET_INST(GC, inst),
mmSPI_COMPUTE_QUEUE_RESET,0x1);drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c void kgd_gfx_v9_program_trap_handler_settings(struct amdgpu_device *adev,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +4
-0+ + end_jiffies = (utimeout *HZ /1000 ) + jiffies; +while (true) { + temp = RREG32_SOC15(GC, GET_INST(GC, inst),
mmCP_HQD_ACTIVE);Quoted 25 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +21
-0+ +if (!(temp &CP_HQD_ACTIVE__ACTIVE_MASK )) +break ; + +if (time_after (jiffies, end_jiffies)) { + queue_addr =0 ; +break ; + } + +usleep_range (500 ,1000 ); + } + +unlock_out: +if (!detect_only) +pr_debug ("queue reset on XCC %i pipe id %i queue id %i %s\n" , + inst, pipe_id, queue_id, !!queue_addr ?"succeeded!" :"failed!" ); +kgd_gfx_v9_release_queue (adev, inst); + +return queue_addr; +} +const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c const struct kfd2kgd_calls gfx_v9_kfd2kgd = { · line 1227
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +0
-0@@ -1172,4 +1227,5 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = { .build_grace_period_packet_info =
kgd_gfx_v9_build_grace_period_packet_info,drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c const struct kfd2kgd_calls gfx_v9_kfd2kgd = { · line 1229
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +0
-0.get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy, .program_trap_handler_settings =
kgd_gfx_v9_program_trap_handler_settings,Review
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +1
-0+ .hqd_detect_and_reset = kgd_gfx_v9_hqd_detect_and_reset };(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.hdrivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h void
Quoted 4 lines
index
ce424615f59b..cf70d8fc3d10100644drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +0
-0@@ -101,3 +101,9 @@ voidkgd_gfx_v9_build_grace_period_packet_info(struct
amdgpu_device*adev,Review
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +6
-0uint32_t grace_period, uint32_t *reg_offset, uint32_t *reg_data); +uint64_t kgd_gfx_v9_hqd_detect_and_reset (struct amdgpu_device *adev, +uint32_t pipe_id, +uint32_t queue_id, +uint32_t inst, +unsigned int utimeout, +bool detect_only);(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.cdrivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c void program_sh_mem_settings(struct
Quoted 4 lines
index
4f48507418d2..037b75a64e66100644drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-0@@ -153,6 +153,19 @@ void program_sh_mem_settings(struct
device_queue_manager*dqm,drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c void program_sh_mem_settings(struct device_queue_manager *dqm, · line 165
Quoted 13 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0static void kfd_hws_hang (struct device_queue_manager *dqm) { +struct device_process_node *cur; +struct qcm_process_device *qpd; +struct queue *q; + +/* Mark all device queues as reset. */ +list_for_each_entry (cur, &dqm->queues, list) { + qpd = cur->qpd; +list_for_each_entry (q, &qpd->queues_list, list) { + q->properties.is_reset = true; + q->process->has_reset_queue = true;
Quoted 4 lines
Both seem to track the same information, but one does it per queue, the other does it per process. Why do you need to track this per queue at all? The only place where I see
q->properties->is_resetused is inupdate_queue. I think you can use the per-process tracking there.
I'll move the per-queue reset bit to the 2nd patch for debugger support since it makes more sense there.
Quoted 5 lines
That said, I think it would be better to track
has_reset_queuein the pdd or qpd because GPU resets are per GPU, and these fields are accessed under the per-GPU DQM lock and reset events sent to user mode are per GPU as well.
Ack'd.
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int update_queue(struct
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +3
-0+ } + } +/* * Issue a GPU reset if HWS is unresponsive */ @@ -880,6 +893,12 @@ static int update_queue(struct
device_queue_manager*dqm, struct queue *q,
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int update_queue(struct device_queue_manager *dqm, struct queue *q, · line 897
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +2
-0else if (prev_active) retval =remove_queue_mes (dqm, q, &pdd->qpd); +/* queue is reset so inaccessable */ +if (q->properties.is_reset) {You could use
pdd->process->has_reset_queue(orpdd->has_reset_queue) here and get rid ofis_reset.
Ack'd.
Quoted 139 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int initialize_cpsch(struct
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4
-0+ retval = -EACCES ; + goto out_unlock; + } +if (retval) {dev_err (dev,"unmap queue failed\n" ); goto out_unlock;@@ -1629,7 +1648,7 @@ static int initialize_cpsch(struct
device_queue_manager*dqm)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int start_cpsch(struct
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1
-1static int start_cpsch (struct device_queue_manager *dqm) {struct device *dev = dqm->dev->adev->dev; -int retval; +int retval, num_hw_queue_slots; retval =0 ;@@ -1682,6 +1701,14 @@ static int start_cpsch(struct
device_queue_manager*dqm)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int start_cpsch(struct device_queue_manager *dqm)
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +2
-0&dqm->wait_times); } +/* setup per-queue reset detection buffer */ + num_hw_queue_slots = dqm->dev->kfd-shared_resources.num_queue_per_pipe *
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +4
-0+ dqm->dev->kfd->shared_resources.num_pipe_per_mec * +NUM_XCC (dqm->dev->xcc_mask); + + dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct
dqm_detect_hang_info);Quoted 1 line
+
dqm->detect_hang_info= kzalloc(dqm->detect_hang_info_size,
GFP_KERNEL);drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int stop_cpsch(struct
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1
-0+dqm_unlock (dqm);return 0 ;@@ -1715,6 +1742,7 @@ static int stop_cpsch(struct
device_queue_manager*dqm)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1
-0kfd_gtt_sa_free (dqm->dev, dqm->fence_mem);if (!dqm->dev->kfd->shared_resources.enable_mes)pm_uninit (&dqm->packet_mgr); +kfree (dqm->detect_hang_info);dqm_unlock (dqm);return 0 ;@@ -1931,6 +1959,143 @@ static int map_queues_cpsch(struct
device_queue_manager*dqm)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm)
Quoted 4 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +1
-0return retval; } +static void set_queue_as_reset(struct device_queue_manager *dqm, structqueue *q,
Quoted 21 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +21
-0+ struct qcm_process_device *qpd) +{ +pr_err ("queue id 0x%0x at pasid 0x%0x is reset\n" , + q->properties.queue_id, q->process->pasid); + + q->properties.is_reset = true; + q->process->has_reset_queue = true; +if (q->properties.is_active) { + q->properties.is_active = false; +decrement_queue_count (dqm, qpd, q); + } +} + +static int detect_queue_hang (struct device_queue_manager *dqm) +{ +int i; + +memset (dqm->detect_hang_info,0 , dqm->detect_hang_info_size); + +for (i =0 ; i <AMDGPU_MAX_QUEUES ; ++i) { +uint32_t mec, pipe, queue;drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm)
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +9
-0+int xcc_id; + + mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe) + / dqm->dev->kfd->shared_resources.num_pipe_per_mec; + +if (mec || !test_bit (i, dqm->dev->kfd- shared_resources.cp_queue_bitmap)) +continue ; + + amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i,&mec, &pipe, &queue);
Quoted 15 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +3
-0+ +for_each_inst (xcc_id, dqm->dev->xcc_mask) { + uint64_t queue_addr = dqm->dev->kfd2kgd-hqd_detect_and_reset(
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +11
-0+ dqm->dev->adev, pipe, queue, xcc_id, + 0, true); +struct dqm_detect_hang_info hang_info; + +if (!queue_addr) +continue ; + + hang_info.pipe_id = pipe; + hang_info.queue_id = queue; + hang_info.xcc_id = xcc_id; + hang_info.queue_address = queue_addr;drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm)
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+ + dqm->detect_hang_info[dqm->detect_hang_count] = hang_info; + dqm->detect_hang_count++; + } + } + +return dqm->detect_hang_count; +} + +static struct queue *find_queue_by_address(struct device_queue_manager*dqm,
uint64_tqueue_address)Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +8
-0+{ +struct device_process_node *cur; +struct qcm_process_device *qpd; +struct queue *q; + +list_for_each_entry (cur, &dqm->queues, list) { + qpd = cur->qpd; +list_for_each_entry (q, &qpd->queues_list, list) {drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm)
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+if (queue_address == q->properties.queue_address) +return q; + } + } + +return NULL ; +} + +/* only for compute queue */ +static int reset_queues_cpsch(struct device_queue_manager *dqm,
uint16_tpasid)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm)
Quoted 7 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +7
-0+{ +int retval =0 , reset_count =0 , i; + +if (pasid) { +dqm_lock (dqm); + + retval = unmap_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_BY_PASID,
Quoted 15 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) · line 2051
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-0+ pasid, USE_DEFAULT_GRACE_PERIOD, true); + +dqm_unlock (dqm); +return retval; + } + +if (!dqm->detect_hang_info || dqm->is_hws_hang) +return -EIO ; + +/* assume dqm locked. */ This is weird, the same function does different things and has different assumptions about its locking context depending where it's called from. Maybe keep the original function and and make this a new function with a distinct name.
Ack'd.
Quoted 10 lines
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +5
-0+if (!detect_queue_hang (dqm)) +return -ENOTRECOVERABLE ; + +for (i =0 ; i < dqm->detect_hang_count; i++) { + struct dqm_detect_hang_info hang_info = dqm-detect_hang_info[i];
+ struct queue *q = find_queue_by_address(dqm,hang_info.queue_address);
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) · line 2062
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +5
-0+struct kfd_process_device *pdd; +uint64_t queue_addr =0 ; +int pipe = hang_info.pipe_id; +int queue = hang_info.queue_id; +int xcc_id = hang_info.xcc_id;Why do you need these local variables.
hang_infois already a local variable. Now you have two copies of this on your stack.
Ack'd. That was left over fluff I forgot to remove.
Thanks,
Jon
Quoted 241 lines
Regards,
Felixdrivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unmap_queues_cpsch(struct
Quoted 40 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +36
-0+ +if (!q) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + + pdd =kfd_get_process_device_data (dqm->dev, q->process); +if (!pdd) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + + queue_addr = dqm->dev->kfd2kgd->hqd_detect_and_reset ( + dqm->dev->adev, pipe, queue, xcc_id, +KFD_UNMAP_LATENCY_MS , false); + +if (queue_addr != q->properties.queue_address) { + retval = -ENOTRECOVERABLE ; + goto reset_fail; + } + +set_queue_as_reset (dqm, q, &pdd->qpd); + reset_count++; + } + +if (reset_count == dqm->detect_hang_count) +kfd_signal_reset_event (dqm->dev); +else + retval = -ENOTRECOVERABLE ; + +reset_fail: + dqm->detect_hang_count =0 ; + +return retval; +} +/* dqm->lock mutex has to be locked before calling this function */ static int unmap_queues_cpsch(struct device_queue_manager *dqm, enum kfd_unmap_queues_filter filter,@@ -1981,11 +2146,14 @@ static int unmap_queues_cpsch(struct
device_queue_manager*dqm,drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unmap_queues_cpsch(struct
Quoted 21 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +8
-5*/ mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ ];if (mqd_mgr->check_preemption_failed (mqd_mgr, dqm- packet_mgr.priv_queue->queue->mqd)) { -while (halt_if_hws_hang) -schedule (); -kfd_hws_hang (dqm); - retval = -ETIME ; - goto out; +if (reset_queues_cpsch (dqm,0 )) { +while (halt_if_hws_hang) +schedule (); + dqm->is_hws_hang = true; +kfd_hws_hang (dqm); + retval = -ETIME ; + goto out; + } }/* We need to reset the grace period value for this device */drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-0@@ -2003,21 +2171,6 @@ static int unmap_queues_cpsch(struct
device_queue_manager*dqm,drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unmap_queues_cpsch(struct device_queue_manager *dqm,
Quoted 12 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-9return retval; } -/* only for compute queue */ -static int reset_queues_cpsch (struct device_queue_manager *dqm, -uint16_t pasid) -{ -int retval; - -dqm_lock (dqm); - - retval = unmap_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_BY_PASID,Review
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +0
-6- pasid, USE_DEFAULT_GRACE_PERIOD, true); - -dqm_unlock (dqm); -return retval; -} -/* dqm->lock mutex has to be locked before calling this function */ static int execute_queues_cpsch(struct device_queue_manager *dqm, enum kfd_unmap_queues_filter filter,(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.hReview
Quoted 30 lines
index
3b9b8eabaacc..dfb36a246637100644drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +12
-0@@ -210,6 +210,13 @@ struct device_queue_manager_asic_ops { struct kfd_node *dev); }; +struct dqm_detect_hang_info { +int pipe_id; +int queue_id; +int xcc_id; +uint64_t queue_address; +}; +/** * struct device_queue_manager * @@ -264,6 +271,11 @@ struct device_queue_manager {uint32_t wait_times;wait_queue_head_t destroy_wait; + +/* for per-queue reset support */ +struct dqm_detect_hang_info *detect_hang_info; +size_t detect_hang_info_size; +int detect_hang_count; }; void device_queue_manager_init_cik((quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdkfd/kfd_events.cdrivers/gpu/drm/amd/amdkfd/kfd_events.c void kfd_signal_reset_event(struct kfd_node
Quoted 12 lines
index
9b33d9d2c9ad..30a55d3733e8100644drivers/gpu/drm/amd/amdkfd/kfd_events.c +1
-0@@ -31,6 +31,7 @@#include <linux/memory.h> #include "kfd_priv.h" #include "kfd_events.h" +#include "kfd_device_queue_manager.h" #include <linux/device.h> /* @@ -1250,6 +1251,20 @@ void kfd_signal_reset_event(struct kfd_node*dev)
drivers/gpu/drm/amd/amdkfd/kfd_events.c void kfd_signal_reset_event(struct kfd_node *dev)
Quoted 12 lines
drivers/gpu/drm/amd/amdkfd/kfd_events.c +9
-0continue ; } +if (dev->dqm->detect_hang_count && !p->has_reset_queue) +continue ; + +if (dev->dqm->detect_hang_count) { +struct amdgpu_task_info *ti; + + ti =amdgpu_vm_get_task_info_pasid (dev->adev, p->pasid); +if (ti) { + DRM_ERROR("Process info: process %s tid %d thread %spid %d\n",
Review
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_events.c +5
-0+ ti->process_name, ti->tgid, ti->task_name, ti->pid); +amdgpu_vm_put_task_info (ti); + } + } +rcu_read_lock (); id =KFD_FIRST_NONSIGNAL_EVENT_ID ;(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.cdrivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c static void update_mqd(struct mqd_manager
Quoted 4 lines
index
66c73825c0a0..84e8ea3a8a0c100644drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +0
-0@@ -321,8 +321,11 @@ static void update_mqd(struct mqd_manager*mm, void *mqd,
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c static void update_mqd(struct mqd_manager · line 321
Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +0
-0static bool check_preemption_failed(struct mqd_manager *mm, void*mqd)
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c static bool check_preemption_failed_v9_4_3(struct
Quoted 13 lines
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +4
-1{struct v9_mqd *m = (struct v9_mqd *)mqd; +uint32_t doorbell_id = m->queue_doorbell_id0; -return kfd_check_hiq_mqd_doorbell_id (mm->dev, m- queue_doorbell_id0, 0); + m->queue_doorbell_id0 =0 ; + +return kfd_check_hiq_mqd_doorbell_id (mm->dev, doorbell_id,0 ); }static int get_wave_state(struct
mqd_manager*mm, void *mqd,drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +0
-0@@ -624,6 +627,7 @@ static bool check_preemption_failed_v9_4_3(struct
mqd_manager*mm, void *mqd)Review
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +1
-0m =get_mqd (mqd + hiq_mqd_size * inst); ret |=kfd_check_hiq_mqd_doorbell_id (mm->dev, m->queue_doorbell_id0, inst); + m->queue_doorbell_id0 =0 ; ++inst; }(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdkfd/kfd_priv.hReview
Quoted 22 lines
index
2b3ec92981e8..e244242fd1b4100644drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4
-0@@ -504,6 +504,7 @@ struct queue_properties {bool is_being_destroyed;bool is_active;bool is_gws; +bool is_reset;uint32_t pm4_target_xcc;bool is_dbg_wa;bool is_user_cu_masked;@@ -982,6 +983,9 @@ struct kfd_process {struct semaphore runtime_enable_sema;bool is_runtime_retry;struct kfd_runtime_info runtime_info; + +/* Tracks queue reset status */ +bool has_reset_queue; };#define KFD_PROCESS_TABLE_SIZE 5/* bits: 32 entries */ (quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdkfd/kfd_process.cdrivers/gpu/drm/amd/amdkfd/kfd_process.c int kfd_process_evict_queues(struct kfd_process
Quoted 4 lines
index
17e42161b015..9a1d69f83a08100644drivers/gpu/drm/amd/amdkfd/kfd_process.c +0
-0@@ -1851,6 +1851,8 @@ int kfd_process_evict_queues(struct kfd_process*p,
uint32_ttrigger)Review
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_process.c +2
-0goto fail; } n_evicted++; + + pdd->dev->dqm->is_hws_hang = false; }return r;(quoted diff) +0
-0
b/drivers/gpu/drm/amd/include/kgd_kfd_interface.hQuoted 14 lines
index
6d094cf3587d..20bb4a0c893e100644drivers/gpu/drm/amd/include/kgd_kfd_interface.h +4
-0@@ -318,6 +318,10 @@ struct kfd2kgd_calls {void (*program_trap_handler_settings)(struct amdgpu_device *adev,uint32_t vmid,uint64_t tba_addr,uint64_t tma_addr,uint32_t inst); +uint64_t (*hqd_detect_and_reset)(struct amdgpu_device *adev, +uint32_t pipe_id,uint32_t queue_id, +uint32_t inst,unsigned int utimeout, + bool detect_only); };#endif /* KGD_KFD_INTERFACE_H_INCLUDED */