drm/amdgpu: Implement MES Suspend and Resume APIs for GFX11
Subsystems (3)
PATCH 1/3 drm/amdgpu: Implement MES Suspend and Resume APIs for GFX11
Add implementation for MES Suspend and Resume APIs to unmap/map all queues for GFX11. Support or GFX12 will be added when the corresponding firmware support is in place.
--- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +30 -2
@@ -421,13 +421,41 @@ static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,static int mes_v11_0_suspend_gang (struct amdgpu_mes *mes,struct mes_suspend_gang_input *input) { -return 0 ; +union MESAPI__SUSPEND mes_suspend_gang_pkt; + +memset (&mes_suspend_gang_pkt,0 ,sizeof (mes_suspend_gang_pkt)); + + mes_suspend_gang_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_suspend_gang_pkt.header.opcode =MES_SCH_API_SUSPEND ; + mes_suspend_gang_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs; + mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; + mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_suspend_gang_pkt,sizeof (mes_suspend_gang_pkt), + offsetof(union MESAPI__SUSPEND , api_status)); }static int mes_v11_0_resume_gang (struct amdgpu_mes *mes,struct mes_resume_gang_input *input) { -return 0 ; +union MESAPI__RESUME mes_resume_gang_pkt; + +memset (&mes_resume_gang_pkt,0 ,sizeof (mes_resume_gang_pkt)); + + mes_resume_gang_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_resume_gang_pkt.header.opcode =MES_SCH_API_RESUME ; + mes_resume_gang_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; + mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_resume_gang_pkt,sizeof (mes_resume_gang_pkt), + offsetof(union MESAPI__RESUME , api_status)); }static int mes_v11_0_query_sched_status (struct amdgpu_mes *mes)
Signature · 1 line
2.35.1
PATCH 2/3 drm/amdkfd: Update queue unmap after VM fault with MES
MEC FW expects MES to unmap all queues when a VM fault is observed on a queue and then resumed once the affected process is terminated. Use the MES Suspend and Resume APIs to achieve this.
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +74 -1
@@ -319,6 +319,56 @@ static int remove_all_queues_mes(struct device_queue_manager *dqm)return retval; } +static int suspend_all_queues_mes (struct device_queue_manager *dqm) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; +int r; +struct mes_suspend_gang_input queue_input; + +if (dqm->is_hws_hang) +return -EIO ; + +memset (&queue_input,0x0 ,sizeof (struct mes_suspend_gang_input )); + queue_input.suspend_all_gangs =1 ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->suspend_gang (&adev->mes, &queue_input); +amdgpu_mes_unlock (&adev->mes); + +if (r) { +dev_err (adev->dev,"failed to suspend gangs from MES\n" ); +dev_err (adev->dev,"MES might be in unrecoverable state, issue a GPU reset\n" ); +kfd_hws_hang (dqm); + } + +return r; +} + +static int resume_all_queues_mes (struct device_queue_manager *dqm) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; +int r; +struct mes_resume_gang_input queue_input; + +if (dqm->is_hws_hang) +return -EIO ; + +memset (&queue_input,0x0 ,sizeof (struct mes_resume_gang_input )); + queue_input.resume_all_gangs =1 ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->resume_gang (&adev->mes, &queue_input); +amdgpu_mes_unlock (&adev->mes); + +if (r) { +dev_err (adev->dev,"failed to resume gangs from MES\n" ); +dev_err (adev->dev,"MES might be in unrecoverable state, issue a GPU reset\n" ); +kfd_hws_hang (dqm); + } + +return r; +} +static void increment_queue_count (struct device_queue_manager *dqm,struct qcm_process_device *qpd,struct queue *q)
@@ -2839,14 +2889,37 @@ int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid) {struct kfd_process_device *pdd;struct kfd_process *p =kfd_lookup_process_by_pasid (pasid); +struct device *dev = dqm->dev->adev->dev;int ret =0 ;if (!p)return -EINVAL ;WARN (debug_evictions,"Evicting pid %d" , p->lead_thread->pid); pdd =kfd_get_process_device_data (dqm->dev, p); -if (pdd) +if (pdd) { +if (dqm->dev->kfd->shared_resources.enable_mes) { + ret =suspend_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Suspending all queues failed" ); + goto out; + } + } ret = dqm->ops.evict_process_queues (dqm, &pdd->qpd); +if (ret) { +dev_err (dev,"Evicting process queues failed" ); + goto out; + } + +if (dqm->dev->kfd->shared_resources.enable_mes) { + ret =resume_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Resuming all queues failed" ); + goto out; + } + } + } + +out:kfd_unref_process (p);return ret;
Signature · 1 line
2.35.1
RE: [PATCH 2/3] drm/amdkfd: Update queue unmap after VM fault with MES
[AMD Official Use Only - AMD Internal Distribution Only]
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Mukul Joshi
Sent: Tuesday, August 13, 2024 2:57 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: [PATCH 2/3] drm/amdkfd: Update queue unmap after VM fault with MES
MEC FW expects MES to unmap all queues when a VM fault is observed on a queue and then resumed once the affected process is terminated. Use the MES Suspend and Resume APIs to achieve this.
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +64 -1
@@ -319,6 +319,56 @@ static int remove_all_queues_mes(struct device_queue_manager *dqm)return retval; } +static int suspend_all_queues_mes (struct device_queue_manager *dqm) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; +int r; +struct mes_suspend_gang_input queue_input; + +if (dqm->is_hws_hang) +return -EIO ; + +memset (&queue_input,0x0 ,sizeof (struct mes_suspend_gang_input )); + queue_input.suspend_all_gangs =1 ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->suspend_gang (&adev->mes, &queue_input); +amdgpu_mes_unlock (&adev->mes); + +if (r) { +dev_err (adev->dev,"failed to suspend gangs from MES\n" ); +dev_err (adev->dev,"MES might be in unrecoverable state, issue a GPU reset\n" ); +kfd_hws_hang (dqm); + } + +return r; +} + +static int resume_all_queues_mes (struct device_queue_manager *dqm) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; +int r; +struct mes_resume_gang_input queue_input; + +if (dqm->is_hws_hang) +return -EIO ; + +memset (&queue_input,0x0 ,sizeof (struct mes_resume_gang_input )); + queue_input.resume_all_gangs =1 ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->resume_gang (&adev->mes, &queue_input); +amdgpu_mes_unlock (&adev->mes); + +if (r) { +dev_err (adev->dev,"failed to resume gangs from MES\n" ); +dev_err (adev->dev,"MES might be in unrecoverable state, issue a GPU reset\n" ); +kfd_hws_hang (dqm); + } + +return r; +} +static void increment_queue_count (struct device_queue_manager *dqm,struct qcm_process_device *qpd,struct queue *q)
@@ -2839,14 +2889,37 @@ int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid) {struct kfd_process_device *pdd;struct kfd_process *p =kfd_lookup_process_by_pasid (pasid); +struct device *dev = dqm->dev->adev->dev;int ret =0 ;if (!p)return -EINVAL ;WARN (debug_evictions,"Evicting pid %d" , p->lead_thread->pid); pdd =kfd_get_process_device_data (dqm->dev, p); -if (pdd) +if (pdd) { +if (dqm->dev->kfd->shared_resources.enable_mes) { + ret =suspend_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Suspending all queues failed" ); + goto out; + } + } ret = dqm->ops.evict_process_queues (dqm, &pdd->qpd); +if (ret) { +dev_err (dev,"Evicting process queues failed" ); + goto out; + } +
evict_process_queues() function also has if(enable_mes) branching. Would it make sense to just call two different functions here one for MES and one for HWS?
Also, if the process is already evicted then there is no need to do suspend_all and resume_all.
+ if (dqm->dev->kfd->shared_resources.enable_mes) {
+ ret = resume_all_queues_mes(dqm);
+ if (ret) {
+ dev_err(dev, "Resuming all queues failed");
+ goto out;
+ }
+ }
+ }
+
+out:
kfd_unref_process(p);
return ret;--
2.35.1
RE: [PATCH 2/3] drm/amdkfd: Update queue unmap after VM fault with MES
[AMD Official Use Only - AMD Internal Distribution Only]
Quoted 121 lines
-----Original Message-----
From: Kasiviswanathan, Harish <[email protected]>
Sent: Wednesday, August 14, 2024 11:17 AM
To: Joshi, Mukul <[email protected]>; [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: RE: [PATCH 2/3] drm/amdkfd: Update queue unmap after VM fault with MES[AMD Official Use Only - AMD Internal Distribution Only]
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Mukul Joshi
Sent: Tuesday, August 13, 2024 2:57 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: [PATCH 2/3] drm/amdkfd: Update queue unmap after VM fault with MESMEC FW expects MES to unmap all queues when a VM fault is observed on a queue and then resumed once the affected process is terminated. Use the MES Suspend and Resume APIs to achieve this.
Signed-off-by Mukul Joshi <[email protected]>--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +26
-0@@ -319,6 +319,56 @@ static int remove_all_queues_mes(struct device_queue_manager *dqm)return retval; } +static int suspend_all_queues_mes (struct device_queue_manager *dqm) { +struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev- >adev; +int r; +struct mes_suspend_gang_input queue_input; + +if (dqm->is_hws_hang) +return -EIO ; + +memset (&queue_input,0x0 ,sizeof (struct mes_suspend_gang_input )); + queue_input.suspend_all_gangs =1 ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->suspend_gang (&adev->mes, &queue_input); +amdgpu_mes_unlock (&adev->mes); + +if (r) { +dev_err (adev->dev,"failed to suspend gangs from MES\n" ); + dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); + kfd_hws_hang(dqm); + } + +return r; +} + +static int resume_all_queues_mes (struct device_queue_manager *dqm) { + struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev-Quoted 1 line
adev;
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +16
-0+int r; +struct mes_resume_gang_input queue_input; + +if (dqm->is_hws_hang) +return -EIO ; + +memset (&queue_input,0x0 ,sizeof (struct mes_resume_gang_input )); + queue_input.resume_all_gangs =1 ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->resume_gang (&adev->mes, &queue_input); +amdgpu_mes_unlock (&adev->mes); + +if (r) { +dev_err (adev->dev,"failed to resume gangs from MES\n" ); + dev_err(adev->dev, "MES might be in unrecoverable state, issue aGPU reset\n");
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +6
-0+kfd_hws_hang (dqm); + } + +return r; +} +static void increment_queue_count (struct device_queue_manager *dqm,struct qcm_process_device *qpd, struct queue *q) @@ -2839,14 +2889,37 @@ intkfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid) { struct kfd_process_device *pdd; struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +14
-1+struct device *dev = dqm->dev->adev->dev;int ret =0 ;if (!p)return -EINVAL ;WARN (debug_evictions,"Evicting pid %d" , p->lead_thread->pid); pdd =kfd_get_process_device_data (dqm->dev, p); -if (pdd) +if (pdd) { +if (dqm->dev->kfd->shared_resources.enable_mes) { + ret =suspend_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Suspending all queues failed" ); + goto out; + } + } ret = dqm->ops.evict_process_queues (dqm, &pdd->qpd); +if (ret) { +dev_err (dev,"Evicting process queues failed" ); + goto out; + } +
Quoted 4 lines
evict_process_queues()function also hasif(enable_mes)branching. Would it make sense to just call two different functions here one for MES and one for HWS?
I think it should be fine to keep it like the way it is. Keeping 2 separate functions one for MES and other for HWS will probably lead to duplication of the code. We have the if (enable_mes) branching at a lot of places in the code and it was done just to avoid duplication.
Quoted 2 lines
Also, if the process is already evicted then there is no need to do
suspend_allandresume_all.
Yes that's a good point. I can put a check to check if the process is already evicted. If it is, then we avoid the suspend and resume.
Thanks,
Mukul
Quoted 18 lines
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +10
-1+if (dqm->dev->kfd->shared_resources.enable_mes) { + ret =resume_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Resuming all queues failed" ); + goto out; + } + } + } + +out:kfd_unref_process (p);return ret; --2.35.1
PATCH 3/3 drm/amdkfd: Update BadOpcode Interrupt handling with MES
In reply to Mukul JoshiBased on the recommendation of MEC FW, update BadOpcode interrupt handling by unmapping all queues, removing the queue that got the interrupt from scheduling and remapping rest of the queues back when using MES scheduler. This is done to prevent the case where unmapping of the bad queue can fail thereby causing a GPU reset.
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 51 +++++++++++++++++++ .../gpu/drm/amd/amdkfd/kfd_int_process_v11.c | 9 ++-- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 + 3 files changed, 58 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +51 -0
@@ -2885,6 +2885,57 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)kfree (dqm); } +int kfd_dqm_suspend_bad_queue_mes (struct kfd_node *knode,u32 pasid,u32 doorbell_id) +{ +struct kfd_process_device *pdd; +struct kfd_process *p =kfd_lookup_process_by_pasid (pasid); +struct device_queue_manager *dqm = knode->dqm; +struct device *dev = dqm->dev->adev->dev; +struct qcm_process_device *qpd; +struct queue *q =NULL ; +int ret =0 ; + +if (!p) +return -EINVAL ; + +dqm_lock (dqm); + + pdd =kfd_get_process_device_data (dqm->dev, p); +if (pdd) { + qpd = &pdd->qpd; + +list_for_each_entry (q, &qpd->queues_list, list) { +if (q->doorbell_id == doorbell_id && q->properties.is_active) { + ret =suspend_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Suspending all queues failed" ); + goto out; + } + + q->properties.is_evicted = true; + q->properties.is_active = false; +decrement_queue_count (dqm, qpd, q); + + ret =remove_queue_mes (dqm, q, qpd); +if (ret) { +dev_err (dev,"Removing bad queue failed" ); + goto out; + } + + ret =resume_all_queues_mes (dqm); +if (ret) +dev_err (dev,"Resuming all queues failed" ); + +break ; + } + } + } + +out: +dqm_unlock (dqm); +return ret; +} +int kfd_dqm_evict_pasid (struct device_queue_manager *dqm,u32 pasid) {struct kfd_process_device *pdd;
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c +6 -3
@@ -330,11 +330,14 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,if (source_id ==SOC15_INTSRC_CP_END_OF_PIPE )kfd_signal_event_interrupt (pasid, context_id0,32 );else if (source_id ==SOC15_INTSRC_CP_BAD_OPCODE && -KFD_DBG_EC_TYPE_IS_PACKET (KFD_CTXID0_CP_BAD_OP_ECODE (context_id0))) -kfd_set_dbg_ev_from_interrupt (dev, pasid, -KFD_CTXID0_DOORBELL_ID (context_id0), +KFD_DBG_EC_TYPE_IS_PACKET (KFD_CTXID0_CP_BAD_OP_ECODE (context_id0))) { +u32 doorbell_id =KFD_CTXID0_DOORBELL_ID (context_id0); + +kfd_set_dbg_ev_from_interrupt (dev, pasid, doorbell_id,KFD_EC_MASK (KFD_CTXID0_CP_BAD_OP_ECODE (context_id0)),NULL ,0 ); +kfd_dqm_suspend_bad_queue_mes (dev, pasid, doorbell_id); + }/* SDMA */ else if (source_id ==SOC21_INTSRC_SDMA_TRAP )
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +1 -0
@@ -1324,6 +1324,7 @@ struct kernel_queue *kernel_queue_init(struct kfd_node *dev, enum kfd_queue_type type);void kernel_queue_uninit (struct kernel_queue *kq);int kfd_dqm_evict_pasid (struct device_queue_manager *dqm,u32 pasid); +int kfd_dqm_suspend_bad_queue_mes (struct kfd_node *knode,u32 pasid,u32 doorbell_id);/* Process Queue Manager */ struct process_queue_node {
Signature · 1 line
2.35.1
RE: [PATCH 3/3] drm/amdkfd: Update BadOpcode Interrupt handling with MES
[AMD Official Use Only - AMD Internal Distribution Only]
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Mukul Joshi
Sent: Tuesday, August 13, 2024 2:57 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: [PATCH 3/3] drm/amdkfd: Update BadOpcode Interrupt handling with MES
Based on the recommendation of MEC FW, update BadOpcode interrupt handling by unmapping all queues, removing the queue that got the interrupt from scheduling and remapping rest of the queues back when using MES scheduler. This is done to prevent the case where unmapping of the bad queue can fail thereby causing a GPU reset.
--- .../drm/amd/amdkfd/kfd_device_queue_manager.c | 51 +++++++++++++++++++ .../gpu/drm/amd/amdkfd/kfd_int_process_v11.c | 9 ++-- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 + 3 files changed, 58 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +51 -0
@@ -2885,6 +2885,57 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)kfree (dqm); } +int kfd_dqm_suspend_bad_queue_mes (struct kfd_node *knode,u32 pasid,u32 doorbell_id) +{ +struct kfd_process_device *pdd; +struct kfd_process *p =kfd_lookup_process_by_pasid (pasid); +struct device_queue_manager *dqm = knode->dqm; +struct device *dev = dqm->dev->adev->dev; +struct qcm_process_device *qpd; +struct queue *q =NULL ; +int ret =0 ; + +if (!p) +return -EINVAL ; + +dqm_lock (dqm); + + pdd =kfd_get_process_device_data (dqm->dev, p); +if (pdd) { + qpd = &pdd->qpd; + +list_for_each_entry (q, &qpd->queues_list, list) { +if (q->doorbell_id == doorbell_id && q->properties.is_active) { + ret =suspend_all_queues_mes (dqm); +if (ret) { +dev_err (dev,"Suspending all queues failed" ); + goto out; + } + + q->properties.is_evicted = true; + q->properties.is_active = false; +decrement_queue_count (dqm, qpd, q); + + ret =remove_queue_mes (dqm, q, qpd); +if (ret) { +dev_err (dev,"Removing bad queue failed" ); + goto out; + } + + ret =resume_all_queues_mes (dqm); +if (ret) +dev_err (dev,"Resuming all queues failed" ); + +break ; + } + } + } + +out: +dqm_unlock (dqm); +return ret; +} +int kfd_dqm_evict_pasid (struct device_queue_manager *dqm,u32 pasid) {struct kfd_process_device *pdd;
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c +6 -3
@@ -330,11 +330,14 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,if (source_id ==SOC15_INTSRC_CP_END_OF_PIPE )kfd_signal_event_interrupt (pasid, context_id0,32 );else if (source_id ==SOC15_INTSRC_CP_BAD_OPCODE && -KFD_DBG_EC_TYPE_IS_PACKET (KFD_CTXID0_CP_BAD_OP_ECODE (context_id0))) -kfd_set_dbg_ev_from_interrupt (dev, pasid, -KFD_CTXID0_DOORBELL_ID (context_id0), +KFD_DBG_EC_TYPE_IS_PACKET (KFD_CTXID0_CP_BAD_OP_ECODE (context_id0))) { +u32 doorbell_id =KFD_CTXID0_DOORBELL_ID (context_id0); + +kfd_set_dbg_ev_from_interrupt (dev, pasid, doorbell_id,KFD_EC_MASK (KFD_CTXID0_CP_BAD_OP_ECODE (context_id0)),NULL ,0 ); +kfd_dqm_suspend_bad_queue_mes (dev, pasid, doorbell_id); + }/* SDMA */ else if (source_id ==SOC21_INTSRC_SDMA_TRAP )
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +1 -0
@@ -1324,6 +1324,7 @@ struct kernel_queue *kernel_queue_init(struct kfd_node *dev, enum kfd_queue_type type);void kernel_queue_uninit (struct kernel_queue *kq);int kfd_dqm_evict_pasid (struct device_queue_manager *dqm,u32 pasid); +int kfd_dqm_suspend_bad_queue_mes (struct kfd_node *knode,u32 pasid,u32 doorbell_id);/* Process Queue Manager */ struct process_queue_node {
--
2.35.1
[AMD Official Use Only - AMD Internal Distribution Only]
Do we need some checks for FW version for backward compatibility? Apart from that a minor typo in the commit message. "Support 'f'or GFX12"
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Mukul Joshi
Sent: Tuesday, August 13, 2024 2:57 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu: Implement MES Suspend and Resume APIs for GFX11
Add implementation for MES Suspend and Resume APIs to unmap/map all queues for GFX11. Support or GFX12 will be added when the corresponding firmware support is in place.
--- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +30 -2
@@ -421,13 +421,41 @@ static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,static int mes_v11_0_suspend_gang (struct amdgpu_mes *mes,struct mes_suspend_gang_input *input) { -return 0 ; +union MESAPI__SUSPEND mes_suspend_gang_pkt; + +memset (&mes_suspend_gang_pkt,0 ,sizeof (mes_suspend_gang_pkt)); + + mes_suspend_gang_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_suspend_gang_pkt.header.opcode =MES_SCH_API_SUSPEND ; + mes_suspend_gang_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs; + mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; + mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_suspend_gang_pkt,sizeof (mes_suspend_gang_pkt), + offsetof(union MESAPI__SUSPEND , api_status)); }static int mes_v11_0_resume_gang (struct amdgpu_mes *mes,struct mes_resume_gang_input *input) { -return 0 ; +union MESAPI__RESUME mes_resume_gang_pkt; + +memset (&mes_resume_gang_pkt,0 ,sizeof (mes_resume_gang_pkt)); + + mes_resume_gang_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_resume_gang_pkt.header.opcode =MES_SCH_API_RESUME ; + mes_resume_gang_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; + mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_resume_gang_pkt,sizeof (mes_resume_gang_pkt), + offsetof(union MESAPI__RESUME , api_status)); }static int mes_v11_0_query_sched_status (struct amdgpu_mes *mes)
--
2.35.1
[AMD Official Use Only - AMD Internal Distribution Only]
Quoted 9 lines
-----Original Message-----
From: Kasiviswanathan, Harish <[email protected]>
Sent: Wednesday, August 14, 2024 10:37 AM
To: Joshi, Mukul <[email protected]>; [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: RE: [PATCH 1/3] drm/amdgpu: Implement MES Suspend and Resume APIs for GFX11
Quoted 5 lines
[AMD Official Use Only - AMD Internal Distribution Only]
Do we need some checks for FW version for backward compatibility? Apart from that a minor typo in the commit message. "Support 'f'or GFX12"
We usually do that in the callers of the function. Having said that, adding this implementation requires me to update the code during suspend/resume, which I missed.
I will fix that and the typo in v2.
Thanks,
Mukul
Quoted 75 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Mukul Joshi
Sent: Tuesday, August 13, 2024 2:57 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Deucher, Alexander <[email protected]>; Joshi, Mukul <[email protected]>
Subject: [PATCH 1/3] drm/amdgpu: Implement MES Suspend and Resume APIs for GFX11Add implementation for MES Suspend and Resume APIs to unmap/map all queues for GFX11. Support or GFX12 will be added when the corresponding firmware support is in place.
Signed-off-by Mukul Joshi <[email protected]>--- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-)drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +0
-0@@ -421,13 +421,41 @@ static int mes_v11_0_unmap_legacy_queue(struct
amdgpu_mes*mes, static int mes_v11_0_suspend_gang(structamdgpu_mes*mes,struct mes_suspend_gang_input *input) {drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +10
-1-return 0 ; +union MESAPI__SUSPEND mes_suspend_gang_pkt; + +memset (&mes_suspend_gang_pkt,0 ,sizeof (mes_suspend_gang_pkt)); + + mes_suspend_gang_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_suspend_gang_pkt.header.opcode =MES_SCH_API_SUSPEND ; + mes_suspend_gang_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs; + mes_suspend_gang_pkt.gang_context_addr = input-Quoted 1 line
gang_context_addr;+ mes_suspend_gang_pkt.suspend_fence_addr = input-
Quoted 1 line
suspend_fence_addr;drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +21
-2+ mes_suspend_gang_pkt.suspend_fence_value = + input->suspend_fence_value; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_suspend_gang_pkt,sizeof (mes_suspend_gang_pkt), + offsetof(union MESAPI__SUSPEND , api_status)); }static int mes_v11_0_resume_gang (struct amdgpu_mes *mes,struct mes_resume_gang_input *input) { -return 0 ; +union MESAPI__RESUME mes_resume_gang_pkt; + +memset (&mes_resume_gang_pkt,0 ,sizeof (mes_resume_gang_pkt)); + + mes_resume_gang_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_resume_gang_pkt.header.opcode =MES_SCH_API_RESUME ; + mes_resume_gang_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; + mes_resume_gang_pkt.gang_context_addr = + input->gang_context_addr; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_resume_gang_pkt,sizeof (mes_resume_gang_pkt), + offsetof(union MESAPI__RESUME , api_status)); }static int mes_v11_0_query_sched_status (struct amdgpu_mes *mes) --2.35.1