drm/kfd: Add CU occupancy support to GFX11
Subsystems (3)
PATCH 1/2 drm/kfd: Add CU occupancy support to GFX11
Port changes from GFX9 to GFX11 mostly as-is.
Minor changes to register access code.
--- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +148 -0
@@ -807,6 +807,153 @@ static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev,return 0 ; } +static void lock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_lock (&adev->srbm_mutex); +mutex_lock (&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_unlock (&adev->grbm_idx_mutex); +mutex_unlock (&adev->srbm_mutex); +} + +/** +* get_wave_count: Read device registers to get number of waves in flight for +* a particular queue. The method also returns the doorbell offset associated +* with the queue. +* +* @adev: Handle of device whose registers are to be read +* @queue_idx: Index of queue in the queue-map bit-field +* @queue_cnt: Stores the wave count and doorbell offset for an active queue +* @inst: xcc's instance number on a multi-XCC setup +*/ +static void get_wave_count (struct amdgpu_device *adev,int queue_idx, +struct kfd_cu_occupancy *queue_cnt,uint32_t inst) +{ +int pipe_idx; +int queue_slot; +unsigned int reg_val; +unsigned int wave_cnt; +/* +* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID +* parameters to read out waves in flight. Get VMID if there are +* non-zero waves in flight. +*/ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; +soc21_grbm_select (adev,1 , pipe_idx, queue_slot,0 ); + reg_val =RREG32_SOC15_IP (GC ,SOC15_REG_OFFSET (GC ,0 , + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val &SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK ; +if (wave_cnt !=0 ) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15 (GC ,0 , regCP_HQD_PQ_DOORBELL_CONTROL) & +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK ) >> +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT ; + } +} + +/** +* kgd_gfx_v11_get_cu_occupancy: Reads relevant registers associated with each +* shader engine and aggregates the number of waves that are in flight for the +* process whose pasid is provided as a parameter. The process could have ZERO +* or more queues running and submitting waves to compute units. +* +* @adev: Handle of device from which to get number of waves in flight +* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset +* for comparison later. +* @max_waves_per_cu: Output parameter updated with maximum number of waves +* possible per Compute Unit +* @inst: xcc's instance number on a multi-XCC setup +* +* Note: It's possible that the device has too many queues (oversubscription) +* in which case a VMID could be remapped to a different PASID. This could lead +* to an inaccurate wave count. Following is a high-level sequence: +* Time T1: vmid = getVmid(); vmid is associated with Pasid P1 +* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 +* In the sequence above wave count obtained from time T1 will be incorrectly +* lost or added to total wave count. +* +* The registers that provide the waves in flight are: +* +* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a +* queue is slotted, OFF if there is no queue. A process could have ZERO or +* more queues slotted and submitting waves to be run on compute units. Even +* when there is a queue it is possible there could be zero wave fronts, this +* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem +* command +* +* For each bit that is ON from above: +* +* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the +* number of waves that are in flight for the queue at specified index. The +* index ranges from 0 to 7. +* +* If non-zero waves are in flight, store the corresponding doorbell offset +* of the queue, along with the wave count. +* +* Determine if the queue belongs to the process by comparing the doorbell +* offset against the process's queues. If it matches, aggregate the wave +* count for the process. +* +* Reading registers referenced above involves programming GRBM appropriately +*/ +static void kgd_gfx_v11_get_cu_occupancy (struct amdgpu_device *adev, +struct kfd_cu_occupancy *cu_occupancy, +int *max_waves_per_cu,uint32_t inst) +{ +int qidx; +int se_idx; +int se_cnt; +int queue_map; +int max_queue_cnt; +DECLARE_BITMAP (cp_queue_bitmap,AMDGPU_MAX_QUEUES ); + +lock_spi_csq_mutexes (adev); +soc21_grbm_select (adev,1 ,0 ,0 ,0 ); + +/* +* Iterate through the shader engines and arrays of the device +* to get number of waves in flight +*/ +bitmap_complement (cp_queue_bitmap, adev->gfx.mec_bitmap[0 ].queue_bitmap, +AMDGPU_MAX_QUEUES ); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; +for (se_idx =0 ; se_idx < se_cnt; se_idx++) { +amdgpu_gfx_select_se_sh (adev, se_idx,0 ,0xffffffff , inst); + queue_map =RREG32_SOC15 (GC ,0 , + regSPI_CSQ_WF_ACTIVE_STATUS); + +for (qidx =0 ; qidx < max_queue_cnt; qidx++) { +/* Skip queues that are not associated with +* compute functions +*/ +if (!test_bit (qidx, cp_queue_bitmap)) +continue ; + +if (!(queue_map & (1 << qidx))) +continue ; + +/* Get number of waves in flight and aggregate them */ +get_wave_count (adev, qidx, &cu_occupancy[qidx], inst); + } + } + +amdgpu_gfx_select_se_sh (adev,0xffffffff ,0xffffffff ,0xffffffff , inst); +soc21_grbm_select (adev,0 ,0 ,0 ,0 ); +unlock_spi_csq_mutexes (adev); + +/* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,
@@ -832,5 +979,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .clear_address_watch = kgd_gfx_v11_clear_address_watch, .hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v11_hqd_reset, + .get_cu_occupancy = kgd_gfx_v11_get_cu_occupancy, .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell };
Signature · 1 line
2.51.1
PATCH 2/2 drm/kfd: Add CU occupancy support to GFX12
Port changes from GFX9 to GFX12 mostly as-is.
Minor changes to register access code.
--- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c | 150 +++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c +149 -1
@@ -368,6 +368,153 @@ static uint32_t kgd_gfx_v12_hqd_sdma_get_doorbell(struct amdgpu_device *adev,return 0 ; } +static void lock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_lock (&adev->srbm_mutex); +mutex_lock (&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_unlock (&adev->grbm_idx_mutex); +mutex_unlock (&adev->srbm_mutex); +} + +/** +* get_wave_count: Read device registers to get number of waves in flight for +* a particular queue. The method also returns the doorbell offset associated +* with the queue. +* +* @adev: Handle of device whose registers are to be read +* @queue_idx: Index of queue in the queue-map bit-field +* @queue_cnt: Stores the wave count and doorbell offset for an active queue +* @inst: xcc's instance number on a multi-XCC setup +*/ +static void get_wave_count (struct amdgpu_device *adev,int queue_idx, +struct kfd_cu_occupancy *queue_cnt,uint32_t inst) +{ +int pipe_idx; +int queue_slot; +unsigned int reg_val; +unsigned int wave_cnt; +/* +* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID +* parameters to read out waves in flight. Get VMID if there are +* non-zero waves in flight. +*/ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; +soc24_grbm_select (adev,1 , pipe_idx, queue_slot,0 ); + reg_val =RREG32_SOC15_IP (GC ,SOC15_REG_OFFSET (GC ,0 , + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val &SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK ; +if (wave_cnt !=0 ) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15 (GC ,0 , regCP_HQD_PQ_DOORBELL_CONTROL) & +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK ) >> +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT ; + } +} + +/** +* kgd_gfx_v12_get_cu_occupancy: Reads relevant registers associated with each +* shader engine and aggregates the number of waves that are in flight for the +* process whose pasid is provided as a parameter. The process could have ZERO +* or more queues running and submitting waves to compute units. +* +* @adev: Handle of device from which to get number of waves in flight +* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset +* for comparison later. +* @max_waves_per_cu: Output parameter updated with maximum number of waves +* possible per Compute Unit +* @inst: xcc's instance number on a multi-XCC setup +* +* Note: It's possible that the device has too many queues (oversubscription) +* in which case a VMID could be remapped to a different PASID. This could lead +* to an inaccurate wave count. Following is a high-level sequence: +* Time T1: vmid = getVmid(); vmid is associated with Pasid P1 +* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 +* In the sequence above wave count obtained from time T1 will be incorrectly +* lost or added to total wave count. +* +* The registers that provide the waves in flight are: +* +* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a +* queue is slotted, OFF if there is no queue. A process could have ZERO or +* more queues slotted and submitting waves to be run on compute units. Even +* when there is a queue it is possible there could be zero wave fronts, this +* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem +* command +* +* For each bit that is ON from above: +* +* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the +* number of waves that are in flight for the queue at specified index. The +* index ranges from 0 to 7. +* +* If non-zero waves are in flight, store the corresponding doorbell offset +* of the queue, along with the wave count. +* +* Determine if the queue belongs to the process by comparing the doorbell +* offset against the process's queues. If it matches, aggregate the wave +* count for the process. +* +* Reading registers referenced above involves programming GRBM appropriately +*/ +static void kgd_gfx_v12_get_cu_occupancy (struct amdgpu_device *adev, +struct kfd_cu_occupancy *cu_occupancy, +int *max_waves_per_cu,uint32_t inst) +{ +int qidx; +int se_idx; +int se_cnt; +int queue_map; +int max_queue_cnt; +DECLARE_BITMAP (cp_queue_bitmap,AMDGPU_MAX_QUEUES ); + +lock_spi_csq_mutexes (adev); +soc24_grbm_select (adev,1 ,0 ,0 ,0 ); + +/* +* Iterate through the shader engines and arrays of the device +* to get number of waves in flight +*/ +bitmap_complement (cp_queue_bitmap, adev->gfx.mec_bitmap[0 ].queue_bitmap, +AMDGPU_MAX_QUEUES ); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; +for (se_idx =0 ; se_idx < se_cnt; se_idx++) { +amdgpu_gfx_select_se_sh (adev, se_idx,0 ,0xffffffff , inst); + queue_map =RREG32_SOC15 (GC ,0 , + regSPI_CSQ_WF_ACTIVE_STATUS); + +for (qidx =0 ; qidx < max_queue_cnt; qidx++) { +/* Skip queues that are not associated with +* compute functions +*/ +if (!test_bit (qidx, cp_queue_bitmap)) +continue ; + +if (!(queue_map & (1 << qidx))) +continue ; + +/* Get number of waves in flight and aggregate them */ +get_wave_count (adev, qidx, &cu_occupancy[qidx], inst); + } + } + +amdgpu_gfx_select_se_sh (adev,0xffffffff ,0xffffffff ,0xffffffff , inst); +soc24_grbm_select (adev,0 ,0 ,0 ,0 ); +unlock_spi_csq_mutexes (adev); + +/* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} +const struct kfd2kgd_calls gfx_v12_kfd2kgd = { .init_interrupts = init_interrupts_v12, .hqd_dump = hqd_dump_v12,
@@ -381,5 +528,6 @@ const struct kfd2kgd_calls gfx_v12_kfd2kgd = { .set_wave_launch_mode = kgd_gfx_v12_set_wave_launch_mode, .set_address_watch = kgd_gfx_v12_set_address_watch, .clear_address_watch = kgd_gfx_v12_clear_address_watch, - .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell, + .get_cu_occupancy = kgd_gfx_v12_get_cu_occupancy, };
Signature · 1 line
2.51.1
RE: [PATCH 2/2] drm/kfd: Add CU occupancy support to GFX12
AMD General
If there are active waves, gfxoff will have no effect. This patch series is
Regards,
-Sreekant
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of David Belanger
Sent: August 17, 2026 3:50 PM
To: [email protected]
Cc: Belanger, David <[email protected]>
Subject: [PATCH 2/2] drm/kfd: Add CU occupancy support to GFX12
Port changes from GFX9 to GFX12 mostly as-is.
Minor changes to register access code.
--- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c | 150 +++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c +150 -0
@@ -368,6 +368,153 @@ static uint32_t kgd_gfx_v12_hqd_sdma_get_doorbell(struct amdgpu_device *adev,return 0 ; } +static void lock_spi_csq_mutexes (struct amdgpu_device *adev) { +mutex_lock (&adev->srbm_mutex); +mutex_lock (&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes (struct amdgpu_device *adev) { +mutex_unlock (&adev->grbm_idx_mutex); +mutex_unlock (&adev->srbm_mutex); +} + +/** +* get_wave_count: Read device registers to get number of waves in +flight for +* a particular queue. The method also returns the doorbell offset +associated +* with the queue. +* +* @adev: Handle of device whose registers are to be read +* @queue_idx: Index of queue in the queue-map bit-field +* @queue_cnt: Stores the wave count and doorbell offset for an active +queue +* @inst: xcc's instance number on a multi-XCC setup */ static void +get_wave_count (struct amdgpu_device *adev,int queue_idx, +struct kfd_cu_occupancy *queue_cnt,uint32_t inst) { +int pipe_idx; +int queue_slot; +unsigned int reg_val; +unsigned int wave_cnt; +/* +* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID +* parameters to read out waves in flight. Get VMID if there are +* non-zero waves in flight. +*/ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; +soc24_grbm_select (adev,1 , pipe_idx, queue_slot,0 ); + reg_val =RREG32_SOC15_IP (GC ,SOC15_REG_OFFSET (GC ,0 , + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val &SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK ; +if (wave_cnt !=0 ) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15 (GC ,0 , regCP_HQD_PQ_DOORBELL_CONTROL) & +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK ) >> +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT ; + } +} + +/** +* kgd_gfx_v12_get_cu_occupancy: Reads relevant registers associated +with each +* shader engine and aggregates the number of waves that are in flight +for the +* process whose pasid is provided as a parameter. The process could +have ZERO +* or more queues running and submitting waves to compute units. +* +* @adev: Handle of device from which to get number of waves in flight +* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset +* for comparison later. +* @max_waves_per_cu: Output parameter updated with maximum number of waves +* possible per Compute Unit +* @inst: xcc's instance number on a multi-XCC setup +* +* Note: It's possible that the device has too many queues +(oversubscription) +* in which case a VMID could be remapped to a different PASID. This +could lead +* to an inaccurate wave count. Following is a high-level sequence: +* Time T1: vmid = getVmid(); vmid is associated with Pasid P1 +* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 +* In the sequence above wave count obtained from time T1 will be +incorrectly +* lost or added to total wave count. +* +* The registers that provide the waves in flight are: +* +* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is +ON if a +* queue is slotted, OFF if there is no queue. A process could have +ZERO or +* more queues slotted and submitting waves to be run on compute +units. Even +* when there is a queue it is possible there could be zero wave +fronts, this +* can happen when queue is waiting on top-of-pipe events - e.g. +waitRegMem +* command +* +* For each bit that is ON from above: +* +* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the +* number of waves that are in flight for the queue at specified index. The +* index ranges from 0 to 7. +* +* If non-zero waves are in flight, store the corresponding doorbell offset +* of the queue, along with the wave count. +* +* Determine if the queue belongs to the process by comparing the doorbell +* offset against the process's queues. If it matches, aggregate the wave +* count for the process. +* +* Reading registers referenced above involves programming GRBM +appropriately */ static void kgd_gfx_v12_get_cu_occupancy (struct +amdgpu_device *adev, +struct kfd_cu_occupancy *cu_occupancy, +int *max_waves_per_cu,uint32_t inst) { +int qidx; +int se_idx; +int se_cnt; +int queue_map; +int max_queue_cnt; +DECLARE_BITMAP (cp_queue_bitmap,AMDGPU_MAX_QUEUES ); + +lock_spi_csq_mutexes (adev); +soc24_grbm_select (adev,1 ,0 ,0 ,0 ); + +/* +* Iterate through the shader engines and arrays of the device +* to get number of waves in flight +*/ +bitmap_complement (cp_queue_bitmap, adev->gfx.mec_bitmap[0 ].queue_bitmap, +AMDGPU_MAX_QUEUES ); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; +for (se_idx =0 ; se_idx < se_cnt; se_idx++) { +amdgpu_gfx_select_se_sh (adev, se_idx,0 ,0xffffffff , inst); + queue_map =RREG32_SOC15 (GC ,0 , + regSPI_CSQ_WF_ACTIVE_STATUS); + +for (qidx =0 ; qidx < max_queue_cnt; qidx++) { +/* Skip queues that are not associated with +* compute functions +*/ +if (!test_bit (qidx, cp_queue_bitmap)) +continue ; + +if (!(queue_map & (1 << qidx))) +continue ; + +/* Get number of waves in flight and aggregate them */ +get_wave_count (adev, qidx, &cu_occupancy[qidx], inst); + } + } + +amdgpu_gfx_select_se_sh (adev,0xffffffff ,0xffffffff ,0xffffffff , inst); +soc24_grbm_select (adev,0 ,0 ,0 ,0 ); +unlock_spi_csq_mutexes (adev);
+
+ /* Update the output parameters and return */
+ *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu *
+ adev->gfx.cu_info.max_waves_per_simd;
+}
+
const struct kfd2kgd_calls gfx_v12_kfd2kgd = {
.init_interrupts = init_interrupts_v12, .hqd_dump = hqd_dump_v12,
@@ -381,5 +528,6 @@ const struct kfd2kgd_calls gfx_v12_kfd2kgd = {
.set_wave_launch_mode = kgd_gfx_v12_set_wave_launch_mode, .set_address_watch = kgd_gfx_v12_set_address_watch, .clear_address_watch = kgd_gfx_v12_clear_address_watch,
- .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell, + .get_cu_occupancy = kgd_gfx_v12_get_cu_occupancy,
};
--
2.51.1
Quoted 3 lines · David Belanger
Port changes from GFX9 to GFX11 mostly as-is.
Minor changes to register access code.
While you are at it, can you port this to gfx10 10 and 10.3 as well?
Quoted 147 lines
Assisted-by Claude:Sonnet 4.6Signed-off-by David Belanger <[email protected]>--- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +132
-0@@ -807,6 +807,153 @@ static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev,return 0 ; } +static void lock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_lock (&adev->srbm_mutex); +mutex_lock (&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_unlock (&adev->grbm_idx_mutex); +mutex_unlock (&adev->srbm_mutex); +} + +/** +* get_wave_count: Read device registers to get number of waves in flight for +* a particular queue. The method also returns the doorbell offset associated +* with the queue. +* +* @adev: Handle of device whose registers are to be read +* @queue_idx: Index of queue in the queue-map bit-field +* @queue_cnt: Stores the wave count and doorbell offset for an active queue +* @inst: xcc's instance number on a multi-XCC setup +*/ +static void get_wave_count (struct amdgpu_device *adev,int queue_idx, +struct kfd_cu_occupancy *queue_cnt,uint32_t inst) +{ +int pipe_idx; +int queue_slot; +unsigned int reg_val; +unsigned int wave_cnt; +/* +* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID +* parameters to read out waves in flight. Get VMID if there are +* non-zero waves in flight. +*/ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; +soc21_grbm_select (adev,1 , pipe_idx, queue_slot,0 ); + reg_val =RREG32_SOC15_IP (GC ,SOC15_REG_OFFSET (GC ,0 , + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val &SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK ; +if (wave_cnt !=0 ) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15 (GC ,0 , regCP_HQD_PQ_DOORBELL_CONTROL) & +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK ) >> +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT ; + } +} + +/** +* kgd_gfx_v11_get_cu_occupancy: Reads relevant registers associated with each +* shader engine and aggregates the number of waves that are in flight for the +* process whose pasid is provided as a parameter. The process could have ZERO +* or more queues running and submitting waves to compute units. +* +* @adev: Handle of device from which to get number of waves in flight +* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset +* for comparison later. +* @max_waves_per_cu: Output parameter updated with maximum number of waves +* possible per Compute Unit +* @inst: xcc's instance number on a multi-XCC setup +* +* Note: It's possible that the device has too many queues (oversubscription) +* in which case a VMID could be remapped to a different PASID. This could lead +* to an inaccurate wave count. Following is a high-level sequence: +* Time T1: vmid = getVmid(); vmid is associated with Pasid P1 +* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 +* In the sequence above wave count obtained from time T1 will be incorrectly +* lost or added to total wave count. +* +* The registers that provide the waves in flight are: +* +* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a +* queue is slotted, OFF if there is no queue. A process could have ZERO or +* more queues slotted and submitting waves to be run on compute units. Even +* when there is a queue it is possible there could be zero wave fronts, this +* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem +* command +* +* For each bit that is ON from above: +* +* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the +* number of waves that are in flight for the queue at specified index. The +* index ranges from 0 to 7. +* +* If non-zero waves are in flight, store the corresponding doorbell offset +* of the queue, along with the wave count. +* +* Determine if the queue belongs to the process by comparing the doorbell +* offset against the process's queues. If it matches, aggregate the wave +* count for the process. +* +* Reading registers referenced above involves programming GRBM appropriately +*/ +static void kgd_gfx_v11_get_cu_occupancy (struct amdgpu_device *adev, +struct kfd_cu_occupancy *cu_occupancy, +int *max_waves_per_cu,uint32_t inst) +{ +int qidx; +int se_idx; +int se_cnt; +int queue_map; +int max_queue_cnt; +DECLARE_BITMAP (cp_queue_bitmap,AMDGPU_MAX_QUEUES ); + +lock_spi_csq_mutexes (adev); +soc21_grbm_select (adev,1 ,0 ,0 ,0 ); + +/* +* Iterate through the shader engines and arrays of the device +* to get number of waves in flight +*/ +bitmap_complement (cp_queue_bitmap, adev->gfx.mec_bitmap[0 ].queue_bitmap, +AMDGPU_MAX_QUEUES ); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; +for (se_idx =0 ; se_idx < se_cnt; se_idx++) { +amdgpu_gfx_select_se_sh (adev, se_idx,0 ,0xffffffff , inst); + queue_map =RREG32_SOC15 (GC ,0 , + regSPI_CSQ_WF_ACTIVE_STATUS); + +for (qidx =0 ; qidx < max_queue_cnt; qidx++) { +/* Skip queues that are not associated with +* compute functions +*/ +if (!test_bit (qidx, cp_queue_bitmap)) +continue ; + +if (!(queue_map & (1 << qidx))) +continue ;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev, · line 951
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +10
-0+ +/* Get number of waves in flight and aggregate them */ +get_wave_count (adev, qidx, &cu_occupancy[qidx], inst); + } + } + +amdgpu_gfx_select_se_sh (adev,0xffffffff ,0xffffffff ,0xffffffff , inst); +soc21_grbm_select (adev,0 ,0 ,0 ,0 ); +unlock_spi_csq_mutexes (adev); +
I think you need to disallow gfxoff around the MMIO accesses unless that is already handled elsewhere. Same comment on the gfx12 implementation.
Alex
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +6
-1+/* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,@@ -832,5 +979,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .clear_address_watch = kgd_gfx_v11_clear_address_watch, .hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v11_hqd_reset, + .get_cu_occupancy = kgd_gfx_v11_get_cu_occupancy, .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell }; --2.51.1
Quoted 7 lines · Alex Deucher
Quoted 3 lines · David Belanger
Port changes from GFX9 to GFX11 mostly as-is.
Minor changes to register access code.While you are at it, can you port this to gfx10 10 and 10.3 as well?
Yes, I can. I am working on the changes for GFX10/10.3. They will be submitted as a separate patch.
Quoted 153 lines
Quoted 153 lines
Assisted-by Claude:Sonnet 4.6Signed-off-by David Belanger <[email protected]>--- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +138
-0@@ -807,6 +807,153 @@ static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev,return 0 ; } +static void lock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_lock (&adev->srbm_mutex); +mutex_lock (&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_unlock (&adev->grbm_idx_mutex); +mutex_unlock (&adev->srbm_mutex); +} + +/** +* get_wave_count: Read device registers to get number of waves in flight for +* a particular queue. The method also returns the doorbell offset associated +* with the queue. +* +* @adev: Handle of device whose registers are to be read +* @queue_idx: Index of queue in the queue-map bit-field +* @queue_cnt: Stores the wave count and doorbell offset for an active queue +* @inst: xcc's instance number on a multi-XCC setup +*/ +static void get_wave_count (struct amdgpu_device *adev,int queue_idx, +struct kfd_cu_occupancy *queue_cnt,uint32_t inst) +{ +int pipe_idx; +int queue_slot; +unsigned int reg_val; +unsigned int wave_cnt; +/* +* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID +* parameters to read out waves in flight. Get VMID if there are +* non-zero waves in flight. +*/ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; +soc21_grbm_select (adev,1 , pipe_idx, queue_slot,0 ); + reg_val =RREG32_SOC15_IP (GC ,SOC15_REG_OFFSET (GC ,0 , + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val &SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK ; +if (wave_cnt !=0 ) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15 (GC ,0 , regCP_HQD_PQ_DOORBELL_CONTROL) & +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK ) >> +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT ; + } +} + +/** +* kgd_gfx_v11_get_cu_occupancy: Reads relevant registers associated with each +* shader engine and aggregates the number of waves that are in flight for the +* process whose pasid is provided as a parameter. The process could have ZERO +* or more queues running and submitting waves to compute units. +* +* @adev: Handle of device from which to get number of waves in flight +* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset +* for comparison later. +* @max_waves_per_cu: Output parameter updated with maximum number of waves +* possible per Compute Unit +* @inst: xcc's instance number on a multi-XCC setup +* +* Note: It's possible that the device has too many queues (oversubscription) +* in which case a VMID could be remapped to a different PASID. This could lead +* to an inaccurate wave count. Following is a high-level sequence: +* Time T1: vmid = getVmid(); vmid is associated with Pasid P1 +* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 +* In the sequence above wave count obtained from time T1 will be incorrectly +* lost or added to total wave count. +* +* The registers that provide the waves in flight are: +* +* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a +* queue is slotted, OFF if there is no queue. A process could have ZERO or +* more queues slotted and submitting waves to be run on compute units. Even +* when there is a queue it is possible there could be zero wave fronts, this +* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem +* command +* +* For each bit that is ON from above: +* +* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the +* number of waves that are in flight for the queue at specified index. The +* index ranges from 0 to 7. +* +* If non-zero waves are in flight, store the corresponding doorbell offset +* of the queue, along with the wave count. +* +* Determine if the queue belongs to the process by comparing the doorbell +* offset against the process's queues. If it matches, aggregate the wave +* count for the process. +* +* Reading registers referenced above involves programming GRBM appropriately +*/ +static void kgd_gfx_v11_get_cu_occupancy (struct amdgpu_device *adev, +struct kfd_cu_occupancy *cu_occupancy, +int *max_waves_per_cu,uint32_t inst) +{ +int qidx; +int se_idx; +int se_cnt; +int queue_map; +int max_queue_cnt; +DECLARE_BITMAP (cp_queue_bitmap,AMDGPU_MAX_QUEUES ); + +lock_spi_csq_mutexes (adev); +soc21_grbm_select (adev,1 ,0 ,0 ,0 ); + +/* +* Iterate through the shader engines and arrays of the device +* to get number of waves in flight +*/ +bitmap_complement (cp_queue_bitmap, adev->gfx.mec_bitmap[0 ].queue_bitmap, +AMDGPU_MAX_QUEUES ); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; +for (se_idx =0 ; se_idx < se_cnt; se_idx++) { +amdgpu_gfx_select_se_sh (adev, se_idx,0 ,0xffffffff , inst); + queue_map =RREG32_SOC15 (GC ,0 , + regSPI_CSQ_WF_ACTIVE_STATUS); + +for (qidx =0 ; qidx < max_queue_cnt; qidx++) { +/* Skip queues that are not associated with +* compute functions +*/ +if (!test_bit (qidx, cp_queue_bitmap)) +continue ; + +if (!(queue_map & (1 << qidx))) +continue ; + +/* Get number of waves in flight and aggregate them */ +get_wave_count (adev, qidx, &cu_occupancy[qidx], inst); + } + } +
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev, · line 951
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +4
-0+amdgpu_gfx_select_se_sh (adev,0xffffffff ,0xffffffff ,0xffffffff , inst); +soc21_grbm_select (adev,0 ,0 ,0 ,0 ); +unlock_spi_csq_mutexes (adev); +I think you need to disallow gfxoff around the MMIO accesses unless that is already handled elsewhere. Same comment on the gfx12 implementation.
I am looking into it further.
AI suggestion is to wrap it in kfd_process.c before calling the hardware specific function:
amdgpu_gfx_off_ctrl(dev->adev, false);dev->kfd2kgd->get_cu_occupancy(...);amdgpu_gfx_off_ctrl(dev->adev, true);
If it makes sense, I could submit that as a separate patch (as common to all implementation, including the original gfx9 implemenation).
Quoted 21 lines
Alex
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +6
-1+/* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,@@ -832,5 +979,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .clear_address_watch = kgd_gfx_v11_clear_address_watch, .hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v11_hqd_reset, + .get_cu_occupancy = kgd_gfx_v11_get_cu_occupancy, .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell }; --2.51.1
Alex Deucher
Acked-byQuoted 6 lines · David
Quoted 2 lines · Alex Deucher
Quoted 1 line · David Belanger
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Port changes from GFX9 to GFX11 mostly as-is.
Minor changes to register access code.While you are at it, can you port this to gfx10 10 and 10.3 as well?
Yes, I can. I am working on the changes for GFX10/10.3. They will be submitted as a separate patch.
Thanks!
Quoted 166 lines
Quoted 155 lines
Quoted 155 lines
Assisted-by Claude:Sonnet 4.6Signed-off-by David Belanger <[email protected]>--- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +138
-0@@ -807,6 +807,153 @@ static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev,return 0 ; } +static void lock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_lock (&adev->srbm_mutex); +mutex_lock (&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes (struct amdgpu_device *adev) +{ +mutex_unlock (&adev->grbm_idx_mutex); +mutex_unlock (&adev->srbm_mutex); +} + +/** +* get_wave_count: Read device registers to get number of waves in flight for +* a particular queue. The method also returns the doorbell offset associated +* with the queue. +* +* @adev: Handle of device whose registers are to be read +* @queue_idx: Index of queue in the queue-map bit-field +* @queue_cnt: Stores the wave count and doorbell offset for an active queue +* @inst: xcc's instance number on a multi-XCC setup +*/ +static void get_wave_count (struct amdgpu_device *adev,int queue_idx, +struct kfd_cu_occupancy *queue_cnt,uint32_t inst) +{ +int pipe_idx; +int queue_slot; +unsigned int reg_val; +unsigned int wave_cnt; +/* +* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID +* parameters to read out waves in flight. Get VMID if there are +* non-zero waves in flight. +*/ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; +soc21_grbm_select (adev,1 , pipe_idx, queue_slot,0 ); + reg_val =RREG32_SOC15_IP (GC ,SOC15_REG_OFFSET (GC ,0 , + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val &SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK ; +if (wave_cnt !=0 ) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15 (GC ,0 , regCP_HQD_PQ_DOORBELL_CONTROL) & +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK ) >> +CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT ; + } +} + +/** +* kgd_gfx_v11_get_cu_occupancy: Reads relevant registers associated with each +* shader engine and aggregates the number of waves that are in flight for the +* process whose pasid is provided as a parameter. The process could have ZERO +* or more queues running and submitting waves to compute units. +* +* @adev: Handle of device from which to get number of waves in flight +* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset +* for comparison later. +* @max_waves_per_cu: Output parameter updated with maximum number of waves +* possible per Compute Unit +* @inst: xcc's instance number on a multi-XCC setup +* +* Note: It's possible that the device has too many queues (oversubscription) +* in which case a VMID could be remapped to a different PASID. This could lead +* to an inaccurate wave count. Following is a high-level sequence: +* Time T1: vmid = getVmid(); vmid is associated with Pasid P1 +* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 +* In the sequence above wave count obtained from time T1 will be incorrectly +* lost or added to total wave count. +* +* The registers that provide the waves in flight are: +* +* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a +* queue is slotted, OFF if there is no queue. A process could have ZERO or +* more queues slotted and submitting waves to be run on compute units. Even +* when there is a queue it is possible there could be zero wave fronts, this +* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem +* command +* +* For each bit that is ON from above: +* +* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the +* number of waves that are in flight for the queue at specified index. The +* index ranges from 0 to 7. +* +* If non-zero waves are in flight, store the corresponding doorbell offset +* of the queue, along with the wave count. +* +* Determine if the queue belongs to the process by comparing the doorbell +* offset against the process's queues. If it matches, aggregate the wave +* count for the process. +* +* Reading registers referenced above involves programming GRBM appropriately +*/ +static void kgd_gfx_v11_get_cu_occupancy (struct amdgpu_device *adev, +struct kfd_cu_occupancy *cu_occupancy, +int *max_waves_per_cu,uint32_t inst) +{ +int qidx; +int se_idx; +int se_cnt; +int queue_map; +int max_queue_cnt; +DECLARE_BITMAP (cp_queue_bitmap,AMDGPU_MAX_QUEUES ); + +lock_spi_csq_mutexes (adev); +soc21_grbm_select (adev,1 ,0 ,0 ,0 ); + +/* +* Iterate through the shader engines and arrays of the device +* to get number of waves in flight +*/ +bitmap_complement (cp_queue_bitmap, adev->gfx.mec_bitmap[0 ].queue_bitmap, +AMDGPU_MAX_QUEUES ); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; +for (se_idx =0 ; se_idx < se_cnt; se_idx++) { +amdgpu_gfx_select_se_sh (adev, se_idx,0 ,0xffffffff , inst); + queue_map =RREG32_SOC15 (GC ,0 , + regSPI_CSQ_WF_ACTIVE_STATUS); + +for (qidx =0 ; qidx < max_queue_cnt; qidx++) { +/* Skip queues that are not associated with +* compute functions +*/ +if (!test_bit (qidx, cp_queue_bitmap)) +continue ; + +if (!(queue_map & (1 << qidx))) +continue ; + +/* Get number of waves in flight and aggregate them */ +get_wave_count (adev, qidx, &cu_occupancy[qidx], inst); + } + } +Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev, · line 951
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +4
-0+amdgpu_gfx_select_se_sh (adev,0xffffffff ,0xffffffff ,0xffffffff , inst); +soc21_grbm_select (adev,0 ,0 ,0 ,0 ); +unlock_spi_csq_mutexes (adev); +I think you need to disallow gfxoff around the MMIO accesses unless that is already handled elsewhere. Same comment on the gfx12 implementation.
I am looking into it further.
Quoted 8 lines
AI suggestion is to wrap it in
kfd_process.cbefore calling the hardware specific function:
amdgpu_gfx_off_ctrl(dev->adev, false);dev->kfd2kgd->get_cu_occupancy(...);amdgpu_gfx_off_ctrl(dev->adev, true);If it makes sense, I could submit that as a separate patch (as common to all implementation, including the original gfx9 implemenation).
Sounds good. Would probably be good to double check all of the paths that call these function pointers which use MMIO.
This series is:
Thanks,
Alex
Quoted 22 lines
Quoted 21 lines
Alex
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +6
-1+/* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} +const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,@@ -832,5 +979,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .clear_address_watch = kgd_gfx_v11_clear_address_watch, .hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v11_hqd_reset, + .get_cu_occupancy = kgd_gfx_v11_get_cu_occupancy, .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell }; --2.51.1