drm/amdgpu: fix transform feedback GDS hang on gfx10 (v2)
Subsystems (2)
PATCH 1/2 drm/amdgpu: fix transform feedback GDS hang on gfx10 (v2)
From: Marek Olšák <[email protected]>
v2: update emit_ib_size
(though it's still wrong because it was wrong before)
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 3 ++- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +2 -1
@@ -24,21 +24,22 @@#ifndef __AMDGPU_GDS_H__#define __AMDGPU_GDS_H__struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size; -uint32_t gds_compute_max_wave_id; +uint32_t gds_compute_max_wave_id; +uint32_t vgt_gs_max_wave_id; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +11 -3
@@ -4211,20 +4211,29 @@ static void gfx_v10_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) }static void gfx_v10_0_ring_emit_ib_gfx (struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,uint32_t flags) {unsigned vmid =AMDGPU_JOB_GET_VMID (job);u32 header, control =0 ; +/* Prevent a hw deadlock due to a wave ID mismatch between ME and GDS. +* This resets the wave ID counters. (needed by transform feedback) +* TODO: This might only be needed on a VMID switch when we change +* the GDS OA mapping, not sure. +*/ +amdgpu_ring_write (ring,PACKET3 (PACKET3_SET_CONFIG_REG ,1 )); +amdgpu_ring_write (ring, mmVGT_GS_MAX_WAVE_ID); +amdgpu_ring_write (ring, ring->adev->gds.vgt_gs_max_wave_id); +if (ib->flags &AMDGPU_IB_FLAG_CE ) header =PACKET3 (PACKET3_INDIRECT_BUFFER_CNST ,2 );else header =PACKET3 (PACKET3_INDIRECT_BUFFER ,2 ); control |= ib->length_dw | (vmid <<24 );if (amdgpu_mcbp && (ib->flags &AMDGPU_IB_FLAG_PREEMPT )) { control |=INDIRECT_BUFFER_PRE_ENB (1 );
@@ -4944,21 +4953,21 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {*/ 5 +/* COND_EXEC */ 7 +/* HDP_flush */ 4 +/* VGT_flush */ 14 +/* CE_META */ 31 +/* DE_META */ 3 +/* CNTX_CTRL */ 5 +/* HDP_INVL */ 8 +8 +/* FENCE x2 */ 2, /* SWITCH_BUFFER */ - .emit_ib_size = 4, /* gfx_v10_0_ring_emit_ib_gfx */ + .emit_ib_size = 7, /* gfx_v10_0_ring_emit_ib_gfx */ .emit_ib = gfx_v10_0_ring_emit_ib_gfx, .emit_fence = gfx_v10_0_ring_emit_fence, .emit_pipeline_sync = gfx_v10_0_ring_emit_pipeline_sync, .emit_vm_flush = gfx_v10_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v10_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v10_0_ring_emit_hdp_flush, .test_ring = gfx_v10_0_ring_test_ring, .test_ib = gfx_v10_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib,
@@ -5092,24 +5101,23 @@ static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev)default :break ; } }static void gfx_v10_0_set_gds_init (struct amdgpu_device *adev) {/* init asic gds info */ switch (adev->asic_type) {case CHIP_NAVI10 : - adev->gds.gds_size =0x10000 ; -break ;default : adev->gds.gds_size =0x10000 ; + adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; }static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap) {
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20190626223521.14347-1-maraeo-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
PATCH 2/2 drm/amdgpu: handle AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID on gfx10
From: Marek Olšák <[email protected]>
--- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +17 -0
@@ -4257,20 +4257,36 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, }static void gfx_v10_0_ring_emit_ib_compute (struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,uint32_t flags) {unsigned vmid =AMDGPU_JOB_GET_VMID (job);u32 control =INDIRECT_BUFFER_VALID | ib->length_dw | (vmid <<24 ); +/* Currently, there is a high possibility to get wave ID mismatch +* between ME and GDS, leading to a hw deadlock, because ME generates +* different wave IDs than the GDS expects. This situation happens +* randomly when at least 5 compute pipes use GDS ordered append. +* The wave IDs generated by ME are also wrong after suspend/resume. +* Those are probably bugs somewhere else in the kernel driver. +* +* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and +* GDS to 0 for this ring (me/pipe). +*/ +if (ib->flags &AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID ) { +amdgpu_ring_write (ring,PACKET3 (PACKET3_SET_CONFIG_REG ,1 )); +amdgpu_ring_write (ring, mmGDS_COMPUTE_MAX_WAVE_ID); +amdgpu_ring_write (ring, ring->adev->gds.gds_compute_max_wave_id); + } +amdgpu_ring_write (ring,PACKET3 (PACKET3_INDIRECT_BUFFER ,2 ));BUG_ON (ib->gpu_addr &0x3 );/* Dword align */ amdgpu_ring_write (ring, #ifdef __BIG_ENDIAN (2 <<0 ) | #endiflower_32_bits (ib->gpu_addr));amdgpu_ring_write (ring,upper_32_bits (ib->gpu_addr));amdgpu_ring_write (ring, control); }
@@ -5103,20 +5119,21 @@ static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev) } }static void gfx_v10_0_set_gds_init (struct amdgpu_device *adev) {/* init asic gds info */ switch (adev->asic_type) {case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; + adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; }static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap)
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20190626223521.14347-2-maraeo-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Re: [PATCH 2/2] drm/amdgpu: handle AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID on gfx10
any reason for not care .emit_ib_size in this one?
-David
Quoted 60 lines · Marek Olšák
From: Marek Olšák <[email protected]>
Signed-off-by Marek Olšák <[email protected]>--- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +17
-0@@ -4257,20 +4257,36 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, }static void gfx_v10_0_ring_emit_ib_compute (struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,uint32_t flags) {unsigned vmid =AMDGPU_JOB_GET_VMID (job);u32 control =INDIRECT_BUFFER_VALID | ib->length_dw | (vmid <<24 ); +/* Currently, there is a high possibility to get wave ID mismatch +* between ME and GDS, leading to a hw deadlock, because ME generates +* different wave IDs than the GDS expects. This situation happens +* randomly when at least 5 compute pipes use GDS ordered append. +* The wave IDs generated by ME are also wrong after suspend/resume. +* Those are probably bugs somewhere else in the kernel driver. +* +* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and +* GDS to 0 for this ring (me/pipe). +*/ +if (ib->flags &AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID ) { +amdgpu_ring_write (ring,PACKET3 (PACKET3_SET_CONFIG_REG ,1 )); +amdgpu_ring_write (ring, mmGDS_COMPUTE_MAX_WAVE_ID); +amdgpu_ring_write (ring, ring->adev->gds.gds_compute_max_wave_id); + } +amdgpu_ring_write (ring,PACKET3 (PACKET3_INDIRECT_BUFFER ,2 ));BUG_ON (ib->gpu_addr &0x3 );/* Dword align */ amdgpu_ring_write (ring, #ifdef __BIG_ENDIAN (2 <<0 ) | #endiflower_32_bits (ib->gpu_addr));amdgpu_ring_write (ring,upper_32_bits (ib->gpu_addr));amdgpu_ring_write (ring, control); }@@ -5103,20 +5119,21 @@ static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev) } }static void gfx_v10_0_set_gds_init (struct amdgpu_device *adev) {/* init asic gds info */ switch (adev->asic_type) {case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; + adev->gds.gds_compute_max_wave_id =0x4ff ;
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev) · line 5139
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; }static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap)
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): b789da4a-2e31-31eb-4706-ea9b6fbc45c1-5C7GfCeVMHo@public.gmane.org
Re: [PATCH 2/2] drm/amdgpu: handle AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID on gfx10
Thanks. I'll push both patches with emit_ib_size updated for this patch.
Marek
Quoted 78 lines · On Thu, Jun 27, 2019 at 3:50 AM zhoucm1 <[email protected]>…
any reason for not care .emit_ib_size in this one?
-David
Review
Quoted 8 lines · Marek Olšák
From: Marek Olšák <[email protected]>
Signed-off-by Marek Olšák <[email protected]>--- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.cdrivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_ring_emit_ib_gfx(struct
Quoted 4 lines
index
6baaa65a1daa..5b807a19bbbf100644drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0@@ -4257,20 +4257,36 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct
amdgpu_ring*ring,drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2
-0}static void gfx_v10_0_ring_emit_ib_compute (struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,uint32_t flags) {unsigned vmid =AMDGPU_JOB_GET_VMID (job);u32 control =INDIRECT_BUFFER_VALID | ib->length_dw | (vmid <<24 ); +/* Currently, there is a high possibility to get wave ID mismatch +* between ME and GDS, leading to a hw deadlock, because ME generates
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +3
-0+* different wave IDs than the GDS expects. This situation happens +* randomly when at least 5 compute pipes use GDS ordered append. +* The wave IDs generated by ME are also wrong after suspend/resume.
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +3
-0+* Those are probably bugs somewhere else in the kernel driver. +* +* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-0+* GDS to 0 for this ring (me/pipe). +*/ +if (ib->flags &AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID ) { + amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG,1));
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_ring_emit_ib_gfx(struct · line 4280
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2
-0+amdgpu_ring_write (ring, mmGDS_COMPUTE_MAX_WAVE_ID); + amdgpu_ring_write(ring,
ring->adev->gds.gds_compute_max_wave_id);drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_rlc_funcs(struct
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2
-0+ } +amdgpu_ring_write (ring,PACKET3 (PACKET3_INDIRECT_BUFFER ,2 ));BUG_ON (ib->gpu_addr &0x3 );/* Dword align */ amdgpu_ring_write (ring, #ifdef __BIG_ENDIAN (2 <<0 ) | #endiflower_32_bits (ib->gpu_addr));amdgpu_ring_write (ring,upper_32_bits (ib->gpu_addr));amdgpu_ring_write (ring, control); }@@ -5103,20 +5119,21 @@ static void gfx_v10_0_set_rlc_funcs(struct
amdgpu_device*adev)Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1
-0} }static void gfx_v10_0_set_gds_init (struct amdgpu_device *adev) {/* init asic gds info */ switch (adev->asic_type) {case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; + adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; }
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev)
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; } static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct
amdgpu_device*adev,Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0u32 bitmap)
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 27.06.19 um 00:35 schrieb Marek Olšák:
Quoted 6 lines
From: Marek Olšák <[email protected]>
v2: update
emit_ib_size
(though it's still wrong because it was wrong before)Signed-off-by Marek Olšák <[email protected]>
Can't judge if this is really the right thing to do because I don't know the details of the hw bug.
But at least of hand I can't see any obvious problems with it, so feel free to add an Acked-by: Christian König <[email protected]> to the series.
Regards,
Christian.
Quoted 106 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 3 ++- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +2
-1@@ -24,21 +24,22 @@#ifndef __AMDGPU_GDS_H__#define __AMDGPU_GDS_H__struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size; -uint32_t gds_compute_max_wave_id; +uint32_t gds_compute_max_wave_id; +uint32_t vgt_gs_max_wave_id; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +11
-3@@ -4211,20 +4211,29 @@ static void gfx_v10_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) }static void gfx_v10_0_ring_emit_ib_gfx (struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,uint32_t flags) {unsigned vmid =AMDGPU_JOB_GET_VMID (job);u32 header, control =0 ; +/* Prevent a hw deadlock due to a wave ID mismatch between ME and GDS. +* This resets the wave ID counters. (needed by transform feedback) +* TODO: This might only be needed on a VMID switch when we change +* the GDS OA mapping, not sure. +*/ +amdgpu_ring_write (ring,PACKET3 (PACKET3_SET_CONFIG_REG ,1 )); +amdgpu_ring_write (ring, mmVGT_GS_MAX_WAVE_ID); +amdgpu_ring_write (ring, ring->adev->gds.vgt_gs_max_wave_id); +if (ib->flags &AMDGPU_IB_FLAG_CE ) header =PACKET3 (PACKET3_INDIRECT_BUFFER_CNST ,2 );else header =PACKET3 (PACKET3_INDIRECT_BUFFER ,2 ); control |= ib->length_dw | (vmid <<24 );if (amdgpu_mcbp && (ib->flags &AMDGPU_IB_FLAG_PREEMPT )) { control |=INDIRECT_BUFFER_PRE_ENB (1 );@@ -4944,21 +4953,21 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {*/ 5 +/* COND_EXEC */ 7 +/* HDP_flush */ 4 +/* VGT_flush */ 14 +/* CE_META */ 31 +/* DE_META */ 3 +/* CNTX_CTRL */ 5 +/* HDP_INVL */ 8 +8 +/* FENCE x2 */ 2, /* SWITCH_BUFFER */ - .emit_ib_size = 4, /* gfx_v10_0_ring_emit_ib_gfx */ + .emit_ib_size = 7, /* gfx_v10_0_ring_emit_ib_gfx */ .emit_ib = gfx_v10_0_ring_emit_ib_gfx, .emit_fence = gfx_v10_0_ring_emit_fence, .emit_pipeline_sync = gfx_v10_0_ring_emit_pipeline_sync, .emit_vm_flush = gfx_v10_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v10_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v10_0_ring_emit_hdp_flush, .test_ring = gfx_v10_0_ring_test_ring, .test_ib = gfx_v10_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib,@@ -5092,24 +5101,23 @@ static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev)default :break ; } }static void gfx_v10_0_set_gds_init (struct amdgpu_device *adev) {/* init asic gds info */ switch (adev->asic_type) {case CHIP_NAVI10 : - adev->gds.gds_size =0x10000 ; -break ;default : adev->gds.gds_size =0x10000 ; + adev->gds.vgt_gs_max_wave_id =0x3ff ;
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev) · line 5123
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; }static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap) {
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx