drm/etnaviv: Backport fix for mmu flushing
Subsystems (3)
PATCH 1/1 drm/etnaviv: Backport fix for mmu flushing
commit 4900dda90af2cb13bc1d4c12ce94b98acc8fe64e upstream
Due to async need_flush updating via other buffer mapping, checking gpu->need_flush in 3 places within etnaviv_buffer_queue can cause GPU hangs.
This occurs due to need_flush being false for the first 2 checks in that function, so that the extra dword does not get accounted for, but by the time we come to check for the third time, gpu->mmu->need_flish is true, which outputs the flush instruction. This causes the prefetch during the final link to be off by 1. This causes GPU hangs.
It causes the ring to contain patterns like this:
0x40000005, /* LINK (8) PREFETCH=0x5,OP=LINK */0x70040010, /* ADDRESS *0x70040010 */0x40000002, /* LINK (8) PREFETCH=0x2,OP=LINK */0x70040000, /* ADDRESS *0x70040000 */0x08010e04, /* LOAD_STATE (1) Base: 0x03810 Size: 1 Fixp: 0 */0x0000001f, /* GL.FLUSH_MMU := FLUSH_FEMMU=1,FLUSH_UNK1=1,FLUSH_UNK2=1,FLUSH_PEMMU=1,FLUSH_UNK4=1 */0x08010e03, /* LOAD_STATE (1) Base: 0x0380C Size: 1 Fixp: 0 */0x00000000, /* GL.FLUSH_CACHE := DEPTH=0,COLOR=0,TEXTURE=0,PE2D=0,TEXTUREVS=0,SHADER_L1=0,SHADER_L2=0,UNK10=0,UNK11=0,DESCRIPTOR_UNK12=0,DESCRIPTOR_UNK13=0 */0x08010e02, /* LOAD_STATE (1) Base: 0x03808 Size: 1 Fixp: 0 */0x00000701, /* GL.SEMAPHORE_TOKEN := FROM=FE,TO=PE,UNK28=0x0 */0x48000000, /* STALL (9) OP=STALL */0x00000701, /* TOKEN FROM=FE,TO=PE,UNK28=0x0 */0x08010e00, /* LOAD_STATE (1) Base: 0x03800 Size: 1 Fixp: 0 */0x00000000, /* GL.PIPE_SELECT := PIPE=PIPE_3D */0x40000035, /* LINK (8) PREFETCH=0x35,OP=LINK */0x70041000, /* ADDRESS *0x70041000 */
Here we see a link with prefetch of 5 dwords starting with the 3rd instruction. It only loads the 5 dwords up and including the final LOAD_STATE. It needs to include the final LINK instruction.
This was seen on imx6q, and the fix is confirmed to stop the GPU hangs.
The commit referenced inadvertently fixed this issue by checking gpu->mmu->need_flush once at the start of the function. Given that this commit is independant, and useful for all version, it seems sensible to backport it to the stable branches.
PATCH 1/1 drm/etnaviv: replace MMU flush marker with flush sequence
From: Lucas Stach <[email protected]>
commit 4900dda90af2cb13bc1d4c12ce94b98acc8fe64e upstream
If a MMU is shared between multiple GPUs, all of them need to flush their TLBs, so a single marker that gets reset on the first flush won't do. Replace the flush marker with a sequence number, so that it's possible to check if the TLB is in sync with the current page table state for each GPU.
--- drivers/gpu/drm/etnaviv/etnaviv_buffer.c | 10 ++++++---- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 2 +- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 1 + drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 8 ++++---- drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 2 +- 5 files changed, 13 insertions(+), 10 deletions(-)
drivers/gpu/drm/etnaviv/etnaviv_buffer.c +6 -4
@@ -258,6 +258,8 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,unsigned int waitlink_offset = buffer->user_size -16 ;u32 return_target, return_dwords;u32 link_target, link_dwords; +unsigned int new_flush_seq =READ_ONCE (gpu->mmu->flush_seq); +bool need_flush = gpu->flush_seq != new_flush_seq;if (drm_debug &DRM_UT_DRIVER )etnaviv_buffer_dump (gpu, buffer,0 ,0x50 );
@@ -270,14 +272,14 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,* need to append a mmu flush load state, followed by a new * link to this buffer - a total of four additional words. */ -if (gpu->mmu->need_flush || gpu->switch_context) { +if (need_flush || gpu->switch_context) {u32 target, extra_dwords;/* link command */ extra_dwords =1 ;/* flush command */ -if (gpu->mmu->need_flush) { +if (need_flush) {if (gpu->mmu->version ==ETNAVIV_IOMMU_V1 ) extra_dwords +=1 ;else
@@ -290,7 +292,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, target =etnaviv_buffer_reserve (gpu, buffer, extra_dwords); -if (gpu->mmu->need_flush) { +if (need_flush) {/* Add the MMU flush */ if (gpu->mmu->version ==ETNAVIV_IOMMU_V1 ) { CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU,
@@ -310,7 +312,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, SYNC_RECIPIENT_PE); } - gpu->mmu->need_flush = false; + gpu->flush_seq = new_flush_seq; }if (gpu->switch_context) {
drivers/gpu/drm/etnaviv/etnaviv_gpu.c +1 -1
@@ -1353,7 +1353,7 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, gpu->active_fence = submit->fence->seqno;if (gpu->lastctx != cmdbuf->ctx) { - gpu->mmu->need_flush = true; + gpu->mmu->flush_seq++; gpu->switch_context = true; gpu->lastctx = cmdbuf->ctx; }
drivers/gpu/drm/etnaviv/etnaviv_gpu.h +1 -0
@@ -138,6 +138,7 @@ struct etnaviv_gpu {struct etnaviv_iommu *mmu;struct etnaviv_cmdbuf_suballoc *cmdbuf_suballoc; +unsigned int flush_seq;/* Power Control: */ struct clk *clk_bus;
drivers/gpu/drm/etnaviv/etnaviv_mmu.c +4 -4
@@ -132,7 +132,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu,*/ if (mmu->last_iova) { mmu->last_iova =0 ; - mmu->need_flush = true; + mmu->flush_seq++;continue ; }
@@ -246,7 +246,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, }list_add_tail (&mapping->mmu_node, &mmu->mappings); - mmu->need_flush = true; + mmu->flush_seq++;mutex_unlock (&mmu->lock);return ret;
@@ -264,7 +264,7 @@ void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,etnaviv_iommu_remove_mapping (mmu, mapping);list_del (&mapping->mmu_node); - mmu->need_flush = true; + mmu->flush_seq++;mutex_unlock (&mmu->lock); }
@@ -346,7 +346,7 @@ int etnaviv_iommu_get_suballoc_va(struct etnaviv_gpu *gpu, dma_addr_t paddr,return ret; } mmu->last_iova = vram_node->start + size; - gpu->mmu->need_flush = true; + mmu->flush_seq++;mutex_unlock (&mmu->lock); *iova = (u32 )vram_node->start;
drivers/gpu/drm/etnaviv/etnaviv_mmu.h +1 -1
@@ -44,7 +44,7 @@ struct etnaviv_iommu {struct list_head mappings;struct drm_mm mm;u32 last_iova; -bool need_flush; +unsigned int flush_seq; };struct etnaviv_gem_object ;
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH 1/1 drm/etnaviv: replace MMU flush marker with flush sequence
In reply to Robert BeckettFrom: Lucas Stach <[email protected]>
commit 4900dda90af2cb13bc1d4c12ce94b98acc8fe64e upstream
If a MMU is shared between multiple GPUs, all of them need to flush their TLBs, so a single marker that gets reset on the first flush won't do. Replace the flush marker with a sequence number, so that it's possible to check if the TLB is in sync with the current page table state for each GPU.
--- drivers/gpu/drm/etnaviv/etnaviv_buffer.c | 10 ++++++---- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 1 + drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 6 +++--- drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-)
drivers/gpu/drm/etnaviv/etnaviv_buffer.c +6 -4
@@ -311,6 +311,8 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state,u32 return_target, return_dwords;u32 link_target, link_dwords;bool switch_context = gpu->exec_state != exec_state; +unsigned int new_flush_seq =READ_ONCE (gpu->mmu->flush_seq); +bool need_flush = gpu->flush_seq != new_flush_seq;lockdep_assert_held (&gpu->lock);
@@ -325,14 +327,14 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state,* need to append a mmu flush load state, followed by a new * link to this buffer - a total of four additional words. */ -if (gpu->mmu->need_flush || switch_context) { +if (need_flush || switch_context) {u32 target, extra_dwords;/* link command */ extra_dwords =1 ;/* flush command */ -if (gpu->mmu->need_flush) { +if (need_flush) {if (gpu->mmu->version ==ETNAVIV_IOMMU_V1 ) extra_dwords +=1 ;else
@@ -345,7 +347,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, target =etnaviv_buffer_reserve (gpu, buffer, extra_dwords); -if (gpu->mmu->need_flush) { +if (need_flush) {/* Add the MMU flush */ if (gpu->mmu->version ==ETNAVIV_IOMMU_V1 ) { CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU,
@@ -365,7 +367,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, SYNC_RECIPIENT_PE); } - gpu->mmu->need_flush = false; + gpu->flush_seq = new_flush_seq; }if (switch_context) {
drivers/gpu/drm/etnaviv/etnaviv_gpu.h +1 -0
@@ -139,6 +139,7 @@ struct etnaviv_gpu {struct etnaviv_iommu *mmu;struct etnaviv_cmdbuf_suballoc *cmdbuf_suballoc; +unsigned int flush_seq;/* Power Control: */ struct clk *clk_bus;
drivers/gpu/drm/etnaviv/etnaviv_mmu.c +3 -3
@@ -261,7 +261,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, }list_add_tail (&mapping->mmu_node, &mmu->mappings); - mmu->need_flush = true; + mmu->flush_seq++; unlock:mutex_unlock (&mmu->lock);
@@ -280,7 +280,7 @@ void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,etnaviv_iommu_remove_mapping (mmu, mapping);list_del (&mapping->mmu_node); - mmu->need_flush = true; + mmu->flush_seq++;mutex_unlock (&mmu->lock); }
@@ -357,7 +357,7 @@ int etnaviv_iommu_get_suballoc_va(struct etnaviv_gpu *gpu, dma_addr_t paddr,mutex_unlock (&mmu->lock);return ret; } - gpu->mmu->need_flush = true; + mmu->flush_seq++;mutex_unlock (&mmu->lock); *iova = (u32 )vram_node->start;
drivers/gpu/drm/etnaviv/etnaviv_mmu.h +1 -1
@@ -48,7 +48,7 @@ struct etnaviv_iommu {struct mutex lock;struct list_head mappings;struct drm_mm mm; -bool need_flush; +unsigned int flush_seq; };struct etnaviv_gem_object ;
Signature · 1 line
2.20.1
PATCH 1/1 drm/etnaviv: replace MMU flush marker with flush sequence
In reply to Robert BeckettFrom: Lucas Stach <[email protected]>
commit 4900dda90af2cb13bc1d4c12ce94b98acc8fe64e upstream
If a MMU is shared between multiple GPUs, all of them need to flush their TLBs, so a single marker that gets reset on the first flush won't do. Replace the flush marker with a sequence number, so that it's possible to check if the TLB is in sync with the current page table state for each GPU.
--- drivers/gpu/drm/etnaviv/etnaviv_buffer.c | 10 ++++++---- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 2 +- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 1 + drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 6 +++--- drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-)
drivers/gpu/drm/etnaviv/etnaviv_buffer.c +6 -4
@@ -257,6 +257,8 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,unsigned int waitlink_offset = buffer->user_size -16 ;u32 return_target, return_dwords;u32 link_target, link_dwords; +unsigned int new_flush_seq =READ_ONCE (gpu->mmu->flush_seq); +bool need_flush = gpu->flush_seq != new_flush_seq;if (drm_debug &DRM_UT_DRIVER )etnaviv_buffer_dump (gpu, buffer,0 ,0x50 );
@@ -269,14 +271,14 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,* need to append a mmu flush load state, followed by a new * link to this buffer - a total of four additional words. */ -if (gpu->mmu->need_flush || gpu->switch_context) { +if (need_flush || gpu->switch_context) {u32 target, extra_dwords;/* link command */ extra_dwords =1 ;/* flush command */ -if (gpu->mmu->need_flush) { +if (need_flush) {if (gpu->mmu->version ==ETNAVIV_IOMMU_V1 ) extra_dwords +=1 ;else
@@ -289,7 +291,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, target =etnaviv_buffer_reserve (gpu, buffer, extra_dwords); -if (gpu->mmu->need_flush) { +if (need_flush) {/* Add the MMU flush */ if (gpu->mmu->version ==ETNAVIV_IOMMU_V1 ) { CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU,
@@ -309,7 +311,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, SYNC_RECIPIENT_PE); } - gpu->mmu->need_flush = false; + gpu->flush_seq = new_flush_seq; }if (gpu->switch_context) {
drivers/gpu/drm/etnaviv/etnaviv_gpu.c +1 -1
@@ -1313,7 +1313,7 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, gpu->active_fence = submit->fence;if (gpu->lastctx != cmdbuf->ctx) { - gpu->mmu->need_flush = true; + gpu->mmu->flush_seq++; gpu->switch_context = true; gpu->lastctx = cmdbuf->ctx; }
drivers/gpu/drm/etnaviv/etnaviv_gpu.h +1 -0
@@ -135,6 +135,7 @@ struct etnaviv_gpu {int irq;struct etnaviv_iommu *mmu; +unsigned int flush_seq;/* Power Control: */ struct clk *clk_bus;
drivers/gpu/drm/etnaviv/etnaviv_mmu.c +3 -3
@@ -134,7 +134,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu,*/ if (mmu->last_iova) { mmu->last_iova =0 ; - mmu->need_flush = true; + mmu->flush_seq++;continue ; }
@@ -197,7 +197,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu,* associated commit requesting this mapping, and retry the * allocation one more time. */ - mmu->need_flush = true; + mmu->flush_seq++; }return ret;
@@ -354,7 +354,7 @@ u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu,* that the FE MMU prefetch won't load invalid entries. */ mmu->last_iova = buf->vram_node.start + buf->size +SZ_64K ; - gpu->mmu->need_flush = true; + mmu->flush_seq++;mutex_unlock (&mmu->lock);return (u32 )buf->vram_node.start;
drivers/gpu/drm/etnaviv/etnaviv_mmu.h +1 -1
@@ -44,7 +44,7 @@ struct etnaviv_iommu {struct list_head mappings;struct drm_mm mm;u32 last_iova; -bool need_flush; +unsigned int flush_seq; };struct etnaviv_gem_object ;
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Lucas Stach
maintainerAm Donnerstag, den 02.04.2020, 18:07 +0100 schrieb Robert Beckett:
Quoted 11 lines
commit
4900dda90af2cb13bc1d4c12ce94b98acc8fe64eupstreamDue to async
need_flushupdating via other buffer mapping, checkinggpu->need_flushin 3 places withinetnaviv_buffer_queuecan cause GPU hangs.This occurs due to
need_flushbeing false for the first 2 checks in that function, so that the extra dword does not get accounted for, but by the time we come to check for the third time,gpu->mmu->need_flishis true, which outputs the flush instruction. This causes the prefetch during the final link to be off by 1. This causes GPU hangs.
Yep, there should have been a READ_ONCE on this state. :/
Quoted 23 lines
It causes the ring to contain patterns like this:
0x40000005, /* LINK (8) PREFETCH=0x5,OP=LINK */0x70040010, /* ADDRESS *0x70040010 */0x40000002, /* LINK (8) PREFETCH=0x2,OP=LINK */0x70040000, /* ADDRESS *0x70040000 */0x08010e04, /*LOAD_STATE(1) Base:0x03810Size: 1 Fixp: 0 */0x0000001f, /* GL.FLUSH_MMU := FLUSH_FEMMU=1,FLUSH_UNK1=1,FLUSH_UNK2=1,FLUSH_PEMMU=1,FLUSH_UNK4=1 */0x08010e03, /*LOAD_STATE(1) Base:0x0380CSize: 1 Fixp: 0 */0x00000000, /* GL.FLUSH_CACHE := DEPTH=0,COLOR=0,TEXTURE=0,PE2D=0,TEXTUREVS=0,SHADER_L1=0,SHADER_L2=0,UNK10=0,UNK11=0,DESCRIPTOR_UNK12=0,DESCRIPTOR_UNK13=0 */0x08010e02, /*LOAD_STATE(1) Base:0x03808Size: 1 Fixp: 0 */0x00000701, /* GL.SEMAPHORE_TOKEN := FROM=FE,TO=PE,UNK28=0x0 */0x48000000, /* STALL (9) OP=STALL */0x00000701, /* TOKEN FROM=FE,TO=PE,UNK28=0x0 */0x08010e00, /*LOAD_STATE(1) Base:0x03800Size: 1 Fixp: 0 */0x00000000, /* GL.PIPE_SELECT := PIPE=PIPE_3D */0x40000035, /* LINK (8) PREFETCH=0x35,OP=LINK */0x70041000, /* ADDRESS *0x70041000 */Here we see a link with prefetch of 5 dwords starting with the 3rd instruction. It only loads the 5 dwords up and including the final
LOAD_STATE. It needs to include the final LINK instruction.
Quoted 6 lines
This was seen on imx6q, and the fix is confirmed to stop the GPU hangs.
The commit referenced inadvertently fixed this issue by checking
gpu->mmu->need_flushonce at the start of the function. Given that this commit is independant, and useful for all version, it seems sensible to backport it to the stable branches.
I agree. Without shared MMUs this doesn't really need to be sequence, but better just to backport this change, which has seen quite some testing, than creating yet another, slightly different, version of this function in the stable branches.
Regards,
Lucas
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 35 lines · Robert Beckett
commit
4900dda90af2cb13bc1d4c12ce94b98acc8fe64eupstreamDue to async
need_flushupdating via other buffer mapping, checkinggpu->need_flushin 3 places withinetnaviv_buffer_queuecan cause GPU hangs.This occurs due to
need_flushbeing false for the first 2 checks in that function, so that the extra dword does not get accounted for, but by the time we come to check for the third time,gpu->mmu->need_flishis true, which outputs the flush instruction. This causes the prefetch during the final link to be off by 1. This causes GPU hangs.It causes the ring to contain patterns like this:
0x40000005, /* LINK (8) PREFETCH=0x5,OP=LINK */0x70040010, /* ADDRESS *0x70040010 */0x40000002, /* LINK (8) PREFETCH=0x2,OP=LINK */0x70040000, /* ADDRESS *0x70040000 */0x08010e04, /*LOAD_STATE(1) Base:0x03810Size: 1 Fixp: 0 */0x0000001f, /* GL.FLUSH_MMU := FLUSH_FEMMU=1,FLUSH_UNK1=1,FLUSH_UNK2=1,FLUSH_PEMMU=1,FLUSH_UNK4=1 */0x08010e03, /*LOAD_STATE(1) Base:0x0380CSize: 1 Fixp: 0 */0x00000000, /* GL.FLUSH_CACHE := DEPTH=0,COLOR=0,TEXTURE=0,PE2D=0,TEXTUREVS=0,SHADER_L1=0,SHADER_L2=0,UNK10=0,UNK11=0,DESCRIPTOR_UNK12=0,DESCRIPTOR_UNK13=0 */0x08010e02, /*LOAD_STATE(1) Base:0x03808Size: 1 Fixp: 0 */0x00000701, /* GL.SEMAPHORE_TOKEN := FROM=FE,TO=PE,UNK28=0x0 */0x48000000, /* STALL (9) OP=STALL */0x00000701, /* TOKEN FROM=FE,TO=PE,UNK28=0x0 */0x08010e00, /*LOAD_STATE(1) Base:0x03800Size: 1 Fixp: 0 */0x00000000, /* GL.PIPE_SELECT := PIPE=PIPE_3D */0x40000035, /* LINK (8) PREFETCH=0x35,OP=LINK */0x70041000, /* ADDRESS *0x70041000 */Here we see a link with prefetch of 5 dwords starting with the 3rd instruction. It only loads the 5 dwords up and including the final
LOAD_STATE. It needs to include the final LINK instruction.
Quoted 8 lines
This was seen on imx6q, and the fix is confirmed to stop the GPU hangs.
The commit referenced inadvertently fixed this issue by checking
gpu->mmu->need_flushonce at the start of the function. Given that this commit is independant, and useful for all version, it seems sensible to backport it to the stable branches.
All now queued up, thanks for the backports.
greg k-h