Make staging driver stable for SRIOV VF (1)
Subsystems (2)
This is the first patch series to make latest staging driver stable for SRIOV VF on both Tonga and Vega. Patches are merged from SRIOV branches or reimplemented, including bug fixes and small features requested by SRIOV users.
Please help reviewing, Thanks.
[PATCH 1/4] drm/amdgpu: always consider virualised device for [PATCH 2/4] drm/amdgpu: workaround for VM fault caused by SDMA [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 1/4 drm/amdgpu: always consider virualised device for checking post
In reply to a message not in the archiveFrom: pding <[email protected]>
The post checking on scratch registers isn't reliable for virtual function.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +4 -4
@@ -669,7 +669,7 @@ void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)* or post is needed if hw reset is performed. * Returns true if need or false if not. */ -bool amdgpu_need_post (struct amdgpu_device *adev) +static bool amdgpu_check_post (struct amdgpu_device *adev) {uint32_t reg;
@@ -692,7 +692,7 @@ bool amdgpu_need_post(struct amdgpu_device *adev) } -static bool amdgpu_vpost_needed (struct amdgpu_device *adev) +bool amdgpu_need_post (struct amdgpu_device *adev) {if (amdgpu_sriov_vf (adev))return false;
@@ -716,7 +716,7 @@ static bool amdgpu_vpost_needed(struct amdgpu_device *adev)return true; } } -return amdgpu_need_post (adev); +return amdgpu_check_post (adev); }/**
@@ -2208,7 +2208,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,amdgpu_device_detect_sriov_bios (adev);/* Post card if necessary */ -if (amdgpu_vpost_needed (adev)) { +if (amdgpu_need_post (adev)) {if (!adev->bios) {dev_err (adev->dev,"no vBIOS found\n" );amdgpu_vf_error_put (AMDGIM_ERROR_VF_NO_VBIOS ,0 ,0 );
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/4 drm/amdgpu: workaround for VM fault caused by SDMA set_wptr
In reply to a message not in the archiveFrom: pding <[email protected]>
The polling memory was standalone in VRAM before, so the HDP flush introduced latency that hides a VM fault issue. Now polling memory leverages the WB in system memory and HDP flush is not required, the VM fault at same page happens.
Add delay back to workaround until the root cause is found.
Tests: VP1 or launch 40 instances of glxinfo at the same time.
--- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 3 +++ 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +3 -0
@@ -381,6 +381,9 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)if (ring->use_doorbell) {/* XXX check if swapping is necessary on BE */ adev->wb.wb[ring->wptr_offs] =lower_32_bits (ring->wptr) <<2 ; +/* workaround: VM fault always happen at page 2046 */ +if (amdgpu_sriov_vf (adev)) +udelay (500 );WDOORBELL32 (ring->doorbell_index,lower_32_bits (ring->wptr) <<2 ); }else {int me = (ring == &ring->adev->sdma.instance[0 ].ring) ?0 :1 ;
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/4 drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveFrom: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +7 -0
@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ; + +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 ))); + }return 0 ; }
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 4/4 drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -6
@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret =readl (((void __iomem *)adev->rmmio) + (reg *4 ));
@@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v; } -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg (adev, reg, v); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX ))writel (v, ((void __iomem *)adev->rmmio) + (reg *4 ));
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -1
@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1 -0
@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1 -0
@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,void amdgpu_fence_driver_suspend (struct amdgpu_device *adev);void amdgpu_fence_driver_resume (struct amdgpu_device *adev);int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void amdgpu_fence_process (struct amdgpu_ring *ring);int amdgpu_fence_wait_empty (struct amdgpu_ring *ring);unsigned amdgpu_fence_count_emitted (struct amdgpu_ring *ring);
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +34 -21
93 lines not on this page · raw message
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveAm 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;
That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));
Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 657
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveThanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
Pixel
Quoted 31 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
Quoted 11 lines
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveQuoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to fence_offs in the GFX8 nor GFX9 code we have in amd-staging-drm-next.
And if it doesn't depend on the fence_offs in the ring printing it like this is just pure speculation.
Regards,
Christian.
Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 17 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
Pixel
Quoted 41 lines
Quoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveMy understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
Pixel
Quoted 5 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.
Quoted 66 lines
And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveThis patch is not for implementation of MCBP handled by driver itself. In SRIOV use case the preemption is handle in host layer. What I do here is just check if the preemption occurs, so there’s no code to handle it.
—
Sincerely Yours,
Pixel
Quoted 18 lines · Pixel
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
Pixel
Quoted 73 lines
Quoted 71 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveHi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.
Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 18 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
Pixel
Quoted 71 lines
Quoted 69 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 57 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 38 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveAm 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 22 lines
From: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu.h struct amdgpu_kiq { · line 888
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex;
You can remove the ring_mutex if you don't use it any more.
Quoted 74 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0+spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-6@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret =readl (((void __iomem *)adev->rmmio) + (reg *4 ));@@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v; } -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg (adev, reg, v); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX ))writel (v, ((void __iomem *)adev->rmmio) + (reg *4 ));drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,void amdgpu_fence_driver_suspend (struct amdgpu_device *adev);void amdgpu_fence_driver_resume (struct amdgpu_device *adev);int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void amdgpu_fence_process (struct amdgpu_ring *ring);int amdgpu_fence_wait_empty (struct amdgpu_ring *ring);unsigned amdgpu_fence_count_emitted (struct amdgpu_ring *ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c line 25
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */
You want to busy wait 100 seconds for an result?
Forget it that is way to long you will certainly run into problems with the NMI much earlier.
Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3int amdgpu_allocate_static_csa (struct amdgpu_device *adev) {@@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)uint32_t amdgpu_virt_kiq_rreg (struct amdgpu_device *adev,uint32_t reg) { -signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 129
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-2BUG_ON (!ring->funcs->emit_rreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_rreg (ring, reg); -amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );
Do you still need to send an interrupt when you busy wait? I don't think so.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 136
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +6
-6amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 );
You need to correctly handle sequence wrap around here.
Quoted 31 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10
-4+ +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0 ; } - val = adev->wb.wb[adev->virt.reg_val_offs];return val;@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; + amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) · line 172
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8
-1+ wait_seq, AMDGPU_FENCE_FLAG_INT);amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 );
Same here. It would probably be better to have a function for this in amdgpu_fence.c
Regards,
Christian.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) · line 178
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-4- r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }/**
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 2/4] drm/amdgpu: workaround for VM fault caused by SDMA set_wptr
In reply to a message not in the archiveAm 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 21 lines
From: pding <[email protected]>
The polling memory was standalone in VRAM before, so the HDP flush introduced latency that hides a VM fault issue. Now polling memory leverages the WB in system memory and HDP flush is not required, the VM fault at same page happens.
Add delay back to workaround until the root cause is found.
Tests: VP1 or launch 40 instances of glxinfo at the same time.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 3 +++ 1 file changed, 3 insertions(+)drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +0
-0
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring) · line 386
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +3
-0@@ -381,6 +381,9 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)if (ring->use_doorbell) {/* XXX check if swapping is necessary on BE */ adev->wb.wb[ring->wptr_offs] =lower_32_bits (ring->wptr) <<2 ; +/* workaround: VM fault always happen at page 2046 */ +if (amdgpu_sriov_vf (adev)) +udelay (500 );
Have you tried using a memory barrier here?
That looks like it will have massive impact on performance.
Regards,
Christian.
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring) · line 389
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +0
-0WDOORBELL32 (ring->doorbell_index,lower_32_bits (ring->wptr) <<2 ); }else {int me = (ring == &ring->adev->sdma.instance[0 ].ring) ?0 :1 ;
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveOK I get it…
when we use the fence_offs to submit fence to HW, in fact it’s a 8 DW fence not a 2 DW.
The format is:
Completed Fence 0x0 Fence written here if frame completed normally Preempted Fence 0x2 Bit set in CP_VMID_PREEMPT and preemption occurred Reset Fence 0x4 Bit is set in CP_VMID_RESET and reset occurred
Preempted then Reset 0x6 Both preemption and reset occurred
Then I notice that the 8DW wb allocation patch is missed on staging driver.
Hi Monk,
Can you take a look? I thought the 8DW WB patch is already upstream.
—
Sincerely Yours,
Pixel
Quoted 12 lines · Christian
Hi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
Quoted 98 lines
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 89 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 69 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 57 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 38 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archivePixel,
On drm-next, we always allocate 8DW for all WB, you can check the wb_get routine on detail
BR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:19
To: Koenig, Christian <[email protected]>; [email protected]; Liu, Monk <[email protected]>
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
OK I get it…
when we use the fence_offs to submit fence to HW, in fact it’s a 8 DW fence not a 2 DW.
The format is:
Completed Fence 0x0 Fence written here if frame completed normally Preempted Fence 0x2 Bit set in CP_VMID_PREEMPT and preemption occurred Reset Fence 0x4 Bit is set in CP_VMID_RESET and reset occurred
Preempted then Reset 0x6 Both preemption and reset occurred
Then I notice that the 8DW wb allocation patch is missed on staging driver.
Hi Monk,
Can you take a look? I thought the 8DW WB patch is already upstream.
—
Sincerely Yours,
Pixel
Quoted 12 lines · Christian
Hi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
Quoted 98 lines
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 89 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 69 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 57 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 38 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 2/4] drm/amdgpu: workaround for VM fault caused by SDMA set_wptr
In reply to a message not in the archiveYes I tried smp_mb but it doesn’t help…
We will follow up this issue continuously until fix the root cause. —
Sincerely Yours,
Pixel
Quoted 31 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 21 lines
From: pding <[email protected]>
The polling memory was standalone in VRAM before, so the HDP flush introduced latency that hides a VM fault issue. Now polling memory leverages the WB in system memory and HDP flush is not required, the VM fault at same page happens.
Add delay back to workaround until the root cause is found.
Tests: VP1 or launch 40 instances of glxinfo at the same time.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 3 +++ 1 file changed, 3 insertions(+)drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +0
-0drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring) · line 386
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +3
-0@@ -381,6 +381,9 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)if (ring->use_doorbell) {/* XXX check if swapping is necessary on BE */ adev->wb.wb[ring->wptr_offs] =lower_32_bits (ring->wptr) <<2 ; +/* workaround: VM fault always happen at page 2046 */ +if (amdgpu_sriov_vf (adev)) +udelay (500 );Have you tried using a memory barrier here?
Quoted 10 lines
That looks like it will have massive impact on performance.
Regards,
Christian.Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +0
-0WDOORBELL32 (ring->doorbell_index,lower_32_bits (ring->wptr) <<2 ); }else {int me = (ring == &ring->adev->sdma.instance[0 ].ring) ?0 :1 ;
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveYes I see the “drm/amdgpu: use 256 bit buffers for all wb allocations (v2)”.
Hi Christian,
So it seems all good, right?
—
Sincerely Yours,
Pixel
Quoted 44 lines · Monk
Pixel,
On drm-next, we always allocate 8DW for all WB, you can check the
wb_getroutine on detailBR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:19
To: Koenig, Christian <[email protected]>; [email protected]; Liu, Monk <[email protected]>
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence viaamdgpu_fence_infoOK I get it…
when we use thefence_offsto submit fence to HW, in fact it’s a 8 DW fence not a 2 DW.The format is:
Completed Fence 0x0 Fence written here if frame completed normally Preempted Fence 0x2 Bit set in CP_VMID_PREEMPT and preemption occurred Reset Fence 0x4 Bit is set in CP_VMID_RESET and reset occurredPreempted then Reset
0x6Both preemption and reset occurredThen I notice that the 8DW wb allocation patch is missed on staging driver.
Hi Monk,
Can you take a look? I thought the 8DW WB patch is already upstream.
—
Sincerely Yours,
Pixel
Quoted 112 lines
Quoted 110 lines · Christian
Hi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 89 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 69 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 57 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 38 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveYes,
CP hw assume the fence is 64bit length, so there are 4 type of fence and need 256bits totally
The preempted fence is at offset 64bit,
And I assume that we print all of those fence type when under SRIOV mode
BR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:19
To: Koenig, Christian <[email protected]>; [email protected]; Liu, Monk <[email protected]>
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
OK I get it…
when we use the fence_offs to submit fence to HW, in fact it’s a 8 DW fence not a 2 DW.
The format is:
Completed Fence 0x0 Fence written here if frame completed normally Preempted Fence 0x2 Bit set in CP_VMID_PREEMPT and preemption occurred Reset Fence 0x4 Bit is set in CP_VMID_RESET and reset occurred
Preempted then Reset 0x6 Both preemption and reset occurred
Then I notice that the 8DW wb allocation patch is missed on staging driver.
Hi Monk,
Can you take a look? I thought the 8DW WB patch is already upstream.
—
Sincerely Yours,
Pixel
Quoted 12 lines · Christian
Hi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
Quoted 98 lines
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 89 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 69 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 57 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 38 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 14 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveWB _init should clear this buffer in very early stage, so it should output 0 if for BARE-METAL or no preemption occurred
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian K?nig
Sent: 2017年10月13日 17:10
To: Ding, Pixel <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
Hi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.
Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 18 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
Pixel
Quoted 72 lines
Quoted 70 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 15 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archive@Ding, Pixel
+ seq_printf(m, "Last preempted 0x%08x\n",
+ le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
Please handle other type fence as well:
Preempted fence
Reset fence
Reset and preempted fence
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian K?nig
Sent: 2017年10月13日 17:10
To: Ding, Pixel <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
Hi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.
Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 18 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
Pixel
Quoted 72 lines
Quoted 70 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 15 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archivePixel
I don't think this will work well, my suggestion is you add a new function like:amdgpu_wreg_kiq_busy(),
which will write registers through KIQ and use polling/busy wait, and the original amdgu_wreg_no_kiq() can be still there.
When you need to disable sleep like in IRQ CONTEXT, you can call wreg_kiq_busy() or wreg_no_kiq(),
But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
From: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -3
@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;
}
- if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
- BUG_ON(in_interrupt());
+ if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
return amdgpu_virt_kiq_wreg(adev, reg, v);
- }
if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -1
@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1 -0
@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1 -0
@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring); void amdgpu_fence_process(struct amdgpu_ring *ring); int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index ab05121..9757df1 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -1
@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) {
- signed long r;
- uint32_t val;
- struct dma_fence *f;
+ signed long timeout = usecs_to_jiffies(MAX_KIQ_REG_WAIT);
+ unsigned long flags;
+ uint32_t val, seq, wait_seq;
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
struct amdgpu_ring *ring = &kiq->ring;
BUG_ON(!ring->funcs->emit_rreg);
- mutex_lock(&kiq->ring_mutex);
+ spin_lock_irqsave(&kiq->ring_lock, flags);
amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);
- amdgpu_fence_emit(ring, &f);
+ wait_seq = ++ring->fence_drv.sync_seq;
+ amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
+ wait_seq, AMDGPU_FENCE_FLAG_INT);
amdgpu_ring_commit(ring);
- mutex_unlock(&kiq->ring_mutex);
-
- r = dma_fence_wait_timeout(f, false, msecs_to_jiffies(MAX_KIQ_REG_WAIT));
- dma_fence_put(f);
- if (r < 1) {
- DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
+ do {
+ seq = amdgpu_fence_read(ring);
+ udelay(5);
+ timeout -= 5;
+ } while (seq < wait_seq && timeout > 0);
+
+ if (timeout < 1) {
+ DRM_ERROR("wait for kiq fence error: %ld\n", timeout);
return ~0;
}
-
val = adev->wb.wb[adev->virt.reg_val_offs];
return val;
@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) {
- signed long r;
+ signed long timeout = usecs_to_jiffies(MAX_KIQ_REG_WAIT);
+ unsigned long flags;
+ uint32_t seq, wait_seq;
struct dma_fence *f;
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
struct amdgpu_ring *ring = &kiq->ring;
BUG_ON(!ring->funcs->emit_wreg);
- mutex_lock(&kiq->ring_mutex);
+ spin_lock_irqsave(&kiq->ring_lock, flags);
amdgpu_ring_alloc(ring, 32);
amdgpu_ring_emit_wreg(ring, reg, v);
- amdgpu_fence_emit(ring, &f);
+ /* amdgpu_fence_emit(ring, &f); */
+ wait_seq = ++ring->fence_drv.sync_seq;
+ amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
+ wait_seq, AMDGPU_FENCE_FLAG_INT);
amdgpu_ring_commit(ring);
- mutex_unlock(&kiq->ring_mutex);
+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+ do {
+ seq = amdgpu_fence_read(ring);
+ udelay(5);
+ timeout -= 5;
+ } while (seq < wait_seq && timeout > 0);
- r = dma_fence_wait_timeout(f, false, msecs_to_jiffies(MAX_KIQ_REG_WAIT));
- if (r < 1)
- DRM_ERROR("wait for kiq fence error: %ld.\n", r);
- dma_fence_put(f);
+ if (timeout < 1)
+ DRM_ERROR("wait for kiq fence error: %ld\n", timeout);
}
/**
--
2.9.5
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveQuoted 18 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 17 lines
From: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h struct amdgpu_kiq { · line 888
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex;You can remove the
ring_mutexif you don't use it any more.
[Pixel] KFD still use the mutex, I didn’t change it also to spin lock, should I?
Quoted 104 lines
Quoted 75 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0+spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-6@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret =readl (((void __iomem *)adev->rmmio) + (reg *4 ));@@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v; } -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg (adev, reg, v); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX ))writel (v, ((void __iomem *)adev->rmmio) + (reg *4 ));drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,void amdgpu_fence_driver_suspend (struct amdgpu_device *adev);void amdgpu_fence_driver_resume (struct amdgpu_device *adev);int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void amdgpu_fence_process (struct amdgpu_ring *ring);int amdgpu_fence_wait_empty (struct amdgpu_ring *ring);unsigned amdgpu_fence_count_emitted (struct amdgpu_ring *ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c line 25
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ You want to busy wait 100 seconds for an result?
Forget it that is way to long you will certainly run into problems with the NMI much earlier.
[Pixel] thanks, would change it to 10s.Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3int amdgpu_allocate_static_csa (struct amdgpu_device *adev) {@@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)uint32_t amdgpu_virt_kiq_rreg (struct amdgpu_device *adev,uint32_t reg) { -signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_rreg);
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 129
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-2-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_rreg (ring, reg); -amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );Do you still need to send an interrupt when you busy wait? I don't think so.
[Pixel]I want to let the IRQ handler to update the last_seq in case of that KIQ is used by other functionality with fence. Is it necessary?
Quoted 16 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-2amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); -drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 136
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +6
-4- r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 );You need to correctly handle sequence wrap around here.
[Pixel] yes thanks, but I didn’t see similar case handling in normal fence handling codepath, Is there? So I assumed 32bit fence seq could not overflow for years.
Quoted 37 lines
Quoted 37 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +13
-5+ +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0 ; } - val = adev->wb.wb[adev->virt.reg_val_offs];return val;@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 136
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +5
-0+do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 );Same here. It would probably be better to have a function for this in
amdgpu_fence.c
[Pixel] get it, will add a wrapper.
Quoted 13 lines
Regards,
Christian.Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-4- r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) · line 177
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-0/** —
Sincerely Yours,
Pixel
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveGet it.
—
Sincerely Yours,
Pixel
Quoted 129 lines · Monk
@Ding, Pixel
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));Please handle other type fence as well:
Preempted fence
Reset fence
Reset and preempted fenceBR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian K?nig
Sent: 2017年10月13日 17:10
To: Ding, Pixel <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence viaamdgpu_fence_infoHi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 18 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 72 lines
Quoted 70 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 15 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveI’m afraid there’s racing issue if polling and IRQ use cases are mixed at the same time.
The original implementation is as your suggested. Is there any benefit to keep to sleepy version? —
Sincerely Yours,
Pixel
Quoted 202 lines · Monk
Pixel
I don't think this will work well, my suggestion is you add a new function like:
amdgpu_wreg_kiq_busy(),which will write registers through KIQ and use polling/busy wait, and the original
amdgu_wreg_no_kiq()can be still there.When you need to disable sleep like in IRQ CONTEXT, you can call
wreg_kiq_busy()orwreg_no_kiq(),But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-3@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;}drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-2-if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg(adev, reg, v);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-1- }if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.cindex 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void
amdgpu_fence_process(struct amdgpu_ring *ring); intamdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsignedamdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cindex ab05121..9757df1 100644drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_tamdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg){drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3-signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_rreg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-1-amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit(ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-6-mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); + +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0; } - val = adev->wb.wb[adev->virt.reg_val_offs]; return val;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-9@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }/** -- 2.9.5
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveThe busy waiting method somehow make the performance worse considering 16 VF and vf0 need to wait 15*time_slice when it try to access registers
BR
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:39
To: Liu, Monk <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
I’m afraid there’s racing issue if polling and IRQ use cases are mixed at the same time.
The original implementation is as your suggested. Is there any benefit to keep to sleepy version? —
Sincerely Yours,
Pixel
Quoted 202 lines · Monk
Pixel
I don't think this will work well, my suggestion is you add a new function like:
amdgpu_wreg_kiq_busy(),which will write registers through KIQ and use polling/busy wait, and the original
amdgu_wreg_no_kiq()can be still there.When you need to disable sleep like in IRQ CONTEXT, you can call
wreg_kiq_busy()orwreg_no_kiq(),But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-3@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;}drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-2-if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg(adev, reg, v);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-1- }if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.cindex 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void
amdgpu_fence_process(struct amdgpu_ring *ring); intamdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsignedamdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cindex ab05121..9757df1 100644drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_tamdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg){drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3-signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_rreg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-1-amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit(ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-6-mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); + +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0; } - val = adev->wb.wb[adev->virt.reg_val_offs]; return val;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-9@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }/** -- 2.9.5
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveWhy there will be racing issue ?
Polling or sleep wait only have different result for the caller, not the job scheduled to KIQ
The sleep waiting is synchroniz sleep, it just release CPU resource to other process/thread, so the order is guaranteed
BR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:39
To: Liu, Monk <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
I’m afraid there’s racing issue if polling and IRQ use cases are mixed at the same time.
The original implementation is as your suggested. Is there any benefit to keep to sleepy version? —
Sincerely Yours,
Pixel
Quoted 202 lines · Monk
Pixel
I don't think this will work well, my suggestion is you add a new function like:
amdgpu_wreg_kiq_busy(),which will write registers through KIQ and use polling/busy wait, and the original
amdgu_wreg_no_kiq()can be still there.When you need to disable sleep like in IRQ CONTEXT, you can call
wreg_kiq_busy()orwreg_no_kiq(),But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-3@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;}drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-2-if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg(adev, reg, v);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-1- }if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.cindex 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void
amdgpu_fence_process(struct amdgpu_ring *ring); intamdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsignedamdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cindex ab05121..9757df1 100644drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_tamdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg){drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3-signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_rreg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-1-amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit(ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-6-mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); + +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0; } - val = adev->wb.wb[adev->virt.reg_val_offs]; return val;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-9@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }/** -- 2.9.5
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveSounds logical to me as well.
Please also add a document describing what the CP does here.
BTW: Is that limited to the GFX pipeline or do the Compute pipes the same?
Regards,
Christian.
Am 13.10.2017 um 11:36 schrieb Ding, Pixel:
Quoted 12 lines
Get it.
—
Sincerely Yours,
Pixel
Quoted 133 lines
Quoted 131 lines · Monk
@Ding, Pixel
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));Please handle other type fence as well:
Preempted fence
Reset fence
Reset and preempted fenceBR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian K?nig
Sent: 2017年10月13日 17:10
To: Ding, Pixel <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence viaamdgpu_fence_infoHi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 18 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 72 lines
Quoted 70 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 15 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveAm 13.10.2017 um 11:35 schrieb Ding, Pixel:
Quoted 31 lines · Christian König
Quoted 21 lines
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 17 lines
From: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h struct amdgpu_kiq { · line 888
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +0
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex;You can remove the
ring_mutexif you don't use it any more.
Quoted 1 line
[Pixel] KFD still use the mutex, I didn’t change it also to spin lock, should I?
Yes, when they use the mutex as well we need to change that to.
Otherwise we run into bad trouble when both try to access the same ring.
BTW: Do they also use the fence? That would be rather problematic.
Quoted 76 lines
Quoted 76 lines
Quoted 76 lines
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0+spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-6@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret =readl (((void __iomem *)adev->rmmio) + (reg *4 ));@@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v; } -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg (adev, reg, v); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX ))writel (v, ((void __iomem *)adev->rmmio) + (reg *4 ));drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,void amdgpu_fence_driver_suspend (struct amdgpu_device *adev);void amdgpu_fence_driver_resume (struct amdgpu_device *adev);int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void amdgpu_fence_process (struct amdgpu_ring *ring);int amdgpu_fence_wait_empty (struct amdgpu_ring *ring);unsigned amdgpu_fence_count_emitted (struct amdgpu_ring *ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-0@@ -22,7 +22,7 @@*/
Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c line 25
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1#include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ You want to busy wait 100 seconds for an result?
Forget it that is way to long you will certainly run into problems with the NMI much earlier.
[Pixel] thanks, would change it to 10s.
Still way to long. I think the maximum we can have here is 1s, maybe even only 100ms.
Quoted 28 lines
Quoted 19 lines
Quoted 19 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3int amdgpu_allocate_static_csa (struct amdgpu_device *adev) {@@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)uint32_t amdgpu_virt_kiq_rreg (struct amdgpu_device *adev,uint32_t reg) { -signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_rreg);Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 129
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-2-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_rreg (ring, reg); -amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );Do you still need to send an interrupt when you busy wait? I don't think so.
Quoted 1 line
[Pixel]I want to let the IRQ handler to update the
last_seqin case of that KIQ is used by other functionality with fence. Is it necessary?
Good point. In this case just keep it for now.
Might as well be that the KFD still needs the fence code.
Please double check that, cause that would make things much more complicated.
Quoted 14 lines
Quoted 6 lines
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-5amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) {Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_init_setting(struct amdgpu_device *adev) · line 136
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +6
-1-DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 );You need to correctly handle sequence wrap around here.
Quoted 1 line
[Pixel] yes thanks, but I didn’t see similar case handling in normal fence handling codepath, Is there? So I assumed 32bit fence seq could not overflow for years.
The normal fence handling code uses a circular buffer, so overflow is actually welcome there and doesn't need special handling.
But 2^32 register writes sounds like something we might actually hit sooner or later.
Regards,
Christian.
Quoted 43 lines
Quoted 35 lines
Quoted 35 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12
-5+ +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0 ; } - val = adev->wb.wb[adev->virt.reg_val_offs];return val;@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags);Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) · line 172
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +6
-0+ +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 );Same here. It would probably be better to have a function for this in
amdgpu_fence.c
Quoted 22 lines
[Pixel] get it, will add a wrapper.
Quoted 21 lines
Regards,
Christian.drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg) · line 178
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-4- r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }/** —
Sincerely Yours,
Pixel
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
In reply to a message not in the archiveOnly the gfx engine has such preemption fence mechanism in CP
-----Original Message-----
From: Koenig, Christian
Sent: 2017年10月13日 18:15
To: Ding, Pixel <[email protected]>; Liu, Monk <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence via amdgpu_fence_info
Sounds logical to me as well.
Please also add a document describing what the CP does here.
BTW: Is that limited to the GFX pipeline or do the Compute pipes the same?
Regards,
Christian.
Am 13.10.2017 um 11:36 schrieb Ding, Pixel:
Quoted 12 lines
Get it.
—
Sincerely Yours,
Pixel
Quoted 135 lines
Quoted 133 lines · Monk
@Ding, Pixel
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));Please handle other type fence as well:
Preempted fence
Reset fence
Reset and preempted fenceBR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian K?nig
Sent: 2017年10月13日 17:10
To: Ding, Pixel <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 3/4] drm/amdgpu: report preemption fence viaamdgpu_fence_infoHi Pixel,
Quoted 1 line
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred.
That is correct.
But my question is why you want to print a different value here:
Review
Quoted 2 lines
(quoted diff) +2
-0+ seq_printf(m, "Last preempted 0x%08x\n", + le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));That code prints two dw after the address where the CPU writes the seqno. Where is the code which setups the CP to do this?
As far as I can see that line should always print just zero (or some random number if the memory is actually not initialized to anything).
Regards,
Christian.Am 13.10.2017 um 11:03 schrieb Ding, Pixel:
Quoted 18 lines
My understanding is that CP will write seqno back to preempted fence offset when preemption occurred. Then if there is a value here we can generally know packet with which fence is preempted. Should driver handle any other thing for this?
—
Sincerely Yours,
PixelQuoted 72 lines
Quoted 70 lines · Christian
Quoted 1 line
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
The question is where is this code for Tonga and Vega? I can't find a reference to
fence_offsin the GFX8 nor GFX9 code we have in amd-staging-drm-next.And if it doesn't depend on the
fence_offsin the ring printing it like this is just pure speculation.Regards,
Christian.Am 13.10.2017 um 10:41 schrieb Ding, Pixel:
Quoted 58 lines
Thanks Christian,
I’m not sure if I get your point, but yes the preemption fence offset could be changed.
Is it OK to limit this information only for SRIOV VF on Tonga and Vega whose format is known? It can help use to identify if MCBP is working correctly or not.
—
Sincerely Yours,
PixelQuoted 39 lines · Christian König
Am 13.10.2017 um 10:26 schrieb Pixel Ding:
Quoted 15 lines
From: pding <[email protected]>
Only report fence for GFX ring. This can help checking MCBP feature.
Signed-off-by pding <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 650
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0@@ -645,6 +645,13 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +break ;That should probably be "continue" instead of break, or otherwise you don't print the other fences any more.
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) · line 653
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3
-0+ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 )));Is the code to put the preemption fence there already upstream?
If yes do we really do this like that for all supported generations?
Regards,
Christian.Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-0+ }return 0 ; }_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveOK, I would keep both methods working together.
—
Sincerely Yours,
Pixel
Quoted 229 lines · Monk
Why there will be racing issue ?
Polling or sleep wait only have different result for the caller, not the job scheduled to KIQ
The sleep waiting is synchroniz sleep, it just release CPU resource to other process/thread, so the order is guaranteed
BR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:39
To: Liu, Monk <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingI’m afraid there’s racing issue if polling and IRQ use cases are mixed at the same time.
The original implementation is as your suggested. Is there any benefit to keep to sleepy version? —
Sincerely Yours,
PixelQuoted 198 lines · Monk
Pixel
I don't think this will work well, my suggestion is you add a new function like:
amdgpu_wreg_kiq_busy(),which will write registers through KIQ and use polling/busy wait, and the original
amdgu_wreg_no_kiq()can be still there.When you need to disable sleep like in IRQ CONTEXT, you can call
wreg_kiq_busy()orwreg_no_kiq(),But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-3@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;}drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-2-if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg(adev, reg, v);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-1- }if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.cindex 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void
amdgpu_fence_process(struct amdgpu_ring *ring); intamdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsignedamdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cindex ab05121..9757df1 100644drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_tamdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg){drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3-signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_rreg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-1-amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit(ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-6-mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); + +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0; } - val = adev->wb.wb[adev->virt.reg_val_offs]; return val;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-8@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }
Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-1/** -- 2.9.5
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archiveHi Monk,
So far I notice that it’s almost impossible to have both methods working together.
For example, if the num_fence_mask is 2, and there’re a series of sequence number like “fence seq 1, busy wait seq 2, busy wait seq 3, fence seq 4”. Then if IRQ handler for seq 1 catch the HW seq number as 2, the fence slot for seq 4 will be wrongly waken up.
If only dma_fence slots are used, the emit function ensures that each slot only has one valid seq number by waiting for the old one. But things go complicated if there’s busy waiting which registers seq number at the same time.
—
Sincerely Yours,
Pixel
Quoted 242 lines · Pixel
OK, I would keep both methods working together.
—
Sincerely Yours,
PixelQuoted 229 lines · Monk
Why there will be racing issue ?
Polling or sleep wait only have different result for the caller, not the job scheduled to KIQ
The sleep waiting is synchroniz sleep, it just release CPU resource to other process/thread, so the order is guaranteed
BR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:39
To: Liu, Monk <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingI’m afraid there’s racing issue if polling and IRQ use cases are mixed at the same time.
The original implementation is as your suggested. Is there any benefit to keep to sleepy version? —
Sincerely Yours,
PixelQuoted 198 lines · Monk
Pixel
I don't think this will work well, my suggestion is you add a new function like:
amdgpu_wreg_kiq_busy(),which will write registers through KIQ and use polling/busy wait, and the original
amdgu_wreg_no_kiq()can be still there.When you need to disable sleep like in IRQ CONTEXT, you can call
wreg_kiq_busy()orwreg_no_kiq(),But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-3@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;}drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-2-if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg(adev, reg, v);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-1- }if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.cindex 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void
amdgpu_fence_process(struct amdgpu_ring *ring); intamdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsignedamdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cindex ab05121..9757df1 100644drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_tamdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg){drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3-signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_rreg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-1-amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit(ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-6-mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); + +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0; } - val = adev->wb.wb[adev->virt.reg_val_offs]; return val;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-8@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-1/** -- 2.9.5
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
In reply to a message not in the archivealright, go for unify scheme then, use busy wait for all user.
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月16日 16:55
To: Liu, Monk <[email protected]>
Cc: Li, Bingley <[email protected]>; [email protected]
Subject: Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessing
Hi Monk,
So far I notice that it’s almost impossible to have both methods working together.
For example, if the num_fence_mask is 2, and there’re a series of sequence number like “fence seq 1, busy wait seq 2, busy wait seq 3, fence seq 4”. Then if IRQ handler for seq 1 catch the HW seq number as 2, the fence slot for seq 4 will be wrongly waken up.
If only dma_fence slots are used, the emit function ensures that each slot only has one valid seq number by waiting for the old one. But things go complicated if there’s busy waiting which registers seq number at the same time.
—
Sincerely Yours,
Pixel
Quoted 242 lines · Pixel
OK, I would keep both methods working together.
—
Sincerely Yours,
PixelQuoted 229 lines · Monk
Why there will be racing issue ?
Polling or sleep wait only have different result for the caller, not the job scheduled to KIQ
The sleep waiting is synchroniz sleep, it just release CPU resource to other process/thread, so the order is guaranteed
BR Monk
-----Original Message-----
From: Ding, Pixel
Sent: 2017年10月13日 17:39
To: Liu, Monk <[email protected]>; [email protected]
Cc: Li, Bingley <[email protected]>
Subject: Re: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingI’m afraid there’s racing issue if polling and IRQ use cases are mixed at the same time.
The original implementation is as your suggested. Is there any benefit to keep to sleepy version? —
Sincerely Yours,
PixelQuoted 198 lines · Monk
Pixel
I don't think this will work well, my suggestion is you add a new function like:
amdgpu_wreg_kiq_busy(),which will write registers through KIQ and use polling/busy wait, and the original
amdgu_wreg_no_kiq()can be still there.When you need to disable sleep like in IRQ CONTEXT, you can call
wreg_kiq_busy()orwreg_no_kiq(),But don't just change the original function
BR Monk
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Pixel Ding
Sent: 2017年10月13日 16:26
To: [email protected]
Cc: Ding, Pixel <[email protected]>; Li, Bingley <[email protected]>
Subject: [PATCH 4/4] drm/amdgpu: busywait KIQ register accessingFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
Signed-off-by pding <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 55 ++++++++++++++++++------------ 6 files changed, 40 insertions(+), 28 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -886,6 +886,7 @@ struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj;struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-3@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); @@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v;}drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-2-if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg(adev, reg, v);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-1- }if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.cindex 2044758..c6c7bf3 100644
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1
-1@@ -109,7 +109,7 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)* Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ -static u32 amdgpu_fence_read (struct amdgpu_ring *ring) +u32 amdgpu_fence_read (struct amdgpu_ring *ring) {struct amdgpu_fence_driver *drv = &ring->fence_drv;u32 seq =0 ;drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1
-0@@ -186,6 +186,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,int r =0 ;mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +1
-0@@ -89,6 +89,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence); +u32 amdgpu_fence_read (struct amdgpu_ring *ring);void
amdgpu_fence_process(struct amdgpu_ring *ring); intamdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsignedamdgpu_fence_count_emitted(struct amdgpu_ring *ring); diff --gita/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cindex ab05121..9757df1 100644drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa(struct amdgpu_device *adev) { @@ -113,28 +113,32 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
uint32_tamdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg){drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-3-signed long r; -uint32_t val; -struct dma_fence *f; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t val, seq, wait_seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq; struct amdgpu_ring *ring = &kiq->ring; BUG_ON(!ring->funcs->emit_rreg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-1-mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc(ring, 32); amdgpu_ring_emit_rreg(ring, reg);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-1-amdgpu_fence_emit (ring, &f); + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit(ring);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-6-mutex_unlock (&kiq->ring_mutex); - - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +spin_unlock_irqrestore (&kiq->ring_lock, flags); +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); + +if (timeout <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout);return ~0; } - val = adev->wb.wb[adev->virt.reg_val_offs]; return val;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-8@@ -142,24 +146,33 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) { -signed long r; +signed long timeout =usecs_to_jiffies (MAX_KIQ_REG_WAIT ); +unsigned long flags; +uint32_t seq, wait_seq;struct dma_fence *f;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock_irqsave (&kiq->ring_lock, flags);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +/* amdgpu_fence_emit(ring, &f); */ + wait_seq = ++ring->fence_drv.sync_seq; +amdgpu_ring_emit_fence (ring, ring->fence_drv.gpu_addr, + wait_seq,AMDGPU_FENCE_FLAG_INT );amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock_irqrestore (&kiq->ring_lock, flags); + +do { + seq =amdgpu_fence_read (ring); +udelay (5 ); + timeout -=5 ; + }while (seq < wait_seq && timeout >0 ); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +if (timeout <1 ) +DRM_ERROR ("wait for kiq fence error: %ld\n" , timeout); }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0
-1/** -- 2.9.5
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx