Make staging driver stable for SRIOV VF (1 v2)
Subsystems (3)
This is the first patch series to make latest staging driver stable for SRIOV VF on both Tonga and Vega. Patches are merged from SRIOV branches or reimplemented, including bug fixes and small features requested by SRIOV users.
v2: "drm/amdgpu: workaround for VM fault caused by SDMA" is dropped.
Please help reviewing, Thanks.
[PATCH 1/3] drm/amdgpu: workaround for VM fault caused by SDMA [PATCH 2/3] drm/amdgpu: report more amdgpu_fence_info v2 [PATCH 3/3] drm/amdgpu: busywait KIQ register accessing v2
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 1/3 drm/amdgpu: workaround for VM fault caused by SDMA set_wptr
In reply to a message not in the archiveFrom: pding <[email protected]>
The polling memory was standalone in VRAM before, so the HDP flush introduced latency that hides a VM fault issue. Now polling memory leverages the WB in system memory and HDP flush is not required, the VM fault at same page happens.
Add delay back to workaround until the root cause is found.
Tests: VP1 or launch 40 instances of glxinfo at the same time.
--- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 3 +++ 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +3 -0
@@ -381,6 +381,9 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)if (ring->use_doorbell) {/* XXX check if swapping is necessary on BE */ adev->wb.wb[ring->wptr_offs] =lower_32_bits (ring->wptr) <<2 ; +/* workaround: VM fault always happen at page 2046 */ +if (amdgpu_sriov_vf (adev)) +udelay (500 );WDOORBELL32 (ring->doorbell_index,lower_32_bits (ring->wptr) <<2 ); }else {int me = (ring == &ring->adev->sdma.instance[0 ].ring) ?0 :1 ;
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/3 drm/amdgpu: report more amdgpu_fence_info v2
In reply to a message not in the archiveFrom: pding <[email protected]>
Only for GFX ring. This can help checking MCBP feature.
v2: report more fence offs.
The fence at the end of the frame will indicate the completion status. If the frame completed normally, the fence is written to the address given in the EVENT_WRITE_EOP packet. If preemption occurred in the previous IB the address is adjusted by 2 DWs. If work submitted in the frame was reset before completion, the fence address is adjusted by four DWs. In the case that preemption occurred, and before preemption completed a reset was initiated, the address will be adjusted with six DWs
--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +13 -0
@@ -645,6 +645,19 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data) atomic_read(&ring->fence_drv.last_seq));seq_printf (m,"Last emitted 0x%08x\n" , ring->fence_drv.sync_seq); + +if (ring->funcs->type !=AMDGPU_RING_TYPE_GFX ) +continue ; + +/* set in CP_VMID_PREEMPT and preemption occurred */ +seq_printf (m,"Last preempted 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +2 ))); +/* set in CP_VMID_RESET and reset occurred */ +seq_printf (m,"Last reset 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +4 ))); +/* Both preemption and reset occurred */ +seq_printf (m,"Last both 0x%08x\n" , +le32_to_cpu (*(ring->fence_drv.cpu_addr +6 ))); }return 0 ; }
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/3 drm/amdgpu: busywait KIQ register accessing v2
In reply to a message not in the archiveFrom: pding <[email protected]>
Register accessing is performed when IRQ is disabled. Never sleep in this function.
Known issue: dead sleep in many use cases of index/data registers.
v2: wrap polling fence functions. don't trigger IRQ for polling in case of wrongly fence signal.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 +--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 53 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 4 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 30 ++++++------- 8 files changed, 78 insertions(+), 29 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
@@ -885,7 +885,7 @@ struct amdgpu_mec {struct amdgpu_kiq {u64 eop_gpu_addr;struct amdgpu_bo *eop_obj; -struct mutex ring_mutex; +spinlock_t ring_lock;struct amdgpu_ring ring;struct amdgpu_irq_src irq; };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +2 -2
@@ -788,7 +788,7 @@ static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)struct dma_fence *f;struct amdgpu_ring *ring = &adev->gfx.kiq.ring; -mutex_lock (&adev->gfx.kiq.ring_mutex); +spin_lock (&adev->gfx.kiq.ring_lock);amdgpu_ring_alloc (ring,12 );/* fence + invalidate_tlbs package*/ amdgpu_ring_write (ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); amdgpu_ring_write(ring,
@@ -796,7 +796,7 @@ static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid) PACKET3_INVALIDATE_TLBS_PASID(pasid));amdgpu_fence_emit (ring, &f);amdgpu_ring_commit (ring); -mutex_unlock (&adev->gfx.kiq.ring_mutex); +spin_unlock (&adev->gfx.kiq.ring_lock); r =dma_fence_wait (f, false);if (r)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +2 -2
@@ -973,7 +973,7 @@ static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)struct dma_fence *f;struct amdgpu_ring *ring = &adev->gfx.kiq.ring; -mutex_lock (&adev->gfx.kiq.ring_mutex); +spin_lock (&adev->gfx.kiq.ring_lock);amdgpu_ring_alloc (ring,12 );/* fence + invalidate_tlbs package*/ amdgpu_ring_write (ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); amdgpu_ring_write(ring,
@@ -983,7 +983,7 @@ static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid) PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(2));amdgpu_fence_emit (ring, &f);amdgpu_ring_commit (ring); -mutex_unlock (&adev->gfx.kiq.ring_mutex); +spin_unlock (&adev->gfx.kiq.ring_lock); r =dma_fence_wait (f, false);if (r)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -6
@@ -109,10 +109,8 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, {uint32_t ret; -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_rreg (adev, reg); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX )) ret =readl (((void __iomem *)adev->rmmio) + (reg *4 ));
@@ -137,10 +135,8 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, adev->last_mm_index = v; } -if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev)) { -BUG_ON (in_interrupt ()); +if (!(acc_flags &AMDGPU_REGS_NO_KIQ ) &&amdgpu_sriov_runtime (adev))return amdgpu_virt_kiq_wreg (adev, reg, v); - }if ((reg *4 ) < adev->rmmio_size && !(acc_flags &AMDGPU_REGS_IDX ))writel (v, ((void __iomem *)adev->rmmio) + (reg *4 ));
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +53 -0
65 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +1 -1
@@ -185,7 +185,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,struct amdgpu_kiq *kiq = &adev->gfx.kiq;int r =0 ; -mutex_init (&kiq->ring_mutex); +spin_lock_init (&kiq->ring_lock); r =amdgpu_wb_get (adev, &adev->virt.reg_val_offs);if (r)
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +4 -0
@@ -89,8 +89,12 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,void amdgpu_fence_driver_suspend (struct amdgpu_device *adev);void amdgpu_fence_driver_resume (struct amdgpu_device *adev);int amdgpu_fence_emit (struct amdgpu_ring *ring,struct dma_fence **fence); +int amdgpu_fence_emit_polling (struct amdgpu_ring *ring,uint32_t *s);void amdgpu_fence_process (struct amdgpu_ring *ring);int amdgpu_fence_wait_empty (struct amdgpu_ring *ring); +signed long amdgpu_fence_wait_polling (struct amdgpu_ring *ring, +uint32_t wait_seq, +signed long timeout);unsigned amdgpu_fence_count_emitted (struct amdgpu_ring *ring);/*
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +13 -17
@@ -22,7 +22,7 @@*/ #include "amdgpu.h" -#define MAX_KIQ_REG_WAIT 100000 +#define MAX_KIQ_REG_WAIT 100000000/* in usecs */ int amdgpu_allocate_static_csa (struct amdgpu_device *adev) {
@@ -114,27 +114,24 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)uint32_t amdgpu_virt_kiq_rreg (struct amdgpu_device *adev,uint32_t reg) {signed long r; -uint32_t val; -struct dma_fence *f; +uint32_t val, seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_rreg); -mutex_lock (&kiq->ring_mutex); +spin_lock (&kiq->ring_lock);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_rreg (ring, reg); -amdgpu_fence_emit (ring, &f); +amdgpu_fence_emit_polling (ring, &seq);amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock (&kiq->ring_lock); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); -dma_fence_put (f); + r =amdgpu_fence_wait_polling (ring, seq,MAX_KIQ_REG_WAIT );if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +DRM_ERROR ("wait for kiq fence error: %ld\n" , r);return ~0 ; } - val = adev->wb.wb[adev->virt.reg_val_offs];return val;
@@ -143,23 +140,22 @@ uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)void amdgpu_virt_kiq_wreg (struct amdgpu_device *adev,uint32_t reg,uint32_t v) {signed long r; -struct dma_fence *f; +uint32_t seq;struct amdgpu_kiq *kiq = &adev->gfx.kiq;struct amdgpu_ring *ring = &kiq->ring;BUG_ON (!ring->funcs->emit_wreg); -mutex_lock (&kiq->ring_mutex); +spin_lock (&kiq->ring_lock);amdgpu_ring_alloc (ring,32 );amdgpu_ring_emit_wreg (ring, reg, v); -amdgpu_fence_emit (ring, &f); +amdgpu_fence_emit_polling (ring, &seq);amdgpu_ring_commit (ring); -mutex_unlock (&kiq->ring_mutex); +spin_unlock (&kiq->ring_lock); - r =dma_fence_wait_timeout (f, false,msecs_to_jiffies (MAX_KIQ_REG_WAIT )); + r =amdgpu_fence_wait_polling (ring, seq,MAX_KIQ_REG_WAIT );if (r <1 ) -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -dma_fence_put (f); +DRM_ERROR ("wait for kiq fence error: %ld\n" , r); }/**
Signature · 6 lines
2.9.5 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx