Skip to contentSkip to conversation

drm/amdgpu: fix IB test MCBP bug

PATCH In review Monk Liu 4 participants 9 messages Mar 2, 2020

Subsystems (2)
lore
Share & tools
lore

Monk Liu

author
#0 lore raw

PATCH 1/3 drm/amdgpu: fix IB test MCBP bug

1)for gfx IB test we shouldn't insert DE meta data

2)we should make sure IB test finished before we send event 3 to hypervisor otherwise the IDLE from event 3 will preempt IB test, which is not designed as a compatible structure for MCBP

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 3 ---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 2 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      | 2 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 351096a..572eb6e 100644
@@ -3195,6 +3195,12 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 	flush_delayed_work(&adev->delayed_init_work);
 	adev->shutdown = true;
 
+	/* make sure IB test finished before entering exclusive mode
+	 * to avoid preemption on IB test
+	 * */
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
 	/* disable all interrupts */
 	amdgpu_irq_disable_all(adev);
 	if (adev->mode_info.mode_config_initialized){
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +0 -3
index 0f35639..0b1511a 100644
@@ -88,9 +88,6 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
 	if (adev->rmmio == NULL)
 		goto done_free;
 
-	if (amdgpu_sriov_vf(adev))
-		amdgpu_virt_request_full_gpu(adev, false);
-
 	if (adev->runpm) {
 		pm_runtime_get_sync(dev->dev);
 		pm_runtime_forbid(dev->dev);
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -1
index 94ca9ff..0555989 100644
@@ -4432,7 +4432,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 		if (flags & AMDGPU_IB_PREEMPTED)
 			control |= INDIRECT_BUFFER_PRE_RESUME(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
 			gfx_v10_0_ring_emit_de_meta(ring,
 				    (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
 	}
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +1 -1
index 393a132..b14f46a3 100644
@@ -6116,7 +6116,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
 			gfx_v8_0_ring_emit_de_meta(ring);
 	}
 
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +1 -1
index 0156479..d8d256e6 100644
@@ -4985,7 +4985,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
 			gfx_v9_0_ring_emit_de_meta(ring);
 	}
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
author
#1 lore raw

PATCH 2/3 drm/amdgpu: don't use pipe1 of gfx10

what:
we found sometimes IDLE fail after vf guest finished IB test on GFX ring1 (pipe1)

why:
below is what CP team stated (Manu):
GFX Pipe 1 is there in HW, but as part of optimization all driver decided not to use pipe 1 at all, otherwise driver has to sacrifice context so all 7 context will not be able for GFX pipe 0. That’s why I skip setting of state for gfx pipe 1 as decided by all driver team

fix:
since CP team won't help us to debug any issues that related with gfx pipe1, so based on above reason, let's skip gfx ring 1 (pipe1) even for both bare-metal and SRIOV

---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +18 -11
index 0555989..afae4cc 100644
@@ -1308,7 +1308,7 @@ static int gfx_v10_0_sw_init(void *handle)
 	case CHIP_NAVI14:
 	case CHIP_NAVI12:
 		adev->gfx.me.num_me = 1;
-		adev->gfx.me.num_pipe_per_me = 2;
+		adev->gfx.me.num_pipe_per_me = 1;
 		adev->gfx.me.num_queue_per_pipe = 1;
 		adev->gfx.mec.num_mec = 2;
 		adev->gfx.mec.num_pipe_per_mec = 4;
@@ -2713,18 +2713,21 @@ static int gfx_v10_0_cp_gfx_start(struct amdgpu_device *adev)
 
 	amdgpu_ring_commit(ring);
 
-	/* submit cs packet to copy state 0 to next available state */
-	ring = &adev->gfx.gfx_ring[1];
-	r = amdgpu_ring_alloc(ring, 2);
-	if (r) {
-		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
-		return r;
-	}
+	if (adev->gfx.me.num_pipe_per_me == 2) {
+		/* submit cs packet to copy state 0 to next available state */
+		ring = &adev->gfx.gfx_ring[1];
 
-	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
-	amdgpu_ring_write(ring, 0);
+		r = amdgpu_ring_alloc(ring, 2);
+		if (r) {
+			DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+			return r;
+		}
 
-	amdgpu_ring_commit(ring);
+		amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+		amdgpu_ring_write(ring, 0);
+
+		amdgpu_ring_commit(ring);
+	}
 
 	return 0;
 }
@@ -2822,6 +2825,9 @@ static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
 	mutex_unlock(&adev->srbm_mutex);
 
 	/* Init gfx ring 1 for pipe 1 */
+	if (adev->gfx.me.num_pipe_per_me == 1)
+		goto do_start;
+
 	mutex_lock(&adev->srbm_mutex);
 	gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
 	ring = &adev->gfx.gfx_ring[1];
@@ -2860,6 +2866,7 @@ static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
 	gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
 	mutex_unlock(&adev->srbm_mutex);
 
+do_start:
 	/* start the ring */
 	gfx_v10_0_cp_gfx_start(adev);
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#2 lore raw

RE: [PATCH 2/3] drm/amdgpu: don't use pipe1 of gfx10

[AMD Official Use Only - Internal Distribution Only]

This has already done by Tianchi.

Regards,
Hawking

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Monk Liu
Sent: Monday, March 2, 2020 17:22
To: [email protected]
Cc: Liu, Monk <[email protected]>
Subject: [PATCH 2/3] drm/amdgpu: don't use pipe1 of gfx10

what:
we found sometimes IDLE fail after vf guest finished IB test on GFX ring1 (pipe1)

why:
below is what CP team stated (Manu):
GFX Pipe 1 is there in HW, but as part of optimization all driver decided not to use pipe 1 at all, otherwise driver has to sacrifice context so all 7 context will not be able for GFX pipe 0. That’s why I skip setting of state for gfx pipe 1 as decided by all driver team

fix:
since CP team won't help us to debug any issues that related with gfx pipe1, so based on above reason, let's skip gfx ring 1 (pipe1) even for both bare-metal and SRIOV

---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +18 -11
index 0555989..afae4cc 100644
@@ -1308,7 +1308,7 @@ static int gfx_v10_0_sw_init(void *handle)
 	case CHIP_NAVI14:
 	case CHIP_NAVI12:
 		adev->gfx.me.num_me = 1;
-		adev->gfx.me.num_pipe_per_me = 2;
+		adev->gfx.me.num_pipe_per_me = 1;
 		adev->gfx.me.num_queue_per_pipe = 1;
 		adev->gfx.mec.num_mec = 2;
 		adev->gfx.mec.num_pipe_per_mec = 4;
@@ -2713,18 +2713,21 @@ static int gfx_v10_0_cp_gfx_start(struct amdgpu_device *adev)
 
 	amdgpu_ring_commit(ring);
 
-	/* submit cs packet to copy state 0 to next available state */
-	ring = &adev->gfx.gfx_ring[1];
-	r = amdgpu_ring_alloc(ring, 2);
-	if (r) {
-		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
-		return r;
-	}
+	if (adev->gfx.me.num_pipe_per_me == 2) {
+		/* submit cs packet to copy state 0 to next available state */
+		ring = &adev->gfx.gfx_ring[1];
 
-	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
-	amdgpu_ring_write(ring, 0);
+		r = amdgpu_ring_alloc(ring, 2);
+		if (r) {
+			DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+			return r;
+		}
 
-	amdgpu_ring_commit(ring);
+		amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+		amdgpu_ring_write(ring, 0);
+
+		amdgpu_ring_commit(ring);
+	}
 
 	return 0;
 }
@@ -2822,6 +2825,9 @@ static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
 	mutex_unlock(&adev->srbm_mutex);
 
 	/* Init gfx ring 1 for pipe 1 */
+	if (adev->gfx.me.num_pipe_per_me == 1)
+		goto do_start;
+
 	mutex_lock(&adev->srbm_mutex);
 	gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
 	ring = &adev->gfx.gfx_ring[1];
@@ -2860,6 +2866,7 @@ static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
 	gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
 	mutex_unlock(&adev->srbm_mutex);
 
+do_start:
 	/* start the ring */
 	gfx_v10_0_cp_gfx_start(adev);
 

--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Chawking.zhang%40amd.com%7C7e7dfb16eb5344c43bcc08d7be8b32e4%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637187377390959192&amp;sdata=2Gb5apKZanCfvgbclKLO%2BAAbasxNamUgw0j567HI%2BBs%3D&amp;reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#3 lore raw

RE: [PATCH 2/3] drm/amdgpu: don't use pipe1 of gfx10

Hi Hawking

I didn't see tianci's patch merged to drm-next, is it passed our review yet ?

Please ignore my patch then
_____________________________________
Monk Liu|GPU Virtualization Team |AMD

-----Original Message-----
From: Zhang, Hawking <[email protected]>
Sent: Monday, March 2, 2020 5:30 PM
To: Liu, Monk <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: RE: [PATCH 2/3] drm/amdgpu: don't use pipe1 of gfx10

[AMD Official Use Only - Internal Distribution Only]

This has already done by Tianchi.

Regards,
Hawking

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Monk Liu
Sent: Monday, March 2, 2020 17:22
To: [email protected]
Cc: Liu, Monk <[email protected]>
Subject: [PATCH 2/3] drm/amdgpu: don't use pipe1 of gfx10

what:
we found sometimes IDLE fail after vf guest finished IB test on GFX ring1 (pipe1)

why:
below is what CP team stated (Manu):
GFX Pipe 1 is there in HW, but as part of optimization all driver decided not to use pipe 1 at all, otherwise driver has to sacrifice context so all 7 context will not be able for GFX pipe 0. That’s why I skip setting of state for gfx pipe 1 as decided by all driver team

fix:
since CP team won't help us to debug any issues that related with gfx pipe1, so based on above reason, let's skip gfx ring 1 (pipe1) even for both bare-metal and SRIOV

---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +18 -11
index 0555989..afae4cc 100644
@@ -1308,7 +1308,7 @@ static int gfx_v10_0_sw_init(void *handle)
 	case CHIP_NAVI14:
 	case CHIP_NAVI12:
 		adev->gfx.me.num_me = 1;
-		adev->gfx.me.num_pipe_per_me = 2;
+		adev->gfx.me.num_pipe_per_me = 1;
 		adev->gfx.me.num_queue_per_pipe = 1;
 		adev->gfx.mec.num_mec = 2;
 		adev->gfx.mec.num_pipe_per_mec = 4;
@@ -2713,18 +2713,21 @@ static int gfx_v10_0_cp_gfx_start(struct amdgpu_device *adev)
 
 	amdgpu_ring_commit(ring);
 
-	/* submit cs packet to copy state 0 to next available state */
-	ring = &adev->gfx.gfx_ring[1];
-	r = amdgpu_ring_alloc(ring, 2);
-	if (r) {
-		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
-		return r;
-	}
+	if (adev->gfx.me.num_pipe_per_me == 2) {
+		/* submit cs packet to copy state 0 to next available state */
+		ring = &adev->gfx.gfx_ring[1];
 
-	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
-	amdgpu_ring_write(ring, 0);
+		r = amdgpu_ring_alloc(ring, 2);
+		if (r) {
+			DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+			return r;
+		}
 
-	amdgpu_ring_commit(ring);
+		amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+		amdgpu_ring_write(ring, 0);
+
+		amdgpu_ring_commit(ring);
+	}
 
 	return 0;
 }
@@ -2822,6 +2825,9 @@ static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
 	mutex_unlock(&adev->srbm_mutex);
 
 	/* Init gfx ring 1 for pipe 1 */
+	if (adev->gfx.me.num_pipe_per_me == 1)
+		goto do_start;
+
 	mutex_lock(&adev->srbm_mutex);
 	gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
 	ring = &adev->gfx.gfx_ring[1];
@@ -2860,6 +2866,7 @@ static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
 	gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
 	mutex_unlock(&adev->srbm_mutex);
 
+do_start:
 	/* start the ring */
 	gfx_v10_0_cp_gfx_start(adev);
 

--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Chawking.zhang%40amd.com%7C7e7dfb16eb5344c43bcc08d7be8b32e4%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637187377390959192&amp;sdata=2Gb5apKZanCfvgbclKLO%2BAAbasxNamUgw0j567HI%2BBs%3D&amp;reserved=0
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#4 lore raw

PATCH 3/3 drm/amdgpu: stop using sratch_reg in IB test

In reply to Monk Liu

scratch_reg0 is used by RLCG for register access usage in SRIOV case.

both CP firmware and driver can invoke RLCG to do certain register access (through scratch_reg0/1/2/3) but rlcg now dosen't have race concern so if two clients are in parallel doing the RLCG reg access then we are colliding,

GFX IB test is a runtime work, so it is forbidden to use scrach_reg0/1/2/3 during IB test period

note:
Although we can only have this change for SRIOV, but looks it doesn't worth the effort to differentiate bare-metal with SRIOV on the GFX ib test

---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 38 +++++++++++++++-------------------
 1 file changed, 17 insertions(+), 21 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +17 -21
index afae4cc..b86a531 100644
@@ -500,29 +500,28 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
 	struct dma_fence *f = NULL;
-	uint32_t scratch;
-	uint32_t tmp = 0;
+	unsigned index;
+	uint64_t gpu_addr;
+	uint32_t tmp;
 	long r;
 
-	r = amdgpu_gfx_scratch_get(adev, &scratch);
-	if (r) {
-		DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r)
 		return r;
-	}
-
-	WREG32(scratch, 0xCAFEDEAD);
 
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256, &ib);
-	if (r) {
-		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+	r = amdgpu_ib_get(adev, NULL, 16, &ib);
+	if (r)
 		goto err1;
-	}
 
-	ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
-	ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
-	ib.ptr[2] = 0xDEADBEEF;
-	ib.length_dw = 3;
+	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib.ptr[2] = lower_32_bits(gpu_addr);
+	ib.ptr[3] = upper_32_bits(gpu_addr);
+	ib.ptr[4] = 0xDEADBEEF;
+	ib.length_dw = 5;
 
 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 	if (r)
@@ -530,15 +529,13 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
-		DRM_ERROR("amdgpu: IB test timed out.\n");
 		r = -ETIMEDOUT;
 		goto err2;
 	} else if (r < 0) {
-		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 		goto err2;
 	}
 
-	tmp = RREG32(scratch);
+	tmp = adev->wb.wb[index];
 	if (tmp == 0xDEADBEEF)
 		r = 0;
 	else
@@ -547,8 +544,7 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(adev, &ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_gfx_scratch_free(adev, scratch);
-
+	amdgpu_device_wb_free(adev, index);
 	return r;
 }
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Reviewed-by on v1 3/3
#5 lore raw

Re: [PATCH 3/3] drm/amdgpu: stop using sratch_reg in IB test

Am 02.03.20 um 10:22 schrieb Monk Liu:

Quoted 12 lines

scratch_reg0 is used by RLCG for register access usage in SRIOV case.

both CP firmware and driver can invoke RLCG to do certain register access (through scratch_reg0/1/2/3) but rlcg now dosen't have race concern so if two clients are in parallel doing the RLCG reg access then we are colliding,

GFX IB test is a runtime work, so it is forbidden to use scrach_reg0/1/2/3 during IB test period

Quoted 6 lines

note:
Although we can only have this change for SRIOV, but looks it doesn't worth the effort to differentiate bare-metal with SRIOV on the GFX ib test

Must have been copy&pasted from some old code. IIRC we stoped using scratch regs for gfx9 for quite a while as well.

Christian.

Quoted 71 lines
---
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 38 +++++++++++++++-------------------
  1 file changed, 17 insertions(+), 21 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +16 -19
index afae4cc..b86a531 100644
@@ -500,29 +500,28 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
  	struct amdgpu_device *adev = ring->adev;
  	struct amdgpu_ib ib;
  	struct dma_fence *f = NULL;
-	uint32_t scratch;
-	uint32_t tmp = 0;
+	unsigned index;
+	uint64_t gpu_addr;
+	uint32_t tmp;
  	long r;
 
-	r = amdgpu_gfx_scratch_get(adev, &scratch);
-	if (r) {
-		DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r)
  		return r;
-	}
-
-	WREG32(scratch, 0xCAFEDEAD);
 
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
  	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256, &ib);
-	if (r) {
-		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+	r = amdgpu_ib_get(adev, NULL, 16, &ib);
+	if (r)
  		goto err1;
-	}
 
-	ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
-	ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
-	ib.ptr[2] = 0xDEADBEEF;
-	ib.length_dw = 3;
+	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib.ptr[2] = lower_32_bits(gpu_addr);
+	ib.ptr[3] = upper_32_bits(gpu_addr);
+	ib.ptr[4] = 0xDEADBEEF;
+	ib.length_dw = 5;
 
  	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
  	if (r)
@@ -530,15 +529,13 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
  	r = dma_fence_wait_timeout(f, false, timeout);
  	if (r == 0) {
-		DRM_ERROR("amdgpu: IB test timed out.\n");
  		r = -ETIMEDOUT;
  		goto err2;
  	} else if (r < 0) {
-		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
  		goto err2;
  	}
 
-	tmp = RREG32(scratch);
+	tmp = adev->wb.wb[index];
  	if (tmp == 0xDEADBEEF)
  		r = 0;
  	else

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) · line 550

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -2
@@ -547,8 +544,7 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
  	amdgpu_ib_free(adev, &ib, NULL);
  	dma_fence_put(f);
  err1:
-	amdgpu_gfx_scratch_free(adev, scratch);
-
+	amdgpu_device_wb_free(adev, index);
  	return r;
  }
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#6 lore raw
In reply to Monk Liu

Am 02.03.20 um 10:22 schrieb Monk Liu:

Quoted 65 lines

1)for gfx IB test we shouldn't insert DE meta data

2)we should make sure IB test finished before we send event 3 to hypervisor otherwise the IDLE from event 3 will preempt IB test, which is not designed as a compatible structure for MCBP

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 3 ---
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 2 +-
  drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      | 2 +-
  drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      | 2 +-
  5 files changed, 9 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 351096a..572eb6e 100644
@@ -3195,6 +3195,12 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
  	flush_delayed_work(&adev->delayed_init_work);
  	adev->shutdown = true;
 
+	/* make sure IB test finished before entering exclusive mode
+	 * to avoid preemption on IB test
+	 * */
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
  	/* disable all interrupts */
  	amdgpu_irq_disable_all(adev);
  	if (adev->mode_info.mode_config_initialized){
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +0 -3
index 0f35639..0b1511a 100644
@@ -88,9 +88,6 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
  	if (adev->rmmio == NULL)
  		goto done_free;
 
-	if (amdgpu_sriov_vf(adev))
-		amdgpu_virt_request_full_gpu(adev, false);
-
  	if (adev->runpm) {
  		pm_runtime_get_sync(dev->dev);
  		pm_runtime_forbid(dev->dev);
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -1
index 94ca9ff..0555989 100644
@@ -4432,7 +4432,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
  		if (flags & AMDGPU_IB_PREEMPTED)
  			control |= INDIRECT_BUFFER_PRE_RESUME(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
  			gfx_v10_0_ring_emit_de_meta(ring,
  				    (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
  	}
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +0 -0
index 393a132..b14f46a3 100644

drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, · line 6119

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +1 -1
@@ -6116,7 +6116,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
  	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
  		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)

Kernel copies also don't use a VMID, so I think that this won't work correctly.

Do you have the job available as well? That would probably be better to test for since only IB tests don't use a job.

Christian.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +0 -0
  			gfx_v8_0_ring_emit_de_meta(ring);
  	}
 
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +0 -0
index 0156479..d8d256e6 100644

drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, · line 4991

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +1 -1
@@ -4985,7 +4985,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
  	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
  		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
  			gfx_v9_0_ring_emit_de_meta(ring);
  	}
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#7 lore raw

Review

Quoted 2 lines
(quoted diff) +1 -1
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)

Kernel copies also don't use a VMID, so I think that this won't work correctly.

Do you have the job available as well? That would probably be better to test for since only IB tests don't use a job.

[ML]
The @job is not passed to the gfx_v10_0_ring_emit_ib_gfx() routine so I tend to modify as less as possible
Besides, for kernel copies: I believe they also do not support MCBP (can you point me the code in source of those kernel copies ?) , so use VMID to test is fine (To support MCBP on gfx ring you need a set of additional preamble IB co work with your workload iB and kernel copies apparently don't have those stuffs)

Thanks
_____________________________________
Monk Liu|GPU Virtualization Team |AMD

-----Original Message-----
From: Christian König <[email protected]>
Sent: Monday, March 2, 2020 5:35 PM
To: Liu, Monk <[email protected]>; [email protected]
Subject: Re: [PATCH 1/3] drm/amdgpu: fix IB test MCBP bug

Am 02.03.20 um 10:22 schrieb Monk Liu:

Quoted 68 lines

1)for gfx IB test we shouldn't insert DE meta data

2)we should make sure IB test finished before we send event 3 to hypervisor otherwise the IDLE from event 3 will preempt IB test, which is not designed as a compatible structure for MCBP

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 3 ---
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 2 +-
  drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      | 2 +-
  drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      | 2 +-
  5 files changed, 9 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 351096a..572eb6e 100644
@@ -3195,6 +3195,12 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
  	flush_delayed_work(&adev->delayed_init_work);
  	adev->shutdown = true;
 
+	/* make sure IB test finished before entering exclusive mode
+	 * to avoid preemption on IB test
+	 * */
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
  	/* disable all interrupts */
  	amdgpu_irq_disable_all(adev);
  	if (adev->mode_info.mode_config_initialized){
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +0 -3
index 0f35639..0b1511a 100644
@@ -88,9 +88,6 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
  	if (adev->rmmio == NULL)
  		goto done_free;
 
-	if (amdgpu_sriov_vf(adev))
-		amdgpu_virt_request_full_gpu(adev, false);
-
  	if (adev->runpm) {
  		pm_runtime_get_sync(dev->dev);
  		pm_runtime_forbid(dev->dev);
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -1
index 94ca9ff..0555989 100644
@@ -4432,7 +4432,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
  		if (flags & AMDGPU_IB_PREEMPTED)
  			control |= INDIRECT_BUFFER_PRE_RESUME(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
  			gfx_v10_0_ring_emit_de_meta(ring,
  				    (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
  	}
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +0 -0
index 393a132..b14f46a3 100644

drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, · line 6119

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +1 -1
@@ -6116,7 +6116,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
  	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
  		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)

Kernel copies also don't use a VMID, so I think that this won't work correctly.

Do you have the job available as well? That would probably be better to test for since only IB tests don't use a job.

Christian.

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +0 -0
  			gfx_v8_0_ring_emit_de_meta(ring);
  	}
 
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +0 -0
index 0156479..d8d256e6 100644

drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, · line 4991

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +1 -1
@@ -4985,7 +4985,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
  	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
  		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
  			gfx_v9_0_ring_emit_de_meta(ring);
  	}
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

maintainer Acked-by
#8 lore raw

Am 02.03.20 um 10:39 schrieb Liu, Monk:

Quoted 9 lines

Review

Quoted 2 lines
(quoted diff) +1 -1
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)

Kernel copies also don't use a VMID, so I think that this won't work correctly.

Do you have the job available as well? That would probably be better to test for since only IB tests don't use a job.

[ML]
The @job is not passed to the gfx_v10_0_ring_emit_ib_gfx() routine so I tend to modify as less as possible
Besides, for kernel copies: I believe they also do not support MCBP (can you point me the code in source of those kernel copies ?) , so use VMID to test is fine (To support MCBP on gfx ring you need a set of additional preamble IB co work with your workload iB and kernel copies apparently don't have those stuffs)

Ah, sorry! My incorrect thinking, kernel copies also don't use the CP but rather the SDMA. So the change is completely irrelevant for the copies.

Regards,
Christian.

Quoted 104 lines

Thanks
_____________________________________
Monk Liu|GPU Virtualization Team |AMD

-----Original Message-----
From: Christian König <[email protected]>
Sent: Monday, March 2, 2020 5:35 PM
To: Liu, Monk <[email protected]>; [email protected]
Subject: Re: [PATCH 1/3] drm/amdgpu: fix IB test MCBP bug

Am 02.03.20 um 10:22 schrieb Monk Liu:

Quoted 68 lines

1)for gfx IB test we shouldn't insert DE meta data

2)we should make sure IB test finished before we send event 3 to hypervisor otherwise the IDLE from event 3 will preempt IB test, which is not designed as a compatible structure for MCBP

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 3 ---
   drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c     | 2 +-
   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      | 2 +-
   drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      | 2 +-
   5 files changed, 9 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 351096a..572eb6e 100644
@@ -3195,6 +3195,12 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
   	flush_delayed_work(&adev->delayed_init_work);
   	adev->shutdown = true;
 
+	/* make sure IB test finished before entering exclusive mode
+	 * to avoid preemption on IB test
+	 * */
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
   	/* disable all interrupts */
   	amdgpu_irq_disable_all(adev);
   	if (adev->mode_info.mode_config_initialized){
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +0 -3
index 0f35639..0b1511a 100644
@@ -88,9 +88,6 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
   	if (adev->rmmio == NULL)
   		goto done_free;
 
-	if (amdgpu_sriov_vf(adev))
-		amdgpu_virt_request_full_gpu(adev, false);
-
   	if (adev->runpm) {
   		pm_runtime_get_sync(dev->dev);
   		pm_runtime_forbid(dev->dev);
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -1
index 94ca9ff..0555989 100644
@@ -4432,7 +4432,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
   		if (flags & AMDGPU_IB_PREEMPTED)
   			control |= INDIRECT_BUFFER_PRE_RESUME(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
   			gfx_v10_0_ring_emit_de_meta(ring,
   				    (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
   	}
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +0 -0
index 393a132..b14f46a3 100644

drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, · line 6119

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +1 -1
@@ -6116,7 +6116,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
   	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
   		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)

Kernel copies also don't use a VMID, so I think that this won't work correctly.

Do you have the job available as well? That would probably be better to test for since only IB tests don't use a job.

Christian.

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +0 -0
   			gfx_v8_0_ring_emit_de_meta(ring);
   	}
 
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +0 -0
index 0156479..d8d256e6 100644
@@ -4985,7 +4985,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
   	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
   		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, · line 4991

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +1 -1
-		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
   			gfx_v9_0_ring_emit_de_meta(ring);
   	}
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx