Skip to contentSkip to conversation

drm/amdgpu: change the fence ring wait timeout

PATCH NAK Roy Sun 5 participants 12 messages Jan 13, 2021 – Jan 19, 2021

Subsystems (2)
lore
Share & tools
lore

Roy Sun

author
#0 lore raw

PATCH 1/1 drm/amdgpu: change the fence ring wait timeout

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48 ++++++++++++++++++++---
 1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +43 -5
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000)
+#define AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
 /*
  * Fences
  * Fences mark an event in the GPUs pipeline and are used
@@ -104,6 +106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
 		*drv->cpu_addr = cpu_to_le32(seq);
 }
 
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring)
+{
+	long tmo_gfx, tmo_mm, tmo;
+	struct amdgpu_device *adev = ring->adev;
+	tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT;
+	if (amdgpu_sriov_vf(adev)) {
+		tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT;
+	}
+	if (amdgpu_sriov_runtime(adev)) {
+		tmo_gfx = 8 * AMDGPU_FENCE_TIMEOUT;
+	} else if (adev->gmc.xgmi.hive_id) {
+		tmo_gfx = AMDGPU_FENCE_GFX_XGMI_TIMEOUT;
+	}
+	if (ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCE ||
+		ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
+		tmo = tmo_mm;
+	else
+		tmo = tmo_gfx;
+	return tmo;
+}
+
 /**
  * amdgpu_fence_read - read a fence value
  *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
 		rcu_read_unlock();
 
 		if (old) {
-			r = dma_fence_wait(old, false);
+			long timeout;
+			timeout = amdgpu_fence_wait_timeout(ring);
+			r = dma_fence_wait_timeout(old, false, timeout);
 			dma_fence_put(old);
 			if (r)
-				return r;
+				return r < 0 ? r : 0;
 		}
 	}
 
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
 		return 0;
 	}
 	rcu_read_unlock();
-
-	r = dma_fence_wait(fence, false);
+	
+	long timeout;
+	timeout = amdgpu_fence_wait_timeout(ring);
+	r = dma_fence_wait_timeout(fence, false, timeout);
 	dma_fence_put(fence);
-	return r;
+	return r < 0 ? r : 0;
 }
 
 /**
Signature · 6 lines
2.28.0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#1 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 23 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Roy Sun
Sent: Wednesday, January 13, 2021 2:36 PM
To: [email protected]
Cc: Sun, Roy <[email protected]>
Subject: [PATCH] drm/amdgpu: change the fence ring wait timeout

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

---
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48
++++++++++++++++++++---

1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c line 45

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
@@ -41,6 +41,8 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)

Please check the format.

Quoted 78 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -0
 /*
 * Fences
 * Fences mark an event in the GPUs pipeline and are used @@ -104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32

seq)
*drv->cpu_addr = cpu_to_le32(seq);
}

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +31 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) {
+long tmo_gfx, tmo_mm, tmo;
+struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT;
+if (amdgpu_sriov_vf(adev)) {
+tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT;
+}
+if (amdgpu_sriov_runtime(adev)) {
+tmo_gfx = 8 * AMDGPU_FENCE_TIMEOUT;
+} else if (adev->gmc.xgmi.hive_id) {
+tmo_gfx = AMDGPU_FENCE_GFX_XGMI_TIMEOUT;
+}
+if (ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE ||
+ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+

/**
* amdgpu_fence_read - read a fence value
*

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +9 -5
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring,
 struct dma_fence **f,
 rcu_read_unlock();
 
 if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring);
+r = dma_fence_wait_timeout(old, false, timeout);
 dma_fence_put(old);
 if (r)
-return r;
+return r < 0 ? r : 0;
 }
 }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring
 *ring)
 return 0;
 }
 rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring);
+r = dma_fence_wait_timeout(fence, false, timeout);
 dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
 }
 

/**
--
2.28.0

Quoted 11 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C2df180bf7aaa419f 387708d8b78d9dfd%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637461166168063291%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=2n3ZbLF0ZT%2FakzxlcBI3c%2F9A0F8EIzvKzDj2OPA0DmE%3D&amp;reserve d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#2 lore raw
In reply to Roy Sun

Dear Roy,

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 1 line

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

Please wrap commit message body after 75 characters. (checkpatch.pl should have shown you that.)

Some typos:

1.  fix*es*
2.  hang*s*
3.  Please add a dot/period at the end of sentences.

If there is a Linux kernel crash, please add the trace.

Please elaborate also. Where and why does the Linux kernel crash, and how does your patch fix this.

How can the bug be reproduced? On what hardware did you see it?

Quoted 10 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48 ++++++++++++++++++++---
  1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c line 45

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
@@ -41,6 +41,8 @@
  #include "amdgpu.h"
  #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000)
+#define AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)

Where did you get the values from, or why did you choose them.

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
  /*
   * Fences
   * Fences mark an event in the GPUs pipeline and are used
@@ -104,6 +106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
  		*drv->cpu_addr = cpu_to_le32(seq);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) · line 116

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +8 -0
  }
 
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring)

See below, but it should be signed long.

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) · line 123

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +7 -0
+{
+	long tmo_gfx, tmo_mm, tmo;
+	struct amdgpu_device *adev = ring->adev;
+	tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT;
+	if (amdgpu_sriov_vf(adev)) {
+		tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT;
+	}

See coding style. No {} needed for one line statements.

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) · line 125

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+	if (amdgpu_sriov_runtime(adev)) {
+		tmo_gfx = 8 * AMDGPU_FENCE_TIMEOUT;

So why not define a special SRIOV macro too. But add a comment, why it should be eight times that.

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) · line 128

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3 -0
+	} else if (adev->gmc.xgmi.hive_id) {
+		tmo_gfx = AMDGPU_FENCE_GFX_XGMI_TIMEOUT;
+	}

Ditto.

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) · line 134

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +6 -0
+	if (ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCE ||
+		ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)

Align on ring?

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) · line 138

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+		tmo = tmo_mm;
+	else
+		tmo = tmo_gfx;
+	return tmo;

The return could happen in the branches.

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, · line 205

Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
+}
+
  /**
   * amdgpu_fence_read - read a fence value
   *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
  		rcu_read_unlock();
 
  		if (old) {
-			r = dma_fence_wait(old, false);
+			long timeout;
+			timeout = amdgpu_fence_wait_timeout(ring);
+			r = dma_fence_wait_timeout(old, false, timeout);

The signature is signed long as far as I see.

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -3
  			dma_fence_put(old);
  			if (r)
-				return r;
+				return r < 0 ? r : 0;
  		}
  	}
 
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
  		return 0;
  	}
  	rcu_read_unlock();
-
-	r = dma_fence_wait(fence, false);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) · line 390

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
+	
+	long timeout;
+	timeout = amdgpu_fence_wait_timeout(ring);
+	r = dma_fence_wait_timeout(fence, false, timeout);
  	dma_fence_put(fence);
-	return r;
+	return r < 0 ? r : 0;
  }
 
  /**

It looks like this is your first patch submission, so welcome to the Linux kernel community. Please keep in mind, that reviewer time is precious and scarce, and that formal and simple mistakes should not happen. No idea, if AMD has some kind of “onboarding” process for their newly employed Linux kernel contributors.

I am looking forward to your the patch iteration and your next contributions.

Kind regards,

Paul
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#3 lore raw
In reply to Roy Sun

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 1 line

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

Regards,
Christian.

Quoted 79 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48 ++++++++++++++++++++---
  1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +38 -4
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
  #include "amdgpu.h"
  #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000)
+#define AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
  /*
   * Fences
   * Fences mark an event in the GPUs pipeline and are used
@@ -104,6 +106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
  		*drv->cpu_addr = cpu_to_le32(seq);
  }
 
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring)
+{
+	long tmo_gfx, tmo_mm, tmo;
+	struct amdgpu_device *adev = ring->adev;
+	tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT;
+	if (amdgpu_sriov_vf(adev)) {
+		tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT;
+	}
+	if (amdgpu_sriov_runtime(adev)) {
+		tmo_gfx = 8 * AMDGPU_FENCE_TIMEOUT;
+	} else if (adev->gmc.xgmi.hive_id) {
+		tmo_gfx = AMDGPU_FENCE_GFX_XGMI_TIMEOUT;
+	}
+	if (ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCE ||
+		ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+		ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
+		tmo = tmo_mm;
+	else
+		tmo = tmo_gfx;
+	return tmo;
+}
+
  /**
   * amdgpu_fence_read - read a fence value
   *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
  		rcu_read_unlock();
 
  		if (old) {
-			r = dma_fence_wait(old, false);
+			long timeout;
+			timeout = amdgpu_fence_wait_timeout(ring);
+			r = dma_fence_wait_timeout(old, false, timeout);
  			dma_fence_put(old);
  			if (r)
-				return r;
+				return r < 0 ? r : 0;
  		}
  	}
 
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
  		return 0;
  	}
  	rcu_read_unlock();
-
-	r = dma_fence_wait(fence, false);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) · line 390

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
+	
+	long timeout;
+	timeout = amdgpu_fence_wait_timeout(ring);
+	r = dma_fence_wait_timeout(fence, false, timeout);
  	dma_fence_put(fence);
-	return r;
+	return r < 0 ? r : 0;
  }
 
  /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#4 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 7 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Quoted 8 lines

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence back. Why we need to wait here forever? We'd better not use forever wait which will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

Quoted 96 lines

Regards,
Christian.

Quoted 4 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 8 lines
  1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3 -0
@@ -41,6 +41,8 @@
  #include "amdgpu.h"
  #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
  /*
   * Fences
   * Fences mark an event in the GPUs pipeline and are used @@ -104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32

seq)

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring *ring,

Quoted 38 lines
  *drv->cpu_addr = cpu_to_le32(seq);
  }
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +31 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) {
+long tmo_gfx, tmo_mm, tmo;
+struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT;
+if (amdgpu_sriov_vf(adev)) {
+tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT;
+}
+if (amdgpu_sriov_runtime(adev)) {
+tmo_gfx = 8 * AMDGPU_FENCE_TIMEOUT;
+} else if (adev->gmc.xgmi.hive_id) {
+tmo_gfx = AMDGPU_FENCE_GFX_XGMI_TIMEOUT;
+}
+if (ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE ||
+ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+
  /**
   * amdgpu_fence_read - read a fence value
   *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
  rcu_read_unlock();
 
  if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring);
+r = dma_fence_wait_timeout(old, false, timeout);
  dma_fence_put(old);
  if (r)
-return r;
+return r < 0 ? r : 0;
  }
  }
 
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -3
  return 0;
  }
  rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring);
+r = dma_fence_wait_timeout(fence, false, timeout);
  dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
  }
 
Quoted 13 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
  /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463 df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

maintainer
#5 lore raw

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 12 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 10 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

Quoted 7 lines
Quoted 4 lines

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence back. Why we need to wait here forever? We'd better not use forever wait which will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory leak or corruption.

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Regards,
Christian.

Quoted 95 lines
Quoted 95 lines

Regards,
Christian.

Quoted 3 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 8 lines
   1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +3 -0
@@ -41,6 +41,8 @@
   #include "amdgpu.h"
   #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
   /*
    * Fences
    * Fences mark an event in the GPUs pipeline and are used @@ -104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32

seq)

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring *ring,

Quoted 38 lines
   *drv->cpu_addr = cpu_to_le32(seq);
   }
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +31 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) {
+long tmo_gfx, tmo_mm, tmo;
+struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT;
+if (amdgpu_sriov_vf(adev)) {
+tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT;
+}
+if (amdgpu_sriov_runtime(adev)) {
+tmo_gfx = 8 * AMDGPU_FENCE_TIMEOUT;
+} else if (adev->gmc.xgmi.hive_id) {
+tmo_gfx = AMDGPU_FENCE_GFX_XGMI_TIMEOUT;
+}
+if (ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE ||
+ring->funcs->type == AMDGPU_RING_TYPE_UVD_ENC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_DEC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+
   /**
    * amdgpu_fence_read - read a fence value
    *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
   rcu_read_unlock();
 
   if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring);
+r = dma_fence_wait_timeout(old, false, timeout);
   dma_fence_put(old);
   if (r)
-return r;
+return r < 0 ? r : 0;
   }
   }
 
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -3
   return 0;
   }
   rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring);
+r = dma_fence_wait_timeout(fence, false, timeout);
   dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
   }
 
Quoted 12 lines
Quoted 12 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
   /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463 df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#6 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 37 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Thursday, January 14, 2021 9:50 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 12 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 10 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

Quoted 7 lines
Quoted 4 lines

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence back. Why we need to wait here forever? We'd better not use forever wait which

will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory leak or corruption.

Quoted 6 lines

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Yes, we hit the issue when vcn ib test fail. Could you give some suggestions about how to fix this?

[  958.301685] failed to read reg:1a6c0
[  959.036645] gmc_v10_0_process_interrupt: 42 callbacks suppressed
[  959.036653] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.038043] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.039014] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.040202] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.041174] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.042353] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.043325] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.044508] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.045480] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.046659] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.047631] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.048815] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.049787] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.050973] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.051950] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.053123] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  967.208705] amdgpu 0000:00:07.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on vcn_enc0 (-110).
[  967.209879] [drm:amdgpu_device_init [amdgpu]] *ERROR* ib ring test failed (-110).
[ 1209.384668] INFO: task modprobe:23957 blocked for more than 120 seconds.
[ 1209.385605]       Tainted: G           OE     5.4.0-58-generic #64~18.04.1-Ubuntu
[ 1209.386451] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1209.387342] modprobe        D    0 23957   1221 0x80004006
[ 1209.387344] Call Trace:
[ 1209.387354]  __schedule+0x293/0x720
[ 1209.387356]  schedule+0x33/0xa0
[ 1209.387357]  schedule_timeout+0x1d3/0x320
[ 1209.387359]  ? schedule+0x33/0xa0
[ 1209.387360]  ? schedule_timeout+0x1d3/0x320
[ 1209.387363]  dma_fence_default_wait+0x1b2/0x1e0
[ 1209.387364]  ? dma_fence_release+0x130/0x130
[ 1209.387366]  dma_fence_wait_timeout+0xfd/0x110
[ 1209.387773]  amdgpu_fence_wait_empty+0x90/0xc0 [amdgpu]
[ 1209.387839]  amdgpu_fence_driver_fini+0xd6/0x110 [amdgpu]
Quoted 109 lines

Regards,
Christian.

Quoted 31 lines
Quoted 31 lines

Regards,
Christian.

Quoted 3 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 9 lines
   1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
   #include "amdgpu.h"

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -1
   #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
   /*
    * Fences
    * Fences mark an event in the GPUs pipeline and are used @@
-104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring,
+u32

seq)

Quoted 3 lines
   *drv->cpu_addr = cpu_to_le32(seq);
   }
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) { long
+tmo_gfx, tmo_mm, tmo; struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT; if

(amdgpu_sriov_vf(adev))

Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+{ tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT; } if
+(amdgpu_sriov_runtime(adev)) { tmo_gfx = 8 *

AMDGPU_FENCE_TIMEOUT;

Quoted 50 lines
Quoted 50 lines
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +14 -0
+} else if (adev->gmc.xgmi.hive_id) { tmo_gfx =
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT; } if (ring->funcs->type ==
+AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE || type ==
+ring->funcs->AMDGPU_RING_TYPE_UVD_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_DEC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+
   /**
    * amdgpu_fence_read - read a fence value
    *

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring · line 200

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring
 *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
   rcu_read_unlock();
 
   if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(old, false, timeout);
   dma_fence_put(old);
   if (r)
-return r;
+return r < 0 ? r : 0;
   }
   }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -3
   return 0;
   }
   rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(fence, false, timeout);
   dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
   }
 
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
   /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
ts.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463

Quoted 1 line
Quoted 1 line
Quoted 8 lines

df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7

Quoted 1 line
Quoted 1 line

C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw

Quoted 1 line
Quoted 1 line

MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda

Quoted 1 line
Quoted 1 line

ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve

Quoted 1 line
Quoted 1 line

d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

maintainer
#7 lore raw

Mhm, we could change amdgpu_fence_wait_empty() to timeout. But I think that waiting forever here is intentional and the right thing to do.

What happens is that we wait for the hardware to make sure that nothing is writing to any memory before we unload the driver.

Now the VCN block has crashed and doesn't respond, but we can't guarantee that it is not accidentally writing anywhere.

The only alternative we have is to time out and proceed with the driver unload, risking corrupting the memory we free during that should the hardware continue to do something.

Regards,
Christian.

Am 18.01.21 um 03:01 schrieb Deng, Emily:

Quoted 110 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 37 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Thursday, January 14, 2021 9:50 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 12 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 10 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

Quoted 7 lines
Quoted 4 lines

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence back. Why we need to wait here forever? We'd better not use forever wait which

will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory leak or corruption.

Quoted 6 lines

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Yes, we hit the issue when vcn ib test fail. Could you give some suggestions about how to fix this?

[  958.301685] failed to read reg:1a6c0
[  959.036645] gmc_v10_0_process_interrupt: 42 callbacks suppressed
[  959.036653] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.038043] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.039014] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.040202] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.041174] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.042353] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.043325] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.044508] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.045480] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.046659] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.047631] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.048815] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.049787] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.050973] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  959.051950] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0 ring:0 vmid:0 pasid:0, for process  pid 0 thread  pid 0)
[  959.053123] amdgpu 0000:00:07.0:   in page starting at address 0x0000000000567000 from client 18
[  967.208705] amdgpu 0000:00:07.0: [drm:amdgpu_ib_ring_tests [amdgpu]] *ERROR* IB test failed on vcn_enc0 (-110).
[  967.209879] [drm:amdgpu_device_init [amdgpu]] *ERROR* ib ring test failed (-110).
[ 1209.384668] INFO: task modprobe:23957 blocked for more than 120 seconds.
[ 1209.385605]       Tainted: G           OE     5.4.0-58-generic #64~18.04.1-Ubuntu
[ 1209.386451] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1209.387342] modprobe        D    0 23957   1221 0x80004006
[ 1209.387344] Call Trace:
[ 1209.387354]  __schedule+0x293/0x720
[ 1209.387356]  schedule+0x33/0xa0
[ 1209.387357]  schedule_timeout+0x1d3/0x320
[ 1209.387359]  ? schedule+0x33/0xa0
[ 1209.387360]  ? schedule_timeout+0x1d3/0x320
[ 1209.387363]  dma_fence_default_wait+0x1b2/0x1e0
Quoted 118 lines
[ 1209.387364]  ? dma_fence_release+0x130/0x130
[ 1209.387366]  dma_fence_wait_timeout+0xfd/0x110
[ 1209.387773]  amdgpu_fence_wait_empty+0x90/0xc0 [amdgpu]
[ 1209.387839]  amdgpu_fence_driver_fini+0xd6/0x110 [amdgpu]
Quoted 111 lines

Regards,
Christian.

Quoted 30 lines
Quoted 30 lines

Regards,
Christian.

Quoted 3 lines
---
    drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 9 lines
    1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
    #include "amdgpu.h"

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -1
    #include "amdgpu_trace.h"
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
    /*
     * Fences
     * Fences mark an event in the GPUs pipeline and are used @@
-104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring *ring,
+u32

seq)

Quoted 3 lines
    *drv->cpu_addr = cpu_to_le32(seq);
    }
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) { long
+tmo_gfx, tmo_mm, tmo; struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT; if

(amdgpu_sriov_vf(adev))

Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+{ tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT; } if
+(amdgpu_sriov_runtime(adev)) { tmo_gfx = 8 *

AMDGPU_FENCE_TIMEOUT;

Quoted 50 lines
Quoted 50 lines
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +14 -0
+} else if (adev->gmc.xgmi.hive_id) { tmo_gfx =
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT; } if (ring->funcs->type ==
+AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE || type ==
+ring->funcs->AMDGPU_RING_TYPE_UVD_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_DEC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+
    /**
     * amdgpu_fence_read - read a fence value
     *

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring · line 200

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring
 *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
    rcu_read_unlock();
 
    if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(old, false, timeout);
    dma_fence_put(old);
    if (r)
-return r;
+return r < 0 ? r : 0;
    }
    }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -3
    return 0;
    }
    rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(fence, false, timeout);
    dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
    }
 
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
    /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
ts.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463 df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve

Quoted 1 line
Quoted 1 line

d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#8 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 13 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Monday, January 18, 2021 3:49 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Mhm, we could change amdgpu_fence_wait_empty() to timeout. But I think that waiting forever here is intentional and the right thing to do.

What happens is that we wait for the hardware to make sure that nothing is writing to any memory before we unload the driver.

Quoted 6 lines

Now the VCN block has crashed and doesn't respond, but we can't guarantee that it is not accidentally writing anywhere.

The only alternative we have is to time out and proceed with the driver unload, risking corrupting the memory we free during that should the hardware continue to do something.

Hi Christian,
Thanks your suggestion, but still not quite clearly, could you detail the solution to avoid kernel not lockup?

Quoted 255 lines

Regards,
Christian.

Am 18.01.21 um 03:01 schrieb Deng, Emily:

Quoted 40 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 38 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Thursday, January 14, 2021 9:50 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 12 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 10 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

Quoted 8 lines
Quoted 4 lines

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence back. Why we need to wait here forever? We'd better not use forever wait which

will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory leak or corruption.

Quoted 7 lines
Quoted 6 lines

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Yes, we hit the issue when vcn ib test fail. Could you give some suggestions

about how to fix this?

Quoted 8 lines
[  958.301685] failed to read reg:1a6c0
[  959.036645] gmc_v10_0_process_interrupt: 42 callbacks suppressed
[  959.036653] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.038043] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.039014] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.040202] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.041174] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.042353] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.043325] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.044508] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.045480] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.046659] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.047631] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.048815] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.049787] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.050973] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 5 lines
[  959.051950] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.053123] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 2 lines
[  967.208705] amdgpu 0000:00:07.0: [drm:amdgpu_ib_ring_tests [amdgpu]]

*ERROR* IB test failed on vcn_enc0 (-110).

Quoted 2 lines
[  967.209879] [drm:amdgpu_device_init [amdgpu]] *ERROR* ib ring test

failed (-110).

Quoted 4 lines


[ 1209.384668] INFO: task modprobe:23957 blocked for more than 120

seconds.

Quoted 2 lines
[ 1209.385605]       Tainted: G           OE     5.4.0-58-generic #64~18.04.1-

Ubuntu

Quoted 2 lines
[ 1209.386451] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"

disables this message.

Quoted 17 lines
[ 1209.387342] modprobe        D    0 23957   1221 0x80004006
[ 1209.387344] Call Trace:
[ 1209.387354]  __schedule+0x293/0x720
[ 1209.387356]  schedule+0x33/0xa0
[ 1209.387357]  schedule_timeout+0x1d3/0x320
[ 1209.387359]  ? schedule+0x33/0xa0
[ 1209.387360]  ? schedule_timeout+0x1d3/0x320
[ 1209.387363]  dma_fence_default_wait+0x1b2/0x1e0
Quoted 115 lines
[ 1209.387364]  ? dma_fence_release+0x130/0x130
[ 1209.387366]  dma_fence_wait_timeout+0xfd/0x110
[ 1209.387773]  amdgpu_fence_wait_empty+0x90/0xc0 [amdgpu]
[ 1209.387839]  amdgpu_fence_driver_fini+0xd6/0x110 [amdgpu]
Quoted 108 lines

Regards,
Christian.

Quoted 31 lines
Quoted 31 lines

Regards,
Christian.

Quoted 3 lines
---
    drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 10 lines
    1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
    #include "amdgpu.h"
    #include "amdgpu_trace.h"

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
    /*
     * Fences
     * Fences mark an event in the GPUs pipeline and are used @@
-104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring
+*ring,
+u32

seq)

Quoted 3 lines
    *drv->cpu_addr = cpu_to_le32(seq);
    }
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) { long
+tmo_gfx, tmo_mm, tmo; struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT; if

(amdgpu_sriov_vf(adev))

Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+{ tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT; } if
+(amdgpu_sriov_runtime(adev)) { tmo_gfx = 8 *

AMDGPU_FENCE_TIMEOUT;

Quoted 59 lines
Quoted 59 lines
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +14 -0
+} else if (adev->gmc.xgmi.hive_id) { tmo_gfx =
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT; } if (ring->funcs->type ==
+AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE || type ==
+ring->funcs->AMDGPU_RING_TYPE_UVD_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_DEC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+
    /**
     * amdgpu_fence_read - read a fence value
     *

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring · line 200

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring
 *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
    rcu_read_unlock();
 
    if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(old, false, timeout);
    dma_fence_put(old);
    if (r)
-return r;
+return r < 0 ? r : 0;
    }
    }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -2
    return 0;
    }
    rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) · line 390

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(fence, false, timeout);
    dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
    }
 
    /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fl
is
ts.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463

Quoted 1 line
Quoted 1 line
Quoted 8 lines

df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7

Quoted 1 line
Quoted 1 line

C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw

Quoted 1 line
Quoted 1 line

MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda

Quoted 1 line
Quoted 1 line

ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#9 lore raw

Am 18.01.21 um 12:56 schrieb Deng, Emily:

Quoted 15 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 13 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Monday, January 18, 2021 3:49 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Mhm, we could change amdgpu_fence_wait_empty() to timeout. But I think that waiting forever here is intentional and the right thing to do.

What happens is that we wait for the hardware to make sure that nothing is writing to any memory before we unload the driver.

Quoted 8 lines
Quoted 6 lines

Now the VCN block has crashed and doesn't respond, but we can't guarantee that it is not accidentally writing anywhere.

The only alternative we have is to time out and proceed with the driver unload, risking corrupting the memory we free during that should the hardware continue to do something.

Hi Christian,
Thanks your suggestion, but still not quite clearly, could you detail the solution to avoid kernel not lockup?

Well as I said that the kernel locks up is intentional here.

Regards,
Christian.

Quoted 243 lines
Quoted 243 lines

Regards,
Christian.

Am 18.01.21 um 03:01 schrieb Deng, Emily:

Quoted 40 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 38 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Thursday, January 14, 2021 9:50 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 12 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 10 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

Quoted 8 lines
Quoted 4 lines

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence back. Why we need to wait here forever? We'd better not use forever wait which

will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory leak or corruption.

Quoted 7 lines
Quoted 6 lines

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Yes, we hit the issue when vcn ib test fail. Could you give some suggestions

about how to fix this?

Quoted 8 lines
[  958.301685] failed to read reg:1a6c0
[  959.036645] gmc_v10_0_process_interrupt: 42 callbacks suppressed
[  959.036653] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.038043] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.039014] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.040202] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.041174] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.042353] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.043325] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.044508] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.045480] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.046659] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.047631] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.048815] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.049787] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.050973] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.051950] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.053123] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 1 line
[  967.208705] amdgpu 0000:00:07.0: [drm:amdgpu_ib_ring_tests [amdgpu]]

*ERROR* IB test failed on vcn_enc0 (-110).

Quoted 1 line
[  967.209879] [drm:amdgpu_device_init [amdgpu]] *ERROR* ib ring test

failed (-110).

Quoted 3 lines

[ 1209.384668] INFO: task modprobe:23957 blocked for more than 120

seconds.

Quoted 1 line
[ 1209.385605]       Tainted: G           OE     5.4.0-58-generic #64~18.04.1-

Ubuntu

Quoted 1 line
[ 1209.386451] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"

disables this message.

Quoted 16 lines
[ 1209.387342] modprobe        D    0 23957   1221 0x80004006
[ 1209.387344] Call Trace:
[ 1209.387354]  __schedule+0x293/0x720
[ 1209.387356]  schedule+0x33/0xa0
[ 1209.387357]  schedule_timeout+0x1d3/0x320
[ 1209.387359]  ? schedule+0x33/0xa0
[ 1209.387360]  ? schedule_timeout+0x1d3/0x320
[ 1209.387363]  dma_fence_default_wait+0x1b2/0x1e0
Quoted 114 lines
[ 1209.387364]  ? dma_fence_release+0x130/0x130
[ 1209.387366]  dma_fence_wait_timeout+0xfd/0x110
[ 1209.387773]  amdgpu_fence_wait_empty+0x90/0xc0 [amdgpu]
[ 1209.387839]  amdgpu_fence_driver_fini+0xd6/0x110 [amdgpu]
Quoted 107 lines

Regards,
Christian.

Quoted 31 lines
Quoted 31 lines

Regards,
Christian.

Quoted 3 lines
---
     drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 10 lines
     1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
     #include "amdgpu.h"
     #include "amdgpu_trace.h"

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
 
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000) #define
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
     /*
      * Fences
      * Fences mark an event in the GPUs pipeline and are used @@
-104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring
+*ring,
+u32

seq)

Quoted 3 lines
     *drv->cpu_addr = cpu_to_le32(seq);
     }
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) { long
+tmo_gfx, tmo_mm, tmo; struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT; if

(amdgpu_sriov_vf(adev))

Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+{ tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT; } if
+(amdgpu_sriov_runtime(adev)) { tmo_gfx = 8 *

AMDGPU_FENCE_TIMEOUT;

Quoted 59 lines
Quoted 59 lines
Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +14 -0
+} else if (adev->gmc.xgmi.hive_id) { tmo_gfx =
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT; } if (ring->funcs->type ==
+AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE || type ==
+ring->funcs->AMDGPU_RING_TYPE_UVD_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_DEC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
+tmo = tmo_gfx;
+return tmo;
+}
+
     /**
      * amdgpu_fence_read - read a fence value
      *

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct amdgpu_ring · line 200

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct amdgpu_ring
 *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
     rcu_read_unlock();
 
     if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(old, false, timeout);
     dma_fence_put(old);
     if (r)
-return r;
+return r < 0 ? r : 0;
     }
     }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -2
     return 0;
     }
     rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -1
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(fence, false, timeout);
     dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
     }
 
     /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fl
is
ts.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463 df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve

Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#10 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 28 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Tuesday, January 19, 2021 12:04 AM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; Sun, Roy <[email protected]>; amd- [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 18.01.21 um 12:56 schrieb Deng, Emily:

Quoted 18 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 16 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Monday, January 18, 2021 3:49 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Mhm, we could change amdgpu_fence_wait_empty() to timeout. But I think that waiting forever here is intentional and the right thing to do.

What happens is that we wait for the hardware to make sure that nothing is writing to any memory before we unload the driver.

Now the VCN block has crashed and doesn't respond, but we can't guarantee that it is not accidentally writing anywhere.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

The only alternative we have is to time out and proceed with the driver unload, risking corrupting the memory we free during that should the hardware continue to do something.

Hi Christian,
Thanks your suggestion, but still not quite clearly, could you detail the

solution to avoid kernel not lockup?

Well as I said that the kernel locks up is intentional here.

So you think the lock up is better than only some memory corruption? Because we could give more time, such as 60s to wait, I don't think the fence won't be signaled within 60s if the engine is good. So when the engine is ok, it won't cause memory corruption with the timeout. When engine is bad, the fence will never be signaled, so even we force completion, it still won't cause memory corruption. As for sriov, when engine is bad, we still could do recover, and do driver reload to make the driver works ok again, so we don't want the kernel lockup.

Quoted 285 lines

Regards,
Christian.

Quoted 10 lines
Quoted 10 lines

Regards,
Christian.

Am 18.01.21 um 03:01 schrieb Deng, Emily:

Quoted 6 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Thursday, January 14, 2021 9:50 PM
To: Deng, Emily <[email protected]>; Sun, Roy

<[email protected]>;

Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 15 lines

[email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 11 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 9 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence

back.

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines

Why we need to wait here forever? We'd better not use forever wait which

will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory

leak

Quoted 113 lines
Quoted 113 lines
Quoted 10 lines
Quoted 8 lines

or corruption.

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Yes, we hit the issue when vcn ib test fail. Could you give some suggestions

about how to fix this?

Quoted 8 lines
[  958.301685] failed to read reg:1a6c0
[  959.036645] gmc_v10_0_process_interrupt: 42 callbacks suppressed
[  959.036653] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.038043] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.039014] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.040202] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.041174] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.042353] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.043325] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.044508] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.045480] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.046659] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.047631] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.048815] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.049787] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.050973] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.051950] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.053123] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 2 lines
[  967.208705] amdgpu 0000:00:07.0: [drm:amdgpu_ib_ring_tests

[amdgpu]]

*ERROR* IB test failed on vcn_enc0 (-110).

Quoted 2 lines
[  967.209879] [drm:amdgpu_device_init [amdgpu]] *ERROR* ib ring

test

failed (-110).

Quoted 3 lines

[ 1209.384668] INFO: task modprobe:23957 blocked for more than 120

seconds.

Quoted 1 line
[ 1209.385605]       Tainted: G           OE     5.4.0-58-generic #64~18.04.1-

Ubuntu

Quoted 1 line
[ 1209.386451] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"

disables this message.

Quoted 44 lines
[ 1209.387342] modprobe        D    0 23957   1221 0x80004006
[ 1209.387344] Call Trace:
[ 1209.387354]  __schedule+0x293/0x720
[ 1209.387356]  schedule+0x33/0xa0
[ 1209.387357]  schedule_timeout+0x1d3/0x320
[ 1209.387359]  ? schedule+0x33/0xa0
[ 1209.387360]  ? schedule_timeout+0x1d3/0x320
[ 1209.387363]  dma_fence_default_wait+0x1b2/0x1e0
[ 1209.387364]  ? dma_fence_release+0x130/0x130
[ 1209.387366]  dma_fence_wait_timeout+0xfd/0x110
[ 1209.387773]  amdgpu_fence_wait_empty+0x90/0xc0 [amdgpu]
[ 1209.387839]  amdgpu_fence_driver_fini+0xd6/0x110 [amdgpu]
Quoted 21 lines

Regards,
Christian.

Quoted 18 lines
Quoted 18 lines

Regards,
Christian.

Quoted 3 lines
---
     drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 11 lines
     1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
     #include "amdgpu.h"
     #include "amdgpu_trace.h"
 
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c line 44

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -0
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000)

#define

Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Quoted 13 lines
Quoted 13 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -0
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
     /*
      * Fences
      * Fences mark an event in the GPUs pipeline and are used

@@
-104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+*ring,
+u32

seq)

Quoted 3 lines
     *drv->cpu_addr = cpu_to_le32(seq);
     }
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) { long
+tmo_gfx, tmo_mm, tmo; struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT; if

(amdgpu_sriov_vf(adev))

Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+{ tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT; } if
+(amdgpu_sriov_runtime(adev)) { tmo_gfx = 8 *

AMDGPU_FENCE_TIMEOUT;

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+} else if (adev->gmc.xgmi.hive_id) { tmo_gfx =
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT; } if (ring->funcs->type ==
+AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE || type ==
+ring->funcs->AMDGPU_RING_TYPE_UVD_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_DEC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+tmo = tmo_gfx;
+return tmo;
+}
+
     /**
      * amdgpu_fence_read - read a fence value
      *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct

amdgpu_ring

Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct · line 200

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
 *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
     rcu_read_unlock();
 
     if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(old, false, timeout);
     dma_fence_put(old);
     if (r)
-return r;
+return r < 0 ? r : 0;
     }
     }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -3
     return 0;
     }
     rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(fence, false, timeout);
     dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
     }
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
     /**

_______________________________________________
amd-gfx mailing list
[email protected]

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

Fl
is
ts.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463

Quoted 1 line
Quoted 1 line

df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7

Quoted 1 line
Quoted 1 line

C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw

Quoted 1 line
Quoted 1 line

MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda

Quoted 1 line
Quoted 1 line

ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve

Quoted 6 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flist
s.freedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEm

Quoted 1 line

ily.Deng%40amd.com%7C5a6d3a94dc7d4190477a08d8bbcaa509%7C3dd896 1fe4884e

Quoted 1 line

608e11a82d994e183d%7C0%7C0%7C637465826316473967%7CUnknown%7 CTWFpbGZsb3

Quoted 1 line

d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0% 3D%7

Quoted 1 line

C1000&amp;sdata=7XUhFg5H0FTO8RByofukETH7yBz9RBHKkUQmHZP3z34%3 D&amp;res

Quoted 1 line

erved=0

Quoted 11 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C5a6d3a94dc7d4190 477a08d8bbcaa509%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637465826316473967%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=7XUhFg5H0FTO8RByofukETH7yBz9RBHKkUQmHZP3z34%3D&amp;reserve d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

maintainer
#11 lore raw

Am 19.01.21 um 04:23 schrieb Deng, Emily:

Quoted 33 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 31 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Tuesday, January 19, 2021 12:04 AM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; Sun, Roy <[email protected]>; amd- [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 18.01.21 um 12:56 schrieb Deng, Emily:

Quoted 21 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 19 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Monday, January 18, 2021 3:49 PM
To: Deng, Emily <[email protected]>; Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Mhm, we could change amdgpu_fence_wait_empty() to timeout. But I think that waiting forever here is intentional and the right thing to do.

What happens is that we wait for the hardware to make sure that nothing is writing to any memory before we unload the driver.

Now the VCN block has crashed and doesn't respond, but we can't guarantee that it is not accidentally writing anywhere.

The only alternative we have is to time out and proceed with the driver unload, risking corrupting the memory we free during that should the hardware continue to do something.

Quoted 6 lines
Quoted 5 lines
Quoted 2 lines

Hi Christian,
Thanks your suggestion, but still not quite clearly, could you detail the

solution to avoid kernel not lockup?

Well as I said that the kernel locks up is intentional here.

So you think the lock up is better than only some memory corruption?

Yes exactly.

Quoted 2 lines

Because we could give more time, such as 60s to wait, I don't think the fence won't be signaled within 60s if the engine is good. So
when the engine is ok, it won't cause memory corruption with the timeout. When engine is bad, the fence will never be signaled, so even we force completion, it still won't cause memory corruption.

Unfortunately it's not that easy. See for example some engines keep updating statistics (how many bytes written, samples encoded etc...) even when the engines is stuck in an endless loop.

Quoted 2 lines

As for sriov, when engine
is bad, we still could do recover, and do driver reload to make the driver works ok again, so we don't want the kernel lockup.

Well that is certainly something you could try, e.g. wait for empty with a timeout and if that times out triggers do a GPU reset and wait again.

Alternative would be to see if we could disable system memory access from the device all together with the PCIe config regs if we can't get it idle again.

Regards,
Christian.

Quoted 273 lines
Quoted 273 lines

Regards,
Christian.

Quoted 10 lines
Quoted 10 lines

Regards,
Christian.

Am 18.01.21 um 03:01 schrieb Deng, Emily:

Quoted 6 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Thursday, January 14, 2021 9:50 PM
To: Deng, Emily <[email protected]>; Sun, Roy

<[email protected]>;

Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 15 lines

[email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 14.01.21 um 03:00 schrieb Deng, Emily:

Quoted 11 lines

[AMD Official Use Only - Internal Distribution Only]

Quoted 9 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Wednesday, January 13, 2021 10:03 PM
To: Sun, Roy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: change the fence ring wait timeout

Am 13.01.21 um 07:36 schrieb Roy Sun:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

This fix bug where when the engine hang, the fence ring will wait without quit and cause kernel crash

NAK, this blocking is intentional unlimited because otherwise we will cause a memory corruption.

What is the actual bug you are trying to fix here?

When some engine hang during initialization, the IB test will fail, and fence will never come back, then never could wait the fence

back.

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines

Why we need to wait here forever? We'd better not use forever wait which

will cause kernel crash and lockup. And we have amdgpu_fence_driver_force_completion will let memory free. We should remove all those forever wait, and replace them with one timeout, and do the correct error process if timeout.

This wait here is to make sure we never overwrite the software fence ring buffer. Without it we would not signal all fences in amdgpu_fence_driver_force_completion() and cause either memory

leak

Quoted 112 lines
Quoted 112 lines
Quoted 10 lines
Quoted 8 lines

or corruption.

Waiting here forever is the right thing to do even when that means that the submission thread gets stuck forever, cause that is still better than memory corruption.

But this should never happen in practice and is only here for precaution. So do you really see that in practice?

Yes, we hit the issue when vcn ib test fail. Could you give some suggestions

about how to fix this?

Quoted 8 lines
[  958.301685] failed to read reg:1a6c0
[  959.036645] gmc_v10_0_process_interrupt: 42 callbacks suppressed
[  959.036653] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.038043] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.039014] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.040202] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.041174] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.042353] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.043325] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.044508] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.045480] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.046659] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.047631] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.048815] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.049787] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.050973] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 4 lines
[  959.051950] amdgpu 0000:00:07.0: [mmhub] page fault (src_id:0

ring:0 vmid:0 pasid:0, for process pid 0 thread pid 0)

[  959.053123] amdgpu 0000:00:07.0:   in page starting at address

0x0000000000567000 from client 18

Quoted 2 lines
[  967.208705] amdgpu 0000:00:07.0: [drm:amdgpu_ib_ring_tests

[amdgpu]]

*ERROR* IB test failed on vcn_enc0 (-110).

Quoted 2 lines
[  967.209879] [drm:amdgpu_device_init [amdgpu]] *ERROR* ib ring

test

failed (-110).

Quoted 2 lines
[ 1209.384668] INFO: task modprobe:23957 blocked for more than 120

seconds.

Quoted 1 line
[ 1209.385605]       Tainted: G           OE     5.4.0-58-generic #64~18.04.1-

Ubuntu

Quoted 1 line
[ 1209.386451] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"

disables this message.

Quoted 44 lines
[ 1209.387342] modprobe        D    0 23957   1221 0x80004006
[ 1209.387344] Call Trace:
[ 1209.387354]  __schedule+0x293/0x720
[ 1209.387356]  schedule+0x33/0xa0
[ 1209.387357]  schedule_timeout+0x1d3/0x320
[ 1209.387359]  ? schedule+0x33/0xa0
[ 1209.387360]  ? schedule_timeout+0x1d3/0x320
[ 1209.387363]  dma_fence_default_wait+0x1b2/0x1e0
[ 1209.387364]  ? dma_fence_release+0x130/0x130
[ 1209.387366]  dma_fence_wait_timeout+0xfd/0x110
[ 1209.387773]  amdgpu_fence_wait_empty+0x90/0xc0 [amdgpu]
[ 1209.387839]  amdgpu_fence_driver_fini+0xd6/0x110 [amdgpu]
Quoted 21 lines

Regards,
Christian.

Quoted 18 lines
Quoted 18 lines

Regards,
Christian.

Quoted 3 lines
---
      drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 48

++++++++++++++++++++---

Quoted 11 lines
      1 file changed, 43 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
index 6b0aeee61b8b..738ea65077ea 100644
@@ -41,6 +41,8 @@
      #include "amdgpu.h"
      #include "amdgpu_trace.h"
 
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c line 44

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -0
+#define AMDGPU_FENCE_TIMEOUT  msecs_to_jiffies(1000)

#define

Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Quoted 13 lines
Quoted 13 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 -0
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT msecs_to_jiffies(2000)
      /*
       * Fences
       * Fences mark an event in the GPUs pipeline and are used

@@
-104,6
+106,38 @@ static void amdgpu_fence_write(struct amdgpu_ring

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+*ring,
+u32

seq)

Quoted 3 lines
      *drv->cpu_addr = cpu_to_le32(seq);
      }
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+/**
+ * amdgpu_fence_wait_timeout - get the fence wait timeout
+ *
+ * @ring: ring the fence is associated with
+ *
+ * Returns the value of the fence wait timeout.
+ */
+long amdgpu_fence_wait_timeout(struct amdgpu_ring *ring) { long
+tmo_gfx, tmo_mm, tmo; struct amdgpu_device *adev = ring->adev;
+tmo_mm = tmo_gfx = AMDGPU_FENCE_TIMEOUT; if

(amdgpu_sriov_vf(adev))

Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +2 -0
+{ tmo_mm = 8 * AMDGPU_FENCE_TIMEOUT; } if
+(amdgpu_sriov_runtime(adev)) { tmo_gfx = 8 *

AMDGPU_FENCE_TIMEOUT;

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +10 -0
+} else if (adev->gmc.xgmi.hive_id) { tmo_gfx =
+AMDGPU_FENCE_GFX_XGMI_TIMEOUT; } if (ring->funcs->type ==
+AMDGPU_RING_TYPE_UVD ||
+ring->funcs->type == AMDGPU_RING_TYPE_VCE || type ==
+ring->funcs->AMDGPU_RING_TYPE_UVD_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_DEC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_ENC || type ==
+ring->funcs->AMDGPU_RING_TYPE_VCN_JPEG)
+tmo = tmo_mm;
+else
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -0
+tmo = tmo_gfx;
+return tmo;
+}
+
      /**
       * amdgpu_fence_read - read a fence value
       *
@@ -166,10 +200,12 @@ int amdgpu_fence_emit(struct

amdgpu_ring

Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_emit(struct · line 200

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
 *ring,

struct dma_fence **f,

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +4 -2
      rcu_read_unlock();
 
      if (old) {
-r = dma_fence_wait(old, false);
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(old, false, timeout);
      dma_fence_put(old);
      if (r)
-return r;
+return r < 0 ? r : 0;
      }
      }
@@ -343,10 +379,12 @@ int amdgpu_fence_wait_empty(struct

amdgpu_ring *ring)

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -3
      return 0;
      }
      rcu_read_unlock();
-
-r = dma_fence_wait(fence, false);
+
+long timeout;
+timeout = amdgpu_fence_wait_timeout(ring); r =
+dma_fence_wait_timeout(fence, false, timeout);
      dma_fence_put(fence);
-return r;
+return r < 0 ? r : 0;
      }
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c int amdgpu_fence_wait_empty(struct · line 390

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +0 -0
      /**

_______________________________________________
amd-gfx mailing list
[email protected]

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

Fl
is
ts.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C8b116229938b463 df87f08d8b7cbf607%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637461433936049544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=HOcLHmmblOUHXATFBl5HC6LOmFq0oXglAh2GFwd6sus%3D&amp;reserve

Quoted 6 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flist
s.freedesktop.org%2Fmailman%2Flistinfo%2Famd-

gfx&amp;data=04%7C01%7CEm ily.Deng%40amd.com%7C5a6d3a94dc7d4190477a08d8bbcaa509%7C3dd896 1fe4884e 608e11a82d994e183d%7C0%7C0%7C637465826316473967%7CUnknown%7 CTWFpbGZsb3 d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0% 3D%7 C1000&amp;sdata=7XUhFg5H0FTO8RByofukETH7yBz9RBHKkUQmHZP3z34%3 D&amp;res

Quoted 1 line

erved=0

Quoted 11 lines
Quoted 11 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.f
reedesktop.org%2Fmailman%2Flistinfo%2Famd- gfx&amp;data=04%7C01%7CEmily.Deng%40amd.com%7C5a6d3a94dc7d4190 477a08d8bbcaa509%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7 C637465826316473967%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAw MDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sda ta=7XUhFg5H0FTO8RByofukETH7yBz9RBHKkUQmHZP3z34%3D&amp;reserve d=0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx