Skip to contentSkip to conversation

drm/amdgpu: drop kiq access while in reset

PATCH In review Vignesh Chander 3 participants 9 messages Jun 24, 2024

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: drop kiq access while in reset

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++----------
 1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +21 -17
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 
 	if ((reg * 4) < adev->rmmio_size) {
 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, 0);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				ret = amdgpu_kiq_rreg(adev, reg, 0);
+				up_read(&adev->reset_domain->sem);
+			}
 		} else {
 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 		}
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
 							 &rlcg_flag)) {
 			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+				up_read(&adev->reset_domain->sem);
+			}
 		} else {
 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 		}
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device *adev,
 
 	if ((reg * 4) < adev->rmmio_size) {
 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, 0);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				amdgpu_kiq_wreg(adev, reg, v, 0);
+				up_read(&adev->reset_domain->sem);
+			}
 		} else {
 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 		}
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
 							 &rlcg_flag)) {
 			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-			up_read(&adev->reset_domain->sem);
-		} else {
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+				up_read(&adev->reset_domain->sem);
+			}
+		    } else {
 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 		}
 	} else {
Signature · 1 line
2.25.1
Reviewed-by on v1 1/1
#1 lore raw
Quoted 4 lines · Vignesh Chander

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 74 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++----------
 1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +21 -17
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 
 	if ((reg * 4) < adev->rmmio_size) {
 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, 0);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				ret = amdgpu_kiq_rreg(adev, reg, 0);
+				up_read(&adev->reset_domain->sem);
+			}
 		} else {
 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 		}
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
 							 &rlcg_flag)) {
 			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+				up_read(&adev->reset_domain->sem);
+			}
 		} else {
 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 		}
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device *adev,
 
 	if ((reg * 4) < adev->rmmio_size) {
 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, 0);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				amdgpu_kiq_wreg(adev, reg, v, 0);
+				up_read(&adev->reset_domain->sem);
+			}
 		} else {
 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 		}
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
 							 &rlcg_flag)) {
 			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-			up_read(&adev->reset_domain->sem);
-		} else {
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+				up_read(&adev->reset_domain->sem);
+			}
+		    } else {
 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 		}
 	} else {
#2 lore raw

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 21 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 10 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++----------
  1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 
Quoted 11 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, · line 620

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
  	if ((reg * 4) < adev->rmmio_size) {
  		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, 0);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				ret = amdgpu_kiq_rreg(adev, reg, 0);
+				up_read(&adev->reset_domain->sem);
+			}

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

Regards,
Christian.

Quoted 53 lines
Quoted 53 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +16 -13
  		} else {
  			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
  		}
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
  							 &rlcg_flag)) {
  			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
  		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+				up_read(&adev->reset_domain->sem);
+			}
  		} else {
  			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
  		}
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device *adev,
 
  	if ((reg * 4) < adev->rmmio_size) {
  		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, 0);
-			up_read(&adev->reset_domain->sem);
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				amdgpu_kiq_wreg(adev, reg, v, 0);
+				up_read(&adev->reset_domain->sem);
+			}
  		} else {
  			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
  		}
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
  							 &rlcg_flag)) {
  			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
  		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-		    amdgpu_sriov_runtime(adev) &&
-		    down_read_trylock(&adev->reset_domain->sem)) {
-			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-			up_read(&adev->reset_domain->sem);
-		} else {
+		    amdgpu_sriov_runtime(adev) {
+			if (down_read_trylock(&adev->reset_domain->sem)) {
+				amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+				up_read(&adev->reset_domain->sem);
+			}
+		    } else {
  			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
  		}
  	} else {
#3 lore raw
Quoted 29 lines · Christian König

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 28 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 17 lines

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++----------

  1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct
 amdgpu_device *adev,
         if ((reg * 4) < adev->rmmio_size) {

          if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && -            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) { -            ret = amdgpu_kiq_rreg(adev, reg, 0); -            up_read(&adev->reset_domain->sem);

Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) { +                ret = amdgpu_kiq_rreg(adev, reg, 0); +                up_read(&adev->reset_domain->sem);
+            }

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

In earlier logic, if it fails to get the lock, it takes the 'else' path. The 'else' path is not meant for sriov/vf.

Thanks,
Lijo

Quoted 61 lines

Regards,
Christian.

Quoted 57 lines
Quoted 57 lines

          } else {
              ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct
 amdgpu_device *adev,
                                &rlcg_flag)) {
               ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag,
 GET_INST(GC, xcc_id));
           } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
           } else {

              ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device
 *adev,
         if ((reg * 4) < adev->rmmio_size) {

          if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) { -            amdgpu_kiq_wreg(adev, reg, v, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) { +                amdgpu_kiq_wreg(adev, reg, v, 0);
+                up_read(&adev->reset_domain->sem);
+            }
          } else {
              writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -5
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct
 amdgpu_device *adev,
                                &rlcg_flag)) {
               amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
 GET_INST(GC, xcc_id));
           } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-            up_read(&adev->reset_domain->sem);
-        } else {
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
+            } else {
               writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));

          }
      } else {

#4 lore raw

Am 24.06.24 um 11:52 schrieb Lazar, Lijo:

Quoted 38 lines · Christian König
Quoted 27 lines

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 26 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 16 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38 ++++++++++++----------
   1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -3
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct
 amdgpu_device *adev,
         if ((reg * 4) < adev->rmmio_size) {
           if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, 0);
Quoted 9 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, · line 620

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -1
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, 0);
+                up_read(&adev->reset_domain->sem);
+            }

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

Quoted 2 lines

In earlier logic, if it fails to get the lock, it takes the 'else' path. The 'else' path is not meant for sriov/vf.

Yeah, but the read or write is then just silently dropped.

That can't be correct either.

Regards,
Christian.

Quoted 64 lines

Thanks,
Lijo

Quoted 60 lines

Regards,
Christian.

Quoted 57 lines
Quoted 57 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
           } else {
               ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
           }
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct
 amdgpu_device *adev,
                                &rlcg_flag)) {
               ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag,
 GET_INST(GC, xcc_id));
           } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
           } else {

              ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -9
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device
 *adev,
         if ((reg * 4) < adev->rmmio_size) {
           if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, 0);
+                up_read(&adev->reset_domain->sem);
+            }
           } else {
               writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
           }
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct
 amdgpu_device *adev,
                                &rlcg_flag)) {
               amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
 GET_INST(GC, xcc_id));
           } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-            up_read(&adev->reset_domain->sem);
-        } else {
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
+            } else {
               writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));

          }
      } else {

#5 lore raw
Quoted 40 lines · Christian König

Am 24.06.24 um 11:52 schrieb Lazar, Lijo:

Quoted 39 lines · Christian König
Quoted 37 lines

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 25 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 15 lines

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38
++++++++++++----------

   1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct
 amdgpu_device *adev,
          if ((reg * 4) < adev->rmmio_size) {

           if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && -            amdgpu_sriov_runtime(adev) &&

Quoted 8 lines
Quoted 8 lines

-            down_read_trylock(&adev->reset_domain->sem)) { -            ret = amdgpu_kiq_rreg(adev, reg, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) { +                ret = amdgpu_kiq_rreg(adev, reg, 0); +                up_read(&adev->reset_domain->sem);
+            }

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

Quoted 7 lines
Quoted 2 lines

In earlier logic, if it fails to get the lock, it takes the 'else' path. The 'else' path is not meant for sriov/vf.

Yeah, but the read or write is then just silently dropped.

That can't be correct either.

These are void funcs. Moreover, the drops will happen if there is concurrent access from another thread while a reset is going on. That is expected and those accesses during a reset won't help anyway.

Thanks,
Lijo

Quoted 68 lines

Regards,
Christian.

Quoted 64 lines

Thanks,
Lijo

Quoted 60 lines

Regards,
Christian.

Quoted 57 lines
Quoted 57 lines

           } else {
               ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct
 amdgpu_device *adev,
                                 &rlcg_flag)) {
                ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag,
 GET_INST(GC, xcc_id));
            } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
            } else {

               ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device
 *adev,
          if ((reg * 4) < adev->rmmio_size) {

           if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) { -            amdgpu_kiq_wreg(adev, reg, v, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) { +                amdgpu_kiq_wreg(adev, reg, v, 0);
+                up_read(&adev->reset_domain->sem);
+            }
           } else {
               writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -5
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct
 amdgpu_device *adev,
                                 &rlcg_flag)) {
                amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
 GET_INST(GC, xcc_id));
            } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-            up_read(&adev->reset_domain->sem);
-        } else {
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
+            } else {
                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));

           }
       } else {

#6 lore raw

Am 24.06.24 um 13:57 schrieb Lazar, Lijo:

Quoted 42 lines · Christian König
Quoted 41 lines

Am 24.06.24 um 11:52 schrieb Lazar, Lijo:

Quoted 40 lines · Christian König
Quoted 28 lines

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 27 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 17 lines
---
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38
++++++++++++----------
    1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -3
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct
 amdgpu_device *adev,
          if ((reg * 4) < adev->rmmio_size) {
            if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, 0);
Quoted 9 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, · line 620

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -1
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, 0);
+                up_read(&adev->reset_domain->sem);
+            }

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

In earlier logic, if it fails to get the lock, it takes the 'else' path. The 'else' path is not meant for sriov/vf.

Quoted 7 lines
Quoted 4 lines

Yeah, but the read or write is then just silently dropped.

That can't be correct either.

These are void funcs. Moreover, the drops will happen if there is concurrent access from another thread while a reset is going on. That is expected and those accesses during a reset won't help anyway.

Nope, Teddy has been working on that for a while as well.

Trying to make those accesses while the reset is going on is wrong in the first place. What we need to do is to grab the reset lock in the higher level function so that the whole sequences of reads and writes are protected.

What this logic here does is to use readl()/writel() from the reset thread itself. Dropping that is incorrect and could lead to broken reset.

So clear NAK from my side to this patch here.

Regards,
Christian.

Quoted 70 lines

Thanks,
Lijo

Quoted 66 lines

Regards,
Christian.

Quoted 63 lines

Thanks,
Lijo

Quoted 60 lines

Regards,
Christian.

Quoted 57 lines
Quoted 57 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
            } else {
                ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
            }
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct
 amdgpu_device *adev,
                                 &rlcg_flag)) {
                ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag,
 GET_INST(GC, xcc_id));
            } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
            } else {

               ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +11 -9
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device
 *adev,
          if ((reg * 4) < adev->rmmio_size) {
            if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, 0);
+                up_read(&adev->reset_domain->sem);
+            }
            } else {
                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
            }
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct
 amdgpu_device *adev,
                                 &rlcg_flag)) {
                amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
 GET_INST(GC, xcc_id));
            } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-            up_read(&adev->reset_domain->sem);
-        } else {
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
+            } else {
                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));

           }
       } else {

#7 lore raw
Quoted 46 lines · Christian König

Am 24.06.24 um 13:57 schrieb Lazar, Lijo:

Quoted 45 lines · Christian König
Quoted 44 lines

Am 24.06.24 um 11:52 schrieb Lazar, Lijo:

Quoted 38 lines · Christian König
Quoted 28 lines

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 27 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 17 lines

---
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38
++++++++++++----------

    1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct
 amdgpu_device *adev,
           if ((reg * 4) < adev->rmmio_size) {

            if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && -            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) { -            ret = amdgpu_kiq_rreg(adev, reg, 0);

Quoted 9 lines
Quoted 6 lines
Quoted 6 lines

-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) { +                ret = amdgpu_kiq_rreg(adev, reg, 0); +                up_read(&adev->reset_domain->sem);
+            }

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

Quoted 3 lines

In earlier logic, if it fails to get the lock, it takes the 'else' path.
The 'else' path is not meant for sriov/vf.

Yeah, but the read or write is then just silently dropped.

Quoted 7 lines
Quoted 5 lines
Quoted 2 lines

That can't be correct either.

These are void funcs. Moreover, the drops will happen if there is concurrent access from another thread while a reset is going on. That is expected and those accesses during a reset won't help anyway.

Nope, Teddy has been working on that for a while as well.

This silent drop is already there in bare metal.

https://elixir.bootlin.com/linux/v6.10-rc5/source/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c#L738

Quoted 8 lines

Trying to make those accesses while the reset is going on is wrong in the first place. What we need to do is to grab the reset lock in the higher level function so that the whole sequences of reads and writes are protected.

What this logic here does is to use readl()/writel() from the reset thread itself. Dropping that is incorrect and could lead to broken reset.

This doesn't change anything for a reset thread. This fixes an already broken path for sriov where it attempts a direct readl()/writel() if taking the lock fails. That is even more broken.

Thanks,
Lijo

Quoted 82 lines

So clear NAK from my side to this patch here.

Regards,
Christian.

Quoted 75 lines

Thanks,
Lijo

Quoted 71 lines

Regards,
Christian.

Quoted 68 lines

Thanks,
Lijo

Quoted 65 lines

Regards,
Christian.

Quoted 62 lines
Quoted 62 lines

            } else {
                ret = readl(((void __iomem *)adev->rmmio) + (reg
* 4));
            }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct
 amdgpu_device *adev,
                                  &rlcg_flag)) {
                 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0,
 rlcg_flag,
 GET_INST(GC, xcc_id));
             } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }

            } else {
                ret = readl(((void __iomem *)adev->rmmio) + (reg
* 4));
            }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device
 *adev,
           if ((reg * 4) < adev->rmmio_size) {

            if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) { -            amdgpu_kiq_wreg(adev, reg, v, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) { +                amdgpu_kiq_wreg(adev, reg, v, 0);
+                up_read(&adev->reset_domain->sem);
+            }
            } else {
                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
            }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -5
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct
 amdgpu_device *adev,
                                  &rlcg_flag)) {
                 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
 GET_INST(GC, xcc_id));
             } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-            up_read(&adev->reset_domain->sem);
-        } else {
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
+            } else {

                writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
            }
        } else {

#8 lore raw

Am 24.06.24 um 14:24 schrieb Lazar, Lijo:

Quoted 49 lines · Christian König
Quoted 48 lines

Am 24.06.24 um 13:57 schrieb Lazar, Lijo:

Quoted 47 lines · Christian König
Quoted 46 lines

Am 24.06.24 um 11:52 schrieb Lazar, Lijo:

Quoted 38 lines · Christian König
Quoted 28 lines

Am 24.06.24 um 08:34 schrieb Lazar, Lijo:

Quoted 27 lines · Vignesh Chander
Quoted 4 lines

correctly handle the case when trylock fails when gpu is about to be reset by dropping the request instead of using mmio

Thanks,
Lijo

Quoted 17 lines
---
     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 38
++++++++++++----------
     1 file changed, 21 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -3
index a7ce8280b17ce7..3cfd24699d691d 100644
@@ -613,10 +613,11 @@ uint32_t amdgpu_device_rreg(struct
 amdgpu_device *adev,
           if ((reg * 4) < adev->rmmio_size) {
             if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, 0);
Quoted 9 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_device_rreg(struct · line 620

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -1
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, 0);
+                up_read(&adev->reset_domain->sem);
+            }

What has actually changed here? As far as I can see that isn't functionally different to the existing code.

Quoted 3 lines

In earlier logic, if it fails to get the lock, it takes the 'else' path.
The 'else' path is not meant for sriov/vf.

Yeah, but the read or write is then just silently dropped.

That can't be correct either.

Quoted 7 lines
Quoted 4 lines
Quoted 3 lines

These are void funcs. Moreover, the drops will happen if there is concurrent access from another thread while a reset is going on. That is expected and those accesses during a reset won't help anyway.

Nope, Teddy has been working on that for a while as well.

This silent drop is already there in bare metal.

https://elixir.bootlin.com/linux/v6.10-rc5/source/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c#L738

That's not for reset, but for hotplug. In that case the silent drop is ok because the device is gone anyway.

Quoted 10 lines
Quoted 7 lines

Trying to make those accesses while the reset is going on is wrong in the first place. What we need to do is to grab the reset lock in the higher level function so that the whole sequences of reads and writes are protected.

What this logic here does is to use readl()/writel() from the reset thread itself. Dropping that is incorrect and could lead to broken reset.

This doesn't change anything for a reset thread. This fixes an already broken path for sriov where it attempts a direct readl()/writel() if taking the lock fails. That is even more broken.

No, as far as I know that's correct behavior.

The lock is there to make sure we can talk to the KIQ. When that isn't possible because of reset we should use direct register access, that's completely intentional behavior.

What we could do is to remove the down_read_trylock() from amdgpu_device_skip_hw_access() and always assert that the reset domain lock is held.

Then we should also remove the down_read_trylock() from amdgpu_device_[rw]reg() and replace it with !amdgpu_in_reset() as it used to be.

This way we will get proper lockdep warnings for all the call chains missing to grab the reset lock.

Regards,
Christian.

Quoted 83 lines

Thanks,
Lijo

Quoted 79 lines

So clear NAK from my side to this patch here.

Regards,
Christian.

Quoted 74 lines

Thanks,
Lijo

Quoted 71 lines

Regards,
Christian.

Quoted 68 lines

Thanks,
Lijo

Quoted 65 lines

Regards,
Christian.

Quoted 62 lines
Quoted 62 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
             } else {
                 ret = readl(((void __iomem *)adev->rmmio) + (reg
 * 4));

            }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
@@ -681,10 +682,11 @@ uint32_t amdgpu_device_xcc_rreg(struct
 amdgpu_device *adev,
                                  &rlcg_flag)) {
                 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0,
 rlcg_flag,
 GET_INST(GC, xcc_id));
             } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }

            } else {
                ret = readl(((void __iomem *)adev->rmmio) + (reg
* 4));
            }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
@@ -740,10 +742,11 @@ void amdgpu_device_wreg(struct amdgpu_device
 *adev,
           if ((reg * 4) < adev->rmmio_size) {
             if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, 0);
-            up_read(&adev->reset_domain->sem);
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, 0);
+                up_read(&adev->reset_domain->sem);
+            }
             } else {
                 writel(v, ((void __iomem *)adev->rmmio) + (reg *
 4));

            }

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -5
@@ -812,11 +815,12 @@ void amdgpu_device_xcc_wreg(struct
 amdgpu_device *adev,
                                  &rlcg_flag)) {
                 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
 GET_INST(GC, xcc_id));
             } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
-            amdgpu_sriov_runtime(adev) &&
-            down_read_trylock(&adev->reset_domain->sem)) {
-            amdgpu_kiq_wreg(adev, reg, v, xcc_id);
-            up_read(&adev->reset_domain->sem);
-        } else {
+            amdgpu_sriov_runtime(adev) {
+            if (down_read_trylock(&adev->reset_domain->sem)) {
+                amdgpu_kiq_wreg(adev, reg, v, xcc_id);
+                up_read(&adev->reset_domain->sem);
+            }
+            } else {
                 writel(v, ((void __iomem *)adev->rmmio) + (reg *

4));
            }
        } else {