Skip to contentSkip to conversation

drm/amdgpu/gfx10: disallow gfxoff around GPU reset

PATCH In review Alex Deucher 3 participants 9 messages Jul 6 – Jul 7

Subsystems (2)
lore
Share & tools
lore

Alex Deucher

author maintainer
#0 lore raw

PATCH 1/7 drm/amdgpu/gfx10: disallow gfxoff around GPU reset

If gfx is hung, we shouldn't need this since gfx won't be powered down, but we could potentially have a case where a queue can be preempted, but due logic errors in the commands, the fence never signals. GFX could potentially go into the off state. Protect against that.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 34 +++++++++++++++++---------
 1 file changed, 23 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +23 -11
index f03cbf358cd7b..516a621d11faa 100644
@@ -9528,6 +9528,7 @@ static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring,
 		return -ENOMEM;
 	}
 
+	amdgpu_gfx_off_ctrl(adev, false);
 	addr = amdgpu_bo_gpu_offset(ring->mqd_obj) +
 		offsetof(struct v10_gfx_mqd, cp_gfx_hqd_active);
 	tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
@@ -9547,28 +9548,33 @@ static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring,
 	r = amdgpu_ring_test_ring(kiq_ring);
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 	if (r)
-		return r;
+		goto fail;
 
 	r = gfx_v10_0_kgq_init_queue(ring, true);
 	if (r) {
 		DRM_ERROR("fail to init kgq\n");
-		return r;
+		goto fail;
 	}
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
 
 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size)) {
 		spin_unlock_irqrestore(&kiq->ring_lock, flags);
-		return -ENOMEM;
+		r = -ENOMEM;
+		goto fail;
 	}
 	kiq->pmf->kiq_map_queues(kiq_ring, ring);
 	amdgpu_ring_commit(kiq_ring);
 	r = amdgpu_ring_test_ring(kiq_ring);
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 	if (r)
-		return r;
+		goto fail;
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+
+fail:
+	amdgpu_gfx_off_ctrl(adev, true);
+	return r;
 }
 
 static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
@@ -9593,13 +9599,14 @@ static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
 		return -ENOMEM;
 	}
 
+	amdgpu_gfx_off_ctrl(adev, false);
 	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
 				   0, 0);
 	amdgpu_ring_commit(kiq_ring);
 	r = amdgpu_ring_test_ring(kiq_ring);
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 	if (r)
-		return r;
+		goto fail;
 
 	/* make sure dequeue is complete*/
 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
@@ -9617,28 +9624,33 @@ static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 	if (r) {
 		dev_err(adev->dev, "fail to wait on hqd deactivate\n");
-		return r;
+		goto fail;
 	}
 
 	r = gfx_v10_0_kcq_init_queue(ring, true);
 	if (r) {
 		dev_err(adev->dev, "fail to init kcq\n");
-		return r;
+		goto fail;
 	}
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size)) {
 		spin_unlock_irqrestore(&kiq->ring_lock, flags);
-		return -ENOMEM;
+		r = -ENOMEM;
+		goto fail;
 	}
 	kiq->pmf->kiq_map_queues(kiq_ring, ring);
 	amdgpu_ring_commit(kiq_ring);
 	r = amdgpu_ring_test_ring(kiq_ring);
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 	if (r)
-		return r;
+		goto fail;
+
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+fail:
+	amdgpu_gfx_off_ctrl(adev, true);
+	return r;
 }
 
 static void gfx_v10_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
Signature · 1 line
2.54.0
author maintainer
#1 lore raw

PATCH 2/7 drm/amdgpu/gfx: disallow gfxoff around compute reset

If gfx is hung, we shouldn't need this since gfx won't be powered down, but we could potentially have a case where a queue can be preempted, but due logic errors in the commands, the fence never signals. GFX could potentially go into the off state. Protect against that.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 5 +++++
 1 file changed, 5 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +5 -0
index 1e275c2e7dd3e..2552561f997a8 100644
@@ -2285,6 +2285,8 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	int ring_err;
 
 	guard(mutex)(&adev->gfx.mec.reset_mutex);
+	if (use_mmio)
+		amdgpu_gfx_off_ctrl(adev, false);
 	/* stop the drm schedulers for all compute queues */
 	amdgpu_gfx_reset_stop_compute_scheds(adev, ring);
 	/* suspend all will determine which queues are hung.
@@ -2388,6 +2390,9 @@ int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
 	if (!ring_err)
 		amdgpu_gfx_reset_start_compute_scheds(adev, ring);
 
+	if (use_mmio)
+		amdgpu_gfx_off_ctrl(adev, true);
+
 	/* If this reset is triggered by non-KCQ, the KCQ result after resume must
 	 * not override the reset result; otherwise a false reset failure is returned
 	 * to the non-KCQ caller
Signature · 1 line
2.54.0
author maintainer
#2 lore raw

PATCH 3/7 drm/amdgpu/gfx11: disallow gfxoff around GPU reset

In reply to Alex Deucher

If gfx is hung, we shouldn't need this since gfx won't be powered down, but we could potentially have a case where a queue can be preempted, but due logic errors in the commands, the fence never signals. GFX could potentially go into the off state. Protect against that.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +7 -1
index b2a3ceda99bc9..162f825cba84f 100644
@@ -6862,8 +6862,14 @@ static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,
 {
 	struct amdgpu_device *adev = ring->adev;
 	bool use_mmio = adev->gfx.me.use_mmio_for_reset;
+	int r;
 
-	return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
+	if (use_mmio)
+		amdgpu_gfx_off_ctrl(adev, false);
+	r = amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
+	if (use_mmio)
+		amdgpu_gfx_off_ctrl(adev, true);
+	return r;
 }
 
 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
Signature · 1 line
2.54.0
author maintainer
#3 lore raw

PATCH 4/7 drm/amdgpu/gfx12: disallow gfxoff around GPU reset

In reply to Alex Deucher

If gfx is hung, we shouldn't need this since gfx won't be powered down, but we could potentially have a case where a queue can be preempted, but due logic errors in the commands, the fence never signals. GFX could potentially go into the off state. Protect against that.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +7 -1
index a35cac55c859d..9055b56e78f4c 100644
@@ -5292,8 +5292,14 @@ static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring,
 {
 	struct amdgpu_device *adev = ring->adev;
 	bool use_mmio = adev->gfx.me.use_mmio_for_reset;
+	int r;
 
-	return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
+	if (use_mmio)
+		amdgpu_gfx_off_ctrl(adev, false);
+	r = amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
+	if (use_mmio)
+		amdgpu_gfx_off_ctrl(adev, true);
+	return r;
 }
 
 static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
Signature · 1 line
2.54.0
author maintainer
#4 lore raw

PATCH 5/7 drm/amdgpu/sdma6: don't do MMIO in MQD init

In reply to Alex Deucher

There's no need to read the register, just use the default value.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +1 -2
index cf3d2997fff88..51e7c825b8bfc 100644
@@ -860,8 +860,7 @@ static int sdma_v6_0_mqd_init(struct amdgpu_device *adev, void *mqd,
 	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 
-	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, 0,
-							regSDMA0_QUEUE0_IB_CNTL));
+	m->sdmax_rlcx_ib_cntl = regSDMA0_QUEUE0_IB_CNTL_DEFAULT;
 
 	m->sdmax_rlcx_doorbell_offset =
 		prop->doorbell_index << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
Signature · 1 line
2.54.0
author maintainer
#5 lore raw

PATCH 6/7 drm/amdgpu/sdma7: don't do MMIO in MQD init

In reply to Alex Deucher

There's no need to read the register, just use the default value.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +1 -2
index 69cb89298a3e0..b3dbcb39d97dd 100644
@@ -878,8 +878,7 @@ static int sdma_v7_0_mqd_init(struct amdgpu_device *adev, void *mqd,
 	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 
-	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, 0,
-							regSDMA0_QUEUE0_IB_CNTL));
+	m->sdmax_rlcx_ib_cntl = 0x00000100; //regSDMA0_QUEUE0_IB_CNTL_DEFAULT
 
 	m->sdmax_rlcx_doorbell_offset =
 		prop->doorbell_index << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
Signature · 1 line
2.54.0
author maintainer
#6 lore raw

PATCH 7/7 drm/amdgpu/sdma7.1: don't do MMIO in MQD init

In reply to Alex Deucher

There's no need to read the register, just use the default value.

Assisted-by Claude:claude-opus-4.6
---
 drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c +1 -2
index 18366e16ef3fa..b71b8b0275ccf 100644
@@ -871,8 +871,7 @@ static int sdma_v7_1_mqd_init(struct amdgpu_device *adev, void *mqd,
 	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 
-	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, 0,
-							regSDMA0_SDMA_QUEUE0_IB_CNTL));
+	m->sdmax_rlcx_ib_cntl = 0x00000100; //regSDMA0_QUEUE0_IB_CNTL_DEFAULT
 
 	m->sdmax_rlcx_doorbell_offset =
 		prop->doorbell_index << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
Signature · 1 line
2.54.0
#7 lore raw
In reply to Alex Deucher
Quoted 17 lines · Alex Deucher

If gfx is hung, we shouldn't need this since gfx won't be powered down, but we could potentially have a case where a queue can be preempted, but due logic errors in the commands, the fence never signals. GFX could potentially go into the off state. Protect against that.

Assisted-by Claude:claude-opus-4.6
---
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 34 +++++++++++++++++---------
  1 file changed, 23 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
index f03cbf358cd7b..516a621d11faa 100644

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring, · line 9531

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -0
@@ -9528,6 +9528,7 @@ static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring,
  		return -ENOMEM;
  	}
 
+	amdgpu_gfx_off_ctrl(adev, false);

Can this be moved to helper begin/end logic - check against sdma/compute/gfx rings? It doesn't harm to include sdma even for SOCs where it's outside of GFX domain.

Thanks,
Lijo

Quoted 97 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +22 -11
  	addr = amdgpu_bo_gpu_offset(ring->mqd_obj) +
  		offsetof(struct v10_gfx_mqd, cp_gfx_hqd_active);
  	tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
@@ -9547,28 +9548,33 @@ static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring,
  	r = amdgpu_ring_test_ring(kiq_ring);
  	spin_unlock_irqrestore(&kiq->ring_lock, flags);
  	if (r)
-		return r;
+		goto fail;
 
  	r = gfx_v10_0_kgq_init_queue(ring, true);
  	if (r) {
  		DRM_ERROR("fail to init kgq\n");
-		return r;
+		goto fail;
  	}
 
  	spin_lock_irqsave(&kiq->ring_lock, flags);
 
  	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size)) {
  		spin_unlock_irqrestore(&kiq->ring_lock, flags);
-		return -ENOMEM;
+		r = -ENOMEM;
+		goto fail;
  	}
  	kiq->pmf->kiq_map_queues(kiq_ring, ring);
  	amdgpu_ring_commit(kiq_ring);
  	r = amdgpu_ring_test_ring(kiq_ring);
  	spin_unlock_irqrestore(&kiq->ring_lock, flags);
  	if (r)
-		return r;
+		goto fail;
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+
+fail:
+	amdgpu_gfx_off_ctrl(adev, true);
+	return r;
  }
 
  static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
@@ -9593,13 +9599,14 @@ static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
  		return -ENOMEM;
  	}
 
+	amdgpu_gfx_off_ctrl(adev, false);
  	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
  				   0, 0);
  	amdgpu_ring_commit(kiq_ring);
  	r = amdgpu_ring_test_ring(kiq_ring);
  	spin_unlock_irqrestore(&kiq->ring_lock, flags);
  	if (r)
-		return r;
+		goto fail;
 
  	/* make sure dequeue is complete*/
  	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
@@ -9617,28 +9624,33 @@ static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
  	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
  	if (r) {
  		dev_err(adev->dev, "fail to wait on hqd deactivate\n");
-		return r;
+		goto fail;
  	}
 
  	r = gfx_v10_0_kcq_init_queue(ring, true);
  	if (r) {
  		dev_err(adev->dev, "fail to init kcq\n");
-		return r;
+		goto fail;
  	}
 
  	spin_lock_irqsave(&kiq->ring_lock, flags);
  	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size)) {
  		spin_unlock_irqrestore(&kiq->ring_lock, flags);
-		return -ENOMEM;
+		r = -ENOMEM;
+		goto fail;
  	}
  	kiq->pmf->kiq_map_queues(kiq_ring, ring);
  	amdgpu_ring_commit(kiq_ring);
  	r = amdgpu_ring_test_ring(kiq_ring);
  	spin_unlock_irqrestore(&kiq->ring_lock, flags);
  	if (r)
-		return r;
+		goto fail;
+
+	r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+fail:
+	amdgpu_gfx_off_ctrl(adev, true);
+	return r;
  }
 
  static void gfx_v10_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
#8 lore raw
Quoted 24 lines · Lijo
Quoted 20 lines · Alex Deucher

If gfx is hung, we shouldn't need this since gfx won't be powered down, but we could potentially have a case where a queue can be preempted, but due logic errors in the commands, the fence never signals. GFX could potentially go into the off state. Protect against that.

Assisted-by Claude:claude-opus-4.6
---
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 34 +++++++++++++++++---------
  1 file changed, 23 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
index f03cbf358cd7b..516a621d11faa 100644
@@ -9528,6 +9528,7 @@ static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring,
              return -ENOMEM;
      }
 
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring, · line 9531

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +1 -0
+     amdgpu_gfx_off_ctrl(adev, false);

Can this be moved to helper begin/end logic - check against sdma/compute/gfx rings? It doesn't harm to include sdma even for SOCs where it's outside of GFX domain.

If something in the reset path fails, the end helper doesn't get run so the ref count will be off. It could potentially be moved to the job_timedout() and kfd queue reset paths however.

Alex

Quoted 103 lines

Thanks,
Lijo

Quoted 97 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +22 -11
      addr = amdgpu_bo_gpu_offset(ring->mqd_obj) +
              offsetof(struct v10_gfx_mqd, cp_gfx_hqd_active);
      tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
@@ -9547,28 +9548,33 @@ static int gfx_v10_0_reset_kgq(struct amdgpu_ring *ring,
      r = amdgpu_ring_test_ring(kiq_ring);
      spin_unlock_irqrestore(&kiq->ring_lock, flags);
      if (r)
-             return r;
+             goto fail;
 
      r = gfx_v10_0_kgq_init_queue(ring, true);
      if (r) {
              DRM_ERROR("fail to init kgq\n");
-             return r;
+             goto fail;
      }
 
      spin_lock_irqsave(&kiq->ring_lock, flags);
 
      if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size)) {
              spin_unlock_irqrestore(&kiq->ring_lock, flags);
-             return -ENOMEM;
+             r = -ENOMEM;
+             goto fail;
      }
      kiq->pmf->kiq_map_queues(kiq_ring, ring);
      amdgpu_ring_commit(kiq_ring);
      r = amdgpu_ring_test_ring(kiq_ring);
      spin_unlock_irqrestore(&kiq->ring_lock, flags);
      if (r)
-             return r;
+             goto fail;
 
-     return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+     r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
+
+fail:
+     amdgpu_gfx_off_ctrl(adev, true);
+     return r;
  }
 
  static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
@@ -9593,13 +9599,14 @@ static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
              return -ENOMEM;
      }
 
+     amdgpu_gfx_off_ctrl(adev, false);
      kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
                                 0, 0);
      amdgpu_ring_commit(kiq_ring);
      r = amdgpu_ring_test_ring(kiq_ring);
      spin_unlock_irqrestore(&kiq->ring_lock, flags);
      if (r)
-             return r;
+             goto fail;
 
      /* make sure dequeue is complete*/
      amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
@@ -9617,28 +9624,33 @@ static int gfx_v10_0_reset_kcq(struct amdgpu_ring *ring,
      amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
      if (r) {
              dev_err(adev->dev, "fail to wait on hqd deactivate\n");
-             return r;
+             goto fail;
      }
 
      r = gfx_v10_0_kcq_init_queue(ring, true);
      if (r) {
              dev_err(adev->dev, "fail to init kcq\n");
-             return r;
+             goto fail;
      }
 
      spin_lock_irqsave(&kiq->ring_lock, flags);
      if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size)) {
              spin_unlock_irqrestore(&kiq->ring_lock, flags);
-             return -ENOMEM;
+             r = -ENOMEM;
+             goto fail;
      }
      kiq->pmf->kiq_map_queues(kiq_ring, ring);
      amdgpu_ring_commit(kiq_ring);
      r = amdgpu_ring_test_ring(kiq_ring);
      spin_unlock_irqrestore(&kiq->ring_lock, flags);
      if (r)
-             return r;
+             goto fail;
+
+     r = amdgpu_ring_reset_helper_end(ring, timedout_fence);
 
-     return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+fail:
+     amdgpu_gfx_off_ctrl(adev, true);
+     return r;
  }
 
  static void gfx_v10_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)