Skip to contentSkip to conversation

drm/amdgpu:implement new TDR feature

PATCH In review Monk Liu 2 participants 11 messages Oct 24, 2017 4 messages not in the archive

Subsystems (3)
lore
Share & tools
lore

Monk Liu

author
#0 lore raw

PATCH 1/6 drm/amdgpu:implement new TDR feature

1,new imple names amdgpu_gpu_recover which gives more hint on what it does compared with gpu_reset

2,gpu_recover unify bare-metal and SR-IOV, only the reset part is implemented differently

3,gpu_recover will increase hang job karma and its context guilty if exceeds limit, which will be canceled in run_job() routine of gpu scheduler with an error "-ECANCELED" set.

4,gpu_recover knows if this reset comes with VRAM lost or not and all jobs will be canceled in run_job() in sched recovery if VRAM lost hit.

Change-Id If3c9380e38f68f73d7a1cdb3faa53e5aa33d1fc5
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h           |   4 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    | 252 ++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c       |  21 ++-
 drivers/gpu/drm/amd/scheduler/gpu_scheduler.c |   4 +-
 4 files changed, 276 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +4 -0
index 6a4178b..5646e61 100644
@@ -178,6 +178,10 @@ extern int amdgpu_cik_support;
 #define CIK_CURSOR_WIDTH 128
 #define CIK_CURSOR_HEIGHT 128
 
+/* GPU RESET flags */
+#define AMDGPU_RESET_INFO_VRAM_LOST  (1 << 0)
+#define AMDGPU_RESET_INFO_FULLRESET	 (1 << 1)
+
 struct amdgpu_device;
 struct amdgpu_ib;
 struct amdgpu_cs_parser;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +252 -0

258 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +18 -3
index a58e3c5..bd8c7cd 100644
@@ -177,6 +177,21 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
 	return fence;
 }
 
+/* Guilty for jobs with a guilty user context,
+ * or jobs whose karma exceed the limit from a kernel context,
+ * or jobs that encountered VRAM LOST issue
+ */
+static bool amdgpu_job_need_canceled(struct amdgpu_job *job) {
+	if (job->base.s_entity->guilty && atomic_read(job->base.s_entity->guilty) == 1)
+		return true;
+	else if (atomic_read(&job->base.karma) > amdgpu_job_hang_limit)
+		return true;
+	else if (job->vram_lost_counter != atomic_read(&job->adev->vram_lost_counter))
+		return true;
+
+	return false;
+}
+
 static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 {
 	struct dma_fence *fence = NULL;
@@ -194,10 +209,10 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 	BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
 	trace_amdgpu_sched_run_job(job);
-	/* skip ib schedule when vram is lost */
-	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
+
+	if (amdgpu_job_need_canceled(job)) {
+		/* skip ib schedule if looks needed, and set error */
 		dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
-		DRM_ERROR("Skip scheduling IBs!\n");
 	} else {
 		r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
 				       &fence);
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +2 -2
index 9cbeade..72ead8d 100644
@@ -525,7 +525,7 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
 					  r);
 			dma_fence_put(fence);
 		} else {
-			DRM_ERROR("Failed to run job!\n");
+			/* fake signal schedule fence here */
 			amd_sched_process_job(NULL, &s_fence->cb);
 		}
 		spin_lock(&sched->job_list_lock);
@@ -664,7 +664,7 @@ static int amd_sched_main(void *param)
 					  r);
 			dma_fence_put(fence);
 		} else {
-			DRM_ERROR("Failed to run job!\n");
+			DRM_INFO("job skipped\n");
 			amd_sched_process_job(NULL, &s_fence->cb);
 		}
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1508824627-19489-1-git-send-email-Monk.Liu-5C7GfCeVMHo@public.gmane.org

author
#2 lore raw

PATCH 2/6 drm/amdgpu:don't change ctx->reset_couner upon query

reset_counter marks the reset counter number once the context is created, shouldn't be changed due to query.

To keep U/K interface on the ctx_query and keep ctx's reset_counter logic compatible with GPU RESET feature, now use another var named "reset_counter_query" to replace the original checked & updated in amdgpu_ctx_query.

Change-Id I369e0b5bb8ea1443750ef2ac5b8a021a09fd5782
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 5646e61..002281d 100644
@@ -740,6 +740,7 @@ struct amdgpu_ctx {
 	struct amdgpu_device    *adev;
 	struct amdgpu_queue_mgr queue_mgr;
 	unsigned		reset_counter;
+	unsigned        reset_counter_query;
 	uint32_t		vram_lost_counter;
 	spinlock_t		ring_lock;
 	struct dma_fence	**fences;
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +3 -2
index 1bf4cdc..c539fb6 100644
@@ -75,6 +75,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 	}
 
 	ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
+	ctx->reset_counter_query = ctx->reset_counter;
 	ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
 	ctx->init_priority = priority;
 	ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
@@ -216,11 +217,11 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
 	/* determine if a GPU reset has occured since the last call */
 	reset_counter = atomic_read(&adev->gpu_reset_counter);
 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
-	if (ctx->reset_counter == reset_counter)
+	if (ctx->reset_counter_query == reset_counter)
 		out->state.reset_status = AMDGPU_CTX_NO_RESET;
 	else
 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
-	ctx->reset_counter = reset_counter;
+	ctx->reset_counter_query = reset_counter;
 
 	mutex_unlock(&mgr->lock);
 	return 0;
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1508824627-19489-2-git-send-email-Monk.Liu-5C7GfCeVMHo@public.gmane.org

#4 lore raw

Re: [PATCH 2/6] drm/amdgpu:don't change ctx->reset_couner upon query

Am 24.10.2017 um 07:57 schrieb Monk Liu:

Quoted 10 lines

reset_counter marks the reset counter number once the context is created, shouldn't be changed due to query.

To keep U/K interface on the ctx_query and keep ctx's reset_counter logic compatible with GPU RESET feature, now use another var named "reset_counter_query" to replace the original checked & updated in amdgpu_ctx_query.

Change-Id I369e0b5bb8ea1443750ef2ac5b8a021a09fd5782

Nice solution to the problem, patch is Reviewed-by: Christian König <[email protected]>.

Quoted 33 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h     | 1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 5 +++--
  2 files changed, 4 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 5646e61..002281d 100644
@@ -740,6 +740,7 @@ struct amdgpu_ctx {
  	struct amdgpu_device    *adev;
  	struct amdgpu_queue_mgr queue_mgr;
  	unsigned		reset_counter;
+	unsigned        reset_counter_query;
  	uint32_t		vram_lost_counter;
  	spinlock_t		ring_lock;
  	struct dma_fence	**fences;
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +1 -0
index 1bf4cdc..c539fb6 100644
@@ -75,6 +75,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
  	}
 
  	ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
+	ctx->reset_counter_query = ctx->reset_counter;
  	ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
  	ctx->init_priority = priority;
  	ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
@@ -216,11 +217,11 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
  	/* determine if a GPU reset has occured since the last call */
  	reset_counter = atomic_read(&adev->gpu_reset_counter);
  	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */

drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c static int amdgpu_ctx_query(struct amdgpu_device *adev, · line 227

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +2 -2
-	if (ctx->reset_counter == reset_counter)
+	if (ctx->reset_counter_query == reset_counter)
  		out->state.reset_status = AMDGPU_CTX_NO_RESET;
  	else
  		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
-	ctx->reset_counter = reset_counter;
+	ctx->reset_counter_query = reset_counter;
 
  	mutex_unlock(&mgr->lock);
  	return 0;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#5 lore raw

PATCH 3/6 drm/amdgpu:implement ctx query2

In reply to a message not in the archive

this query will give flag bits to indicate what happend on the given context

Change-Id I638efef736de77519388199d000ca8f62b4dc89a
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 37 +++++++++++++++++++++++++++++++++
 include/uapi/drm/amdgpu_drm.h           |  8 +++++++
 2 files changed, 45 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +37 -0
index c539fb6..d71dc16 100644
@@ -227,6 +227,40 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int amdgpu_ctx_query2(struct amdgpu_device *adev,
+	struct amdgpu_fpriv *fpriv, uint32_t id,
+	union drm_amdgpu_ctx_out *out)
+{
+	struct amdgpu_ctx *ctx;
+	struct amdgpu_ctx_mgr *mgr;
+
+	if (!fpriv)
+		return -EINVAL;
+
+	mgr = &fpriv->ctx_mgr;
+	mutex_lock(&mgr->lock);
+	ctx = idr_find(&mgr->ctx_handles, id);
+	if (!ctx) {
+		mutex_unlock(&mgr->lock);
+		return -EINVAL;
+	}
+
+	out->state.flags = 0x0;
+	out->state.hangs = 0x0;
+
+	if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
+
+	if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
+
+	if (atomic_read(&ctx->guilty))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
+
+	mutex_unlock(&mgr->lock);
+	return 0;
+}
+
 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *filp)
 {
@@ -258,6 +292,9 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 	case AMDGPU_CTX_OP_QUERY_STATE:
 		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
 		break;
+	case AMDGPU_CTX_OP_QUERY_STATE2:
+		r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
+		break;
 	default:
 		return -EINVAL;
 	}
include/uapi/drm/amdgpu_drm.h +8 -0
index 74f0b5d..f7a4cf1 100644
@@ -163,6 +163,7 @@ union drm_amdgpu_bo_list {
 #define AMDGPU_CTX_OP_ALLOC_CTX	1
 #define AMDGPU_CTX_OP_FREE_CTX	2
 #define AMDGPU_CTX_OP_QUERY_STATE	3
+#define AMDGPU_CTX_OP_QUERY_STATE2	4
 
 /* GPU reset status */
 #define AMDGPU_CTX_NO_RESET		0
@@ -173,6 +174,13 @@ union drm_amdgpu_bo_list {
 /* unknown cause */
 #define AMDGPU_CTX_UNKNOWN_RESET	3
 
+/* indicate gpu reset occured after ctx created */
+#define AMDGPU_CTX_QUERY2_FLAGS_RESET    (1<<0)
+/* indicate vram lost occured after ctx created */
+#define AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST (1<<1)
+/* indicate some job from this context once cause gpu hang */
+#define AMDGPU_CTX_QUERY2_FLAGS_GUILTY   (1<<2)
+
 /* Context priority level */
 #define AMDGPU_CTX_PRIORITY_UNSET       -2048
 #define AMDGPU_CTX_PRIORITY_VERY_LOW    -1023
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1508824627-19489-3-git-send-email-Monk.Liu-5C7GfCeVMHo@public.gmane.org

Reviewed-by
#7 lore raw

Re: [PATCH 3/6] drm/amdgpu:implement ctx query2

Am 24.10.2017 um 07:57 schrieb Monk Liu:

Quoted 5 lines

this query will give flag bits to indicate what happend on the given context

Change-Id I638efef736de77519388199d000ca8f62b4dc89a
Quoted 76 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 37 +++++++++++++++++++++++++++++++++
  include/uapi/drm/amdgpu_drm.h           |  8 +++++++
  2 files changed, 45 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +37 -0
index c539fb6..d71dc16 100644
@@ -227,6 +227,40 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
  	return 0;
  }
 
+static int amdgpu_ctx_query2(struct amdgpu_device *adev,
+	struct amdgpu_fpriv *fpriv, uint32_t id,
+	union drm_amdgpu_ctx_out *out)
+{
+	struct amdgpu_ctx *ctx;
+	struct amdgpu_ctx_mgr *mgr;
+
+	if (!fpriv)
+		return -EINVAL;
+
+	mgr = &fpriv->ctx_mgr;
+	mutex_lock(&mgr->lock);
+	ctx = idr_find(&mgr->ctx_handles, id);
+	if (!ctx) {
+		mutex_unlock(&mgr->lock);
+		return -EINVAL;
+	}
+
+	out->state.flags = 0x0;
+	out->state.hangs = 0x0;
+
+	if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
+
+	if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
+
+	if (atomic_read(&ctx->guilty))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
+
+	mutex_unlock(&mgr->lock);
+	return 0;
+}
+
  int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
  		     struct drm_file *filp)
  {
@@ -258,6 +292,9 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
  	case AMDGPU_CTX_OP_QUERY_STATE:
  		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
  		break;
+	case AMDGPU_CTX_OP_QUERY_STATE2:
+		r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
+		break;
  	default:
  		return -EINVAL;
  	}
include/uapi/drm/amdgpu_drm.h +1 -0
index 74f0b5d..f7a4cf1 100644
@@ -163,6 +163,7 @@ union drm_amdgpu_bo_list {
  #define AMDGPU_CTX_OP_ALLOC_CTX	1
  #define AMDGPU_CTX_OP_FREE_CTX	2
  #define AMDGPU_CTX_OP_QUERY_STATE	3
+#define AMDGPU_CTX_OP_QUERY_STATE2	4
 
  /* GPU reset status */
  #define AMDGPU_CTX_NO_RESET		0
@@ -173,6 +174,13 @@ union drm_amdgpu_bo_list {
  /* unknown cause */
  #define AMDGPU_CTX_UNKNOWN_RESET	3
 

include/uapi/drm/amdgpu_drm.h union drm_amdgpu_bo_list { · line 186

Quoted 10 lines
include/uapi/drm/amdgpu_drm.h +7 -0
+/* indicate gpu reset occured after ctx created */
+#define AMDGPU_CTX_QUERY2_FLAGS_RESET    (1<<0)
+/* indicate vram lost occured after ctx created */
+#define AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST (1<<1)
+/* indicate some job from this context once cause gpu hang */
+#define AMDGPU_CTX_QUERY2_FLAGS_GUILTY   (1<<2)
+
  /* Context priority level */
  #define AMDGPU_CTX_PRIORITY_UNSET       -2048
  #define AMDGPU_CTX_PRIORITY_VERY_LOW    -1023

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#8 lore raw

PATCH 4/6 drm/amdgpu:use new gpu recover routine

In reply to a message not in the archive

retire gpu_reset and sriov_gpu_reset and use new implemented amdgpu_gpu_recover

Change-Id Ib1de998eaffe7ed2bdd398a4f1b4ca62f9abf2a6
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 307 +----------------------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  |  10 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c    |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |   5 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |   1 -
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c      |   2 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      |   2 +-
 8 files changed, 19 insertions(+), 312 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -1
index 002281d..f04fd06 100644
@@ -1844,7 +1844,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i))
 
 /* Common functions */
-int amdgpu_gpu_reset(struct amdgpu_device *adev);
+int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job* job);
 bool amdgpu_need_backup(struct amdgpu_device *adev);
 void amdgpu_pci_config_reset(struct amdgpu_device *adev);
 bool amdgpu_need_post(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +9 -298

319 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +5 -5
index f43319c..e64d93f 100644
@@ -694,25 +694,25 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
 }
 
 /**
- * amdgpu_debugfs_gpu_reset - manually trigger a gpu reset
+ * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
  *
  * Manually trigger a gpu reset at the next fence wait.
  */
-static int amdgpu_debugfs_gpu_reset(struct seq_file *m, void *data)
+static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct amdgpu_device *adev = dev->dev_private;
 
-	seq_printf(m, "gpu reset\n");
-	amdgpu_gpu_reset(adev);
+	seq_printf(m, "gpu recover\n");
+	amdgpu_gpu_recover(adev, NULL);
 
 	return 0;
 }
 
 static const struct drm_info_list amdgpu_debugfs_fence_list[] = {
 	{"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
-	{"amdgpu_gpu_reset", &amdgpu_debugfs_gpu_reset, 0, NULL}
+	{"amdgpu_gpu_recover", &amdgpu_debugfs_gpu_recover, 0, NULL}
 };
 
 static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = {
drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +1 -1
index 47c5ce9..9c2f87c 100644
@@ -88,7 +88,7 @@ static void amdgpu_irq_reset_work_func(struct work_struct *work)
 						  reset_work);
 
 	if (!amdgpu_sriov_vf(adev))
-		amdgpu_gpu_reset(adev);
+		amdgpu_gpu_recover(adev, NULL);
 }
 
 /* Disable *all* interrupts */
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +1 -4
index bd8c7cd..7a601ef 100644
@@ -37,10 +37,7 @@ static void amdgpu_job_timedout(struct amd_sched_job *s_job)
 		  atomic_read(&job->ring->fence_drv.last_seq),
 		  job->ring->fence_drv.sync_seq);
 
-	if (amdgpu_sriov_vf(job->adev))
-		amdgpu_sriov_gpu_reset(job->adev, job);
-	else
-		amdgpu_gpu_reset(job->adev);
+	amdgpu_gpu_recover(job->adev, job);
 }
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -1
index b89d37f..3a661aa 100644
@@ -285,7 +285,6 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
-int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job);
 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
 int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -1
index b4906d2..f8522a0 100644
@@ -254,7 +254,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 	}
 
 	/* Trigger recovery due to world switch failure */
-	amdgpu_sriov_gpu_reset(adev, NULL);
+	amdgpu_gpu_recover(adev, NULL);
 }
 
 static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +1 -1
index c25a831..dae6d3a 100644
@@ -514,7 +514,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
 	}
 
 	/* Trigger recovery due to world switch failure */
-	amdgpu_sriov_gpu_reset(adev, NULL);
+	amdgpu_gpu_recover(adev, NULL);
 }
 
 static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
author
#9 lore raw

PATCH 5/6 drm/amdgpu:cleanup in_sriov_reset and lock_reset

In reply to a message not in the archive

since now gpu reset is unified with gpu_recover
for both bare-metal and SR-IOV:

1)rename in_sriov_reset to in_gpu_reset
2)move lock_reset from adev->virt to adev

Change-Id I9f4dbab9a4c916fbc156f669824d15ddcd0f2322
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 3 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 9 +++++----
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c    | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c  | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 2 --
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   | 1 -
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      | 6 +++---
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c      | 6 +++---
 8 files changed, 15 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -1
index f04fd06..ae9e7d8 100644
@@ -1650,7 +1650,8 @@ struct amdgpu_device {
 
 	/* record last mm index being written through WREG32*/
 	unsigned long last_mm_index;
-	bool                            in_sriov_reset;
+	bool                            in_gpu_reset;
+	struct mutex  lock_reset;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +5 -4
index 085c328..676a8bc 100644
@@ -2161,6 +2161,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	mutex_init(&adev->mn_lock);
 	mutex_init(&adev->virt.vf_errors.lock);
 	hash_init(adev->mn_hash);
+	mutex_init(&adev->lock_reset);
 
 	amdgpu_check_arguments(adev);
 
@@ -2963,9 +2964,9 @@ int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job)
 
 	dev_info(adev->dev, "GPU reset begin!\n");
 
-	mutex_lock(&adev->virt.lock_reset);
+	mutex_lock(&adev->lock_reset);
 	atomic_inc(&adev->gpu_reset_counter);
-	adev->in_sriov_reset = 1;
+	adev->in_gpu_reset = 1;
 
 	/* block TTM */
 	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
@@ -3074,8 +3075,8 @@ int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job)
 	}
 
 	amdgpu_vf_error_trans_all(adev);
-	adev->in_sriov_reset = 0;
-	mutex_unlock(&adev->virt.lock_reset);
+	adev->in_gpu_reset = 0;
+	mutex_unlock(&adev->lock_reset);
 	return r;
 }
 
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +1 -1
index 447d446..76f531b 100644
@@ -264,7 +264,7 @@ static int psp_hw_start(struct psp_context *psp)
 	struct amdgpu_device *adev = psp->adev;
 	int ret;
 
-	if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+	if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 		ret = psp_bootloader_load_sysdrv(psp);
 		if (ret)
 			return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +1 -1
index 6564902..edc37cc 100644
@@ -370,7 +370,7 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 		return 0;
 	}
 
-	if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+	if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 		err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
 					amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 					AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -2
index e97f80f..c249725 100644
@@ -107,8 +107,6 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 	adev->enable_virtual_display = true;
 	adev->cg_flags = 0;
 	adev->pg_flags = 0;
-
-	mutex_init(&adev->virt.lock_reset);
 }
 
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -1
index 3a661aa..a710384 100644
@@ -238,7 +238,6 @@ struct amdgpu_virt {
 	uint64_t			csa_vmid0_addr;
 	bool chained_ib_support;
 	uint32_t			reg_val_offs;
-	struct mutex                    lock_reset;
 	struct amdgpu_irq_src		ack_irq;
 	struct amdgpu_irq_src		rcv_irq;
 	struct work_struct		flr_work;
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +3 -3
index b8002ac..72ddb6e 100644
@@ -4807,7 +4807,7 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
 
 	gfx_v8_0_kiq_setting(ring);
 
-	if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4844,7 +4844,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 	struct vi_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-	if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
+	if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
 		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
 		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -4856,7 +4856,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
-	} else if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	} else if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +3 -3
index 8738b13..d598d78 100644
@@ -2732,7 +2732,7 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
 
 	gfx_v9_0_kiq_setting(ring);
 
-	if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
@@ -2770,7 +2770,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
 	struct v9_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-	if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
+	if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
 		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -2782,7 +2782,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
 
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
-	} else if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	} else if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
author
#10 lore raw

PATCH 6/6 drm/amdgpu:cleanup ucode_init_bo

In reply to a message not in the archive

1,no sriov check since gpu recover is unified 2,need CPU_ACCESS_REQUIRED flag for VRAM if SRIOV because otherwise after following PIN the first allocated VRAM bo is wasted due to some TTM mgr reason.

Change-Id I4d029f2da8bb463942c7861d3e52f309bdba9576
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +2 -1
index edc37cc..4939e5a 100644
@@ -370,9 +370,10 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 		return 0;
 	}
 
-	if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
+	if (!adev->in_gpu_reset) {
 		err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
 					amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+					amdgpu_sriov_vf(adev) ? AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS|AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :
 					AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 					NULL, NULL, 0, bo);
 		if (err) {
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#11 lore raw
In reply to a message not in the archive

Am 24.10.2017 um 07:57 schrieb Monk Liu:

Quoted 10 lines

1,new imple names amdgpu_gpu_recover which gives more hint on what it does compared with gpu_reset

2,gpu_recover unify bare-metal and SR-IOV, only the reset part is implemented differently

3,gpu_recover will increase hang job karma and its context guilty if exceeds limit, which will be canceled in run_job() routine of gpu scheduler with an error "-ECANCELED" set.

Quoted 5 lines

4,gpu_recover knows if this reset comes with VRAM lost or not and all jobs will be canceled in run_job() in sched recovery if VRAM lost hit.

Change-Id If3c9380e38f68f73d7a1cdb3faa53e5aa33d1fc5

Need to take a close look at this one and #4-#6, but at first glance they looked good.

I would reorder the patches and submit #2 and #3 immediately and then resend the rest. And in general please split patches into changes to amd/amdgpu and am/scheduler.

Regards,
Christian.

Quoted 339 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h           |   4 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    | 252 ++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c       |  21 ++-
  drivers/gpu/drm/amd/scheduler/gpu_scheduler.c |   4 +-
  4 files changed, 276 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +4 -0
index 6a4178b..5646e61 100644
@@ -178,6 +178,10 @@ extern int amdgpu_cik_support;
  #define CIK_CURSOR_WIDTH 128
  #define CIK_CURSOR_HEIGHT 128
 
+/* GPU RESET flags */
+#define AMDGPU_RESET_INFO_VRAM_LOST  (1 << 0)
+#define AMDGPU_RESET_INFO_FULLRESET	 (1 << 1)
+
  struct amdgpu_device;
  struct amdgpu_ib;
  struct amdgpu_cs_parser;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +252 -0
index a07544d..159d3d5 100644
@@ -3116,6 +3116,258 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
  	return r;
  }
 
+static int amdgpu_reset(struct amdgpu_device *adev, uint64_t* reset_flags)
+{
+	int r;
+	bool need_full_reset, vram_lost = 0;
+
+	need_full_reset = amdgpu_need_full_reset(adev);
+
+	if (!need_full_reset) {
+		amdgpu_pre_soft_reset(adev);
+		r = amdgpu_soft_reset(adev);
+		amdgpu_post_soft_reset(adev);
+		if (r || amdgpu_check_soft_reset(adev)) {
+			DRM_INFO("soft reset failed, will fallback to full reset!\n");
+			need_full_reset = true;
+		}
+
+	}
+
+	if (need_full_reset) {
+		r = amdgpu_suspend(adev);
+
+retry:
+		amdgpu_atombios_scratch_regs_save(adev);
+		r = amdgpu_asic_reset(adev);
+		amdgpu_atombios_scratch_regs_restore(adev);
+		/* post card */
+		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+
+		if (!r) {
+			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
+			r = amdgpu_resume_phase1(adev);
+			if (r)
+				goto out;
+
+			vram_lost = amdgpu_check_vram_lost(adev);
+			if (vram_lost) {
+				DRM_ERROR("VRAM is lost!\n");
+				atomic_inc(&adev->vram_lost_counter);
+			}
+
+			r = amdgpu_ttm_recover_gart(adev);
+			if (r)
+				goto out;
+
+			r = amdgpu_resume_phase2(adev);
+			if (r)
+				goto out;
+
+			if (vram_lost)
+				amdgpu_fill_reset_magic(adev);
+		}
+	}
+
+out:
+	if (!r) {
+		amdgpu_irq_gpu_reset_resume_helper(adev);
+		r = amdgpu_ib_ring_tests(adev);
+		if (r) {
+			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
+			r = amdgpu_suspend(adev);
+			need_full_reset = true;
+			goto retry;
+		}
+	}
+
+	if (reset_flags) {
+		if (vram_lost)
+			(*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
+
+		if (need_full_reset)
+			(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
+	}
+
+	return r;
+}
+
+static int amdgpu_reset_sriov(struct amdgpu_device *adev, uint64_t *reset_flags, bool from_hypervisor)
+{
+	int r;
+
+	if (from_hypervisor)
+		r = amdgpu_virt_request_full_gpu(adev, true);
+	else
+		r = amdgpu_virt_reset_gpu(adev);
+	if (r)
+		return r;
+
+	/* Resume IP prior to SMC */
+	r = amdgpu_sriov_reinit_early(adev);
+	if (r)
+		goto error;
+
+	/* we need recover gart prior to run SMC/CP/SDMA resume */
+	amdgpu_ttm_recover_gart(adev);
+
+	/* now we are okay to resume SMC/CP/SDMA */
+	r = amdgpu_sriov_reinit_late(adev);
+	if (r)
+		goto error;
+
+	amdgpu_irq_gpu_reset_resume_helper(adev);
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
+		dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
+
+error:
+	/* release full control of GPU after ib test */
+	amdgpu_virt_release_full_gpu(adev, true);
+
+	if (reset_flags) {
+		/* will get vram_lost from GIM in future, now all
+		 * reset request considered VRAM LOST
+		 */
+		(*reset_flags) |= ~AMDGPU_RESET_INFO_VRAM_LOST;
+		atomic_inc(&adev->vram_lost_counter);
+
+		/* VF FLR or hotlink reset is always full-reset */
+		(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
+	}
+
+	return r;
+}
+
+int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job)
+{
+	struct drm_atomic_state *state = NULL;
+	uint64_t reset_flags = 0;
+	int i, r, resched;
+
+	if (!amdgpu_check_soft_reset(adev)) {
+		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
+		return 0;
+	}
+
+	dev_info(adev->dev, "GPU reset begin!\n");
+
+	mutex_lock(&adev->virt.lock_reset);
+	atomic_inc(&adev->gpu_reset_counter);
+	adev->in_sriov_reset = 1;
+
+	/* block TTM */
+	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
+	/* store modesetting */
+	if (amdgpu_device_has_dc_support(adev))
+		state = drm_atomic_helper_suspend(adev->ddev);
+
+	/* block scheduler */
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+
+		/* only focus on the ring hit timeout if &job not NULL */
+		if (job && job->ring->idx != i)
+			continue;
+
+		kthread_park(ring->sched.thread);
+		amd_sched_hw_job_reset(&ring->sched, &job->base);
+
+		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+		amdgpu_fence_driver_force_completion(ring);
+	}
+
+	if (amdgpu_sriov_vf(adev))
+		r = amdgpu_reset_sriov(adev, &reset_flags, job ? false : true);
+	else
+		r = amdgpu_reset(adev, &reset_flags);
+
+	if (!r) {
+		if ((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) {
+			struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+			struct amdgpu_bo *bo, *tmp;
+			struct dma_fence *fence = NULL, *next = NULL;
+
+			DRM_INFO("recover vram bo from shadow\n");
+			mutex_lock(&adev->shadow_list_lock);
+			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
+				next = NULL;
+				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
+				if (fence) {
+					r = dma_fence_wait(fence, false);
+					if (r) {
+						WARN(r, "recovery from shadow isn't completed\n");
+						break;
+					}
+				}
+
+				dma_fence_put(fence);
+				fence = next;
+			}
+			mutex_unlock(&adev->shadow_list_lock);
+			if (fence) {
+				r = dma_fence_wait(fence, false);
+				if (r)
+					WARN(r, "recovery from shadow isn't completed\n");
+			}
+			dma_fence_put(fence);
+		}
+
+		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			amd_sched_job_recovery(&ring->sched);
+			kthread_unpark(ring->sched.thread);
+		}
+	} else {
+		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			kthread_unpark(adev->rings[i]->sched.thread);
+		}
+	}
+
+	if (amdgpu_device_has_dc_support(adev)) {
+		if (drm_atomic_helper_resume(adev->ddev, state))
+			dev_info(adev->dev, "drm resume failed:%d\n", r);
+		amdgpu_dm_display_resume(adev);
+	} else {
+		drm_helper_resume_force_mode(adev->ddev);
+	}
+
+	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+
+	if (r) {
+		/* bad news, how to tell it to userspace ? */
+		dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
+		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
+	} else {
+		dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
+	}
+
+	amdgpu_vf_error_trans_all(adev);
+	adev->in_sriov_reset = 0;
+	mutex_unlock(&adev->virt.lock_reset);
+	return r;
+}
+
  void amdgpu_get_pcie_info(struct amdgpu_device *adev)
  {
  	u32 mask;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +18 -3
index a58e3c5..bd8c7cd 100644
@@ -177,6 +177,21 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
  	return fence;
  }
 
+/* Guilty for jobs with a guilty user context,
+ * or jobs whose karma exceed the limit from a kernel context,
+ * or jobs that encountered VRAM LOST issue
+ */
+static bool amdgpu_job_need_canceled(struct amdgpu_job *job) {
+	if (job->base.s_entity->guilty && atomic_read(job->base.s_entity->guilty) == 1)
+		return true;
+	else if (atomic_read(&job->base.karma) > amdgpu_job_hang_limit)
+		return true;
+	else if (job->vram_lost_counter != atomic_read(&job->adev->vram_lost_counter))
+		return true;
+
+	return false;
+}
+
  static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
  {
  	struct dma_fence *fence = NULL;
@@ -194,10 +209,10 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
  	BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
  	trace_amdgpu_sched_run_job(job);
-	/* skip ib schedule when vram is lost */
-	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
+
+	if (amdgpu_job_need_canceled(job)) {
+		/* skip ib schedule if looks needed, and set error */
  		dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
-		DRM_ERROR("Skip scheduling IBs!\n");
  	} else {
  		r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
  				       &fence);
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +1 -1
index 9cbeade..72ead8d 100644
@@ -525,7 +525,7 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
  					  r);
  			dma_fence_put(fence);
  		} else {
-			DRM_ERROR("Failed to run job!\n");
+			/* fake signal schedule fence here */
  			amd_sched_process_job(NULL, &s_fence->cb);
  		}
  		spin_lock(&sched->job_list_lock);

drivers/gpu/drm/amd/scheduler/gpu_scheduler.c static int amd_sched_main(void *param) · line 670

Quoted 8 lines
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +1 -1
@@ -664,7 +664,7 @@ static int amd_sched_main(void *param)
  					  r);
  			dma_fence_put(fence);
  		} else {
-			DRM_ERROR("Failed to run job!\n");
+			DRM_INFO("job skipped\n");
  			amd_sched_process_job(NULL, &s_fence->cb);
  		}
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 2216c31a-6e2a-ebf8-c310-90b03dc15fd8-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

author
#13 lore raw

Alright

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: 2017年10月24日 16:07
To: Liu, Monk <[email protected]>; [email protected]
Subject: Re: [PATCH 1/6] drm/amdgpu:implement new TDR feature

Am 24.10.2017 um 07:57 schrieb Monk Liu:

Quoted 10 lines

1,new imple names amdgpu_gpu_recover which gives more hint on what it does compared with gpu_reset

2,gpu_recover unify bare-metal and SR-IOV, only the reset part is implemented differently

3,gpu_recover will increase hang job karma and its context guilty if exceeds limit, which will be canceled in run_job() routine of gpu scheduler with an error "-ECANCELED" set.

Quoted 5 lines

4,gpu_recover knows if this reset comes with VRAM lost or not and all jobs will be canceled in run_job() in sched recovery if VRAM lost hit.

Change-Id If3c9380e38f68f73d7a1cdb3faa53e5aa33d1fc5

Need to take a close look at this one and #4-#6, but at first glance they looked good.

I would reorder the patches and submit #2 and #3 immediately and then resend the rest. And in general please split patches into changes to amd/amdgpu and am/scheduler.

Regards,
Christian.

Quoted 342 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h           |   4 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    | 252 ++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c       |  21 ++-
  drivers/gpu/drm/amd/scheduler/gpu_scheduler.c |   4 +-
  4 files changed, 276 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +4 -0
index 6a4178b..5646e61 100644
@@ -178,6 +178,10 @@ extern int amdgpu_cik_support;
  #define CIK_CURSOR_WIDTH 128
  #define CIK_CURSOR_HEIGHT 128
 
+/* GPU RESET flags */
+#define AMDGPU_RESET_INFO_VRAM_LOST  (1 << 0)
+#define AMDGPU_RESET_INFO_FULLRESET	 (1 << 1)
+
  struct amdgpu_device;
  struct amdgpu_ib;
  struct amdgpu_cs_parser;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +252 -0
index a07544d..159d3d5 100644
@@ -3116,6 +3116,258 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
  	return r;
  }
 
+static int amdgpu_reset(struct amdgpu_device *adev, uint64_t* 
+reset_flags) {
+	int r;
+	bool need_full_reset, vram_lost = 0;
+
+	need_full_reset = amdgpu_need_full_reset(adev);
+
+	if (!need_full_reset) {
+		amdgpu_pre_soft_reset(adev);
+		r = amdgpu_soft_reset(adev);
+		amdgpu_post_soft_reset(adev);
+		if (r || amdgpu_check_soft_reset(adev)) {
+			DRM_INFO("soft reset failed, will fallback to full reset!\n");
+			need_full_reset = true;
+		}
+
+	}
+
+	if (need_full_reset) {
+		r = amdgpu_suspend(adev);
+
+retry:
+		amdgpu_atombios_scratch_regs_save(adev);
+		r = amdgpu_asic_reset(adev);
+		amdgpu_atombios_scratch_regs_restore(adev);
+		/* post card */
+		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+
+		if (!r) {
+			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
+			r = amdgpu_resume_phase1(adev);
+			if (r)
+				goto out;
+
+			vram_lost = amdgpu_check_vram_lost(adev);
+			if (vram_lost) {
+				DRM_ERROR("VRAM is lost!\n");
+				atomic_inc(&adev->vram_lost_counter);
+			}
+
+			r = amdgpu_ttm_recover_gart(adev);
+			if (r)
+				goto out;
+
+			r = amdgpu_resume_phase2(adev);
+			if (r)
+				goto out;
+
+			if (vram_lost)
+				amdgpu_fill_reset_magic(adev);
+		}
+	}
+
+out:
+	if (!r) {
+		amdgpu_irq_gpu_reset_resume_helper(adev);
+		r = amdgpu_ib_ring_tests(adev);
+		if (r) {
+			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
+			r = amdgpu_suspend(adev);
+			need_full_reset = true;
+			goto retry;
+		}
+	}
+
+	if (reset_flags) {
+		if (vram_lost)
+			(*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
+
+		if (need_full_reset)
+			(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
+	}
+
+	return r;
+}
+
+static int amdgpu_reset_sriov(struct amdgpu_device *adev, uint64_t 
+*reset_flags, bool from_hypervisor) {
+	int r;
+
+	if (from_hypervisor)
+		r = amdgpu_virt_request_full_gpu(adev, true);
+	else
+		r = amdgpu_virt_reset_gpu(adev);
+	if (r)
+		return r;
+
+	/* Resume IP prior to SMC */
+	r = amdgpu_sriov_reinit_early(adev);
+	if (r)
+		goto error;
+
+	/* we need recover gart prior to run SMC/CP/SDMA resume */
+	amdgpu_ttm_recover_gart(adev);
+
+	/* now we are okay to resume SMC/CP/SDMA */
+	r = amdgpu_sriov_reinit_late(adev);
+	if (r)
+		goto error;
+
+	amdgpu_irq_gpu_reset_resume_helper(adev);
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
+		dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
+
+error:
+	/* release full control of GPU after ib test */
+	amdgpu_virt_release_full_gpu(adev, true);
+
+	if (reset_flags) {
+		/* will get vram_lost from GIM in future, now all
+		 * reset request considered VRAM LOST
+		 */
+		(*reset_flags) |= ~AMDGPU_RESET_INFO_VRAM_LOST;
+		atomic_inc(&adev->vram_lost_counter);
+
+		/* VF FLR or hotlink reset is always full-reset */
+		(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
+	}
+
+	return r;
+}
+
+int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job 
+*job) {
+	struct drm_atomic_state *state = NULL;
+	uint64_t reset_flags = 0;
+	int i, r, resched;
+
+	if (!amdgpu_check_soft_reset(adev)) {
+		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
+		return 0;
+	}
+
+	dev_info(adev->dev, "GPU reset begin!\n");
+
+	mutex_lock(&adev->virt.lock_reset);
+	atomic_inc(&adev->gpu_reset_counter);
+	adev->in_sriov_reset = 1;
+
+	/* block TTM */
+	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
+	/* store modesetting */
+	if (amdgpu_device_has_dc_support(adev))
+		state = drm_atomic_helper_suspend(adev->ddev);
+
+	/* block scheduler */
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+
+		/* only focus on the ring hit timeout if &job not NULL */
+		if (job && job->ring->idx != i)
+			continue;
+
+		kthread_park(ring->sched.thread);
+		amd_sched_hw_job_reset(&ring->sched, &job->base);
+
+		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+		amdgpu_fence_driver_force_completion(ring);
+	}
+
+	if (amdgpu_sriov_vf(adev))
+		r = amdgpu_reset_sriov(adev, &reset_flags, job ? false : true);
+	else
+		r = amdgpu_reset(adev, &reset_flags);
+
+	if (!r) {
+		if ((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) {
+			struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+			struct amdgpu_bo *bo, *tmp;
+			struct dma_fence *fence = NULL, *next = NULL;
+
+			DRM_INFO("recover vram bo from shadow\n");
+			mutex_lock(&adev->shadow_list_lock);
+			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
+				next = NULL;
+				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
+				if (fence) {
+					r = dma_fence_wait(fence, false);
+					if (r) {
+						WARN(r, "recovery from shadow isn't completed\n");
+						break;
+					}
+				}
+
+				dma_fence_put(fence);
+				fence = next;
+			}
+			mutex_unlock(&adev->shadow_list_lock);
+			if (fence) {
+				r = dma_fence_wait(fence, false);
+				if (r)
+					WARN(r, "recovery from shadow isn't completed\n");
+			}
+			dma_fence_put(fence);
+		}
+
+		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			amd_sched_job_recovery(&ring->sched);
+			kthread_unpark(ring->sched.thread);
+		}
+	} else {
+		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			kthread_unpark(adev->rings[i]->sched.thread);
+		}
+	}
+
+	if (amdgpu_device_has_dc_support(adev)) {
+		if (drm_atomic_helper_resume(adev->ddev, state))
+			dev_info(adev->dev, "drm resume failed:%d\n", r);
+		amdgpu_dm_display_resume(adev);
+	} else {
+		drm_helper_resume_force_mode(adev->ddev);
+	}
+
+	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+
+	if (r) {
+		/* bad news, how to tell it to userspace ? */
+		dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
+		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
+	} else {
+		dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
+	}
+
+	amdgpu_vf_error_trans_all(adev);
+	adev->in_sriov_reset = 0;
+	mutex_unlock(&adev->virt.lock_reset);
+	return r;
+}
+
  void amdgpu_get_pcie_info(struct amdgpu_device *adev)
  {
  	u32 mask;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +17 -3
index a58e3c5..bd8c7cd 100644
@@ -177,6 +177,21 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
  	return fence;
  }
 
+/* Guilty for jobs with a guilty user context,
+ * or jobs whose karma exceed the limit from a kernel context,
+ * or jobs that encountered VRAM LOST issue  */ static bool 
+amdgpu_job_need_canceled(struct amdgpu_job *job) {
+	if (job->base.s_entity->guilty && atomic_read(job->base.s_entity->guilty) == 1)
+		return true;
+	else if (atomic_read(&job->base.karma) > amdgpu_job_hang_limit)
+		return true;
+	else if (job->vram_lost_counter != atomic_read(&job->adev->vram_lost_counter))
+		return true;
+
+	return false;
+}
+
  static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
  {
  	struct dma_fence *fence = NULL;
@@ -194,10 +209,10 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
  	BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
  	trace_amdgpu_sched_run_job(job);
-	/* skip ib schedule when vram is lost */
-	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
+
+	if (amdgpu_job_need_canceled(job)) {
+		/* skip ib schedule if looks needed, and set error */
  		dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
-		DRM_ERROR("Skip scheduling IBs!\n");
  	} else {
  		r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
  				       &fence);
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +1 -1
index 9cbeade..72ead8d 100644
@@ -525,7 +525,7 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
  					  r);
  			dma_fence_put(fence);
  		} else {
-			DRM_ERROR("Failed to run job!\n");
+			/* fake signal schedule fence here */
  			amd_sched_process_job(NULL, &s_fence->cb);
  		}
  		spin_lock(&sched->job_list_lock);

drivers/gpu/drm/amd/scheduler/gpu_scheduler.c static int amd_sched_main(void *param) · line 670

Quoted 8 lines
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +1 -1
@@ -664,7 +664,7 @@ static int amd_sched_main(void *param)
  					  r);
  			dma_fence_put(fence);
  		} else {
-			DRM_ERROR("Failed to run job!\n");
+			DRM_INFO("job skipped\n");
  			amd_sched_process_job(NULL, &s_fence->cb);
  		}
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#14 lore raw
In reply to a message not in the archive

The reset patches resubmitted, please review

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: 2017年10月24日 16:07
To: Liu, Monk <[email protected]>; [email protected]
Subject: Re: [PATCH 1/6] drm/amdgpu:implement new TDR feature

Am 24.10.2017 um 07:57 schrieb Monk Liu:

Quoted 10 lines

1,new imple names amdgpu_gpu_recover which gives more hint on what it does compared with gpu_reset

2,gpu_recover unify bare-metal and SR-IOV, only the reset part is implemented differently

3,gpu_recover will increase hang job karma and its context guilty if exceeds limit, which will be canceled in run_job() routine of gpu scheduler with an error "-ECANCELED" set.

Quoted 5 lines

4,gpu_recover knows if this reset comes with VRAM lost or not and all jobs will be canceled in run_job() in sched recovery if VRAM lost hit.

Change-Id If3c9380e38f68f73d7a1cdb3faa53e5aa33d1fc5

Need to take a close look at this one and #4-#6, but at first glance they looked good.

I would reorder the patches and submit #2 and #3 immediately and then resend the rest. And in general please split patches into changes to amd/amdgpu and am/scheduler.

Regards,
Christian.

Quoted 342 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h           |   4 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c    | 252 ++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c       |  21 ++-
  drivers/gpu/drm/amd/scheduler/gpu_scheduler.c |   4 +-
  4 files changed, 276 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +4 -0
index 6a4178b..5646e61 100644
@@ -178,6 +178,10 @@ extern int amdgpu_cik_support;
  #define CIK_CURSOR_WIDTH 128
  #define CIK_CURSOR_HEIGHT 128
 
+/* GPU RESET flags */
+#define AMDGPU_RESET_INFO_VRAM_LOST  (1 << 0)
+#define AMDGPU_RESET_INFO_FULLRESET	 (1 << 1)
+
  struct amdgpu_device;
  struct amdgpu_ib;
  struct amdgpu_cs_parser;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +252 -0
index a07544d..159d3d5 100644
@@ -3116,6 +3116,258 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
  	return r;
  }
 
+static int amdgpu_reset(struct amdgpu_device *adev, uint64_t* 
+reset_flags) {
+	int r;
+	bool need_full_reset, vram_lost = 0;
+
+	need_full_reset = amdgpu_need_full_reset(adev);
+
+	if (!need_full_reset) {
+		amdgpu_pre_soft_reset(adev);
+		r = amdgpu_soft_reset(adev);
+		amdgpu_post_soft_reset(adev);
+		if (r || amdgpu_check_soft_reset(adev)) {
+			DRM_INFO("soft reset failed, will fallback to full reset!\n");
+			need_full_reset = true;
+		}
+
+	}
+
+	if (need_full_reset) {
+		r = amdgpu_suspend(adev);
+
+retry:
+		amdgpu_atombios_scratch_regs_save(adev);
+		r = amdgpu_asic_reset(adev);
+		amdgpu_atombios_scratch_regs_restore(adev);
+		/* post card */
+		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+
+		if (!r) {
+			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
+			r = amdgpu_resume_phase1(adev);
+			if (r)
+				goto out;
+
+			vram_lost = amdgpu_check_vram_lost(adev);
+			if (vram_lost) {
+				DRM_ERROR("VRAM is lost!\n");
+				atomic_inc(&adev->vram_lost_counter);
+			}
+
+			r = amdgpu_ttm_recover_gart(adev);
+			if (r)
+				goto out;
+
+			r = amdgpu_resume_phase2(adev);
+			if (r)
+				goto out;
+
+			if (vram_lost)
+				amdgpu_fill_reset_magic(adev);
+		}
+	}
+
+out:
+	if (!r) {
+		amdgpu_irq_gpu_reset_resume_helper(adev);
+		r = amdgpu_ib_ring_tests(adev);
+		if (r) {
+			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
+			r = amdgpu_suspend(adev);
+			need_full_reset = true;
+			goto retry;
+		}
+	}
+
+	if (reset_flags) {
+		if (vram_lost)
+			(*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
+
+		if (need_full_reset)
+			(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
+	}
+
+	return r;
+}
+
+static int amdgpu_reset_sriov(struct amdgpu_device *adev, uint64_t 
+*reset_flags, bool from_hypervisor) {
+	int r;
+
+	if (from_hypervisor)
+		r = amdgpu_virt_request_full_gpu(adev, true);
+	else
+		r = amdgpu_virt_reset_gpu(adev);
+	if (r)
+		return r;
+
+	/* Resume IP prior to SMC */
+	r = amdgpu_sriov_reinit_early(adev);
+	if (r)
+		goto error;
+
+	/* we need recover gart prior to run SMC/CP/SDMA resume */
+	amdgpu_ttm_recover_gart(adev);
+
+	/* now we are okay to resume SMC/CP/SDMA */
+	r = amdgpu_sriov_reinit_late(adev);
+	if (r)
+		goto error;
+
+	amdgpu_irq_gpu_reset_resume_helper(adev);
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
+		dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
+
+error:
+	/* release full control of GPU after ib test */
+	amdgpu_virt_release_full_gpu(adev, true);
+
+	if (reset_flags) {
+		/* will get vram_lost from GIM in future, now all
+		 * reset request considered VRAM LOST
+		 */
+		(*reset_flags) |= ~AMDGPU_RESET_INFO_VRAM_LOST;
+		atomic_inc(&adev->vram_lost_counter);
+
+		/* VF FLR or hotlink reset is always full-reset */
+		(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
+	}
+
+	return r;
+}
+
+int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job 
+*job) {
+	struct drm_atomic_state *state = NULL;
+	uint64_t reset_flags = 0;
+	int i, r, resched;
+
+	if (!amdgpu_check_soft_reset(adev)) {
+		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
+		return 0;
+	}
+
+	dev_info(adev->dev, "GPU reset begin!\n");
+
+	mutex_lock(&adev->virt.lock_reset);
+	atomic_inc(&adev->gpu_reset_counter);
+	adev->in_sriov_reset = 1;
+
+	/* block TTM */
+	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
+	/* store modesetting */
+	if (amdgpu_device_has_dc_support(adev))
+		state = drm_atomic_helper_suspend(adev->ddev);
+
+	/* block scheduler */
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+
+		/* only focus on the ring hit timeout if &job not NULL */
+		if (job && job->ring->idx != i)
+			continue;
+
+		kthread_park(ring->sched.thread);
+		amd_sched_hw_job_reset(&ring->sched, &job->base);
+
+		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+		amdgpu_fence_driver_force_completion(ring);
+	}
+
+	if (amdgpu_sriov_vf(adev))
+		r = amdgpu_reset_sriov(adev, &reset_flags, job ? false : true);
+	else
+		r = amdgpu_reset(adev, &reset_flags);
+
+	if (!r) {
+		if ((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) {
+			struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+			struct amdgpu_bo *bo, *tmp;
+			struct dma_fence *fence = NULL, *next = NULL;
+
+			DRM_INFO("recover vram bo from shadow\n");
+			mutex_lock(&adev->shadow_list_lock);
+			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
+				next = NULL;
+				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
+				if (fence) {
+					r = dma_fence_wait(fence, false);
+					if (r) {
+						WARN(r, "recovery from shadow isn't completed\n");
+						break;
+					}
+				}
+
+				dma_fence_put(fence);
+				fence = next;
+			}
+			mutex_unlock(&adev->shadow_list_lock);
+			if (fence) {
+				r = dma_fence_wait(fence, false);
+				if (r)
+					WARN(r, "recovery from shadow isn't completed\n");
+			}
+			dma_fence_put(fence);
+		}
+
+		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			amd_sched_job_recovery(&ring->sched);
+			kthread_unpark(ring->sched.thread);
+		}
+	} else {
+		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			kthread_unpark(adev->rings[i]->sched.thread);
+		}
+	}
+
+	if (amdgpu_device_has_dc_support(adev)) {
+		if (drm_atomic_helper_resume(adev->ddev, state))
+			dev_info(adev->dev, "drm resume failed:%d\n", r);
+		amdgpu_dm_display_resume(adev);
+	} else {
+		drm_helper_resume_force_mode(adev->ddev);
+	}
+
+	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+
+	if (r) {
+		/* bad news, how to tell it to userspace ? */
+		dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
+		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
+	} else {
+		dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
+	}
+
+	amdgpu_vf_error_trans_all(adev);
+	adev->in_sriov_reset = 0;
+	mutex_unlock(&adev->virt.lock_reset);
+	return r;
+}
+
  void amdgpu_get_pcie_info(struct amdgpu_device *adev)
  {
  	u32 mask;
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +17 -3
index a58e3c5..bd8c7cd 100644
@@ -177,6 +177,21 @@ static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
  	return fence;
  }
 
+/* Guilty for jobs with a guilty user context,
+ * or jobs whose karma exceed the limit from a kernel context,
+ * or jobs that encountered VRAM LOST issue  */ static bool 
+amdgpu_job_need_canceled(struct amdgpu_job *job) {
+	if (job->base.s_entity->guilty && atomic_read(job->base.s_entity->guilty) == 1)
+		return true;
+	else if (atomic_read(&job->base.karma) > amdgpu_job_hang_limit)
+		return true;
+	else if (job->vram_lost_counter != atomic_read(&job->adev->vram_lost_counter))
+		return true;
+
+	return false;
+}
+
  static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
  {
  	struct dma_fence *fence = NULL;
@@ -194,10 +209,10 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
  	BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
  	trace_amdgpu_sched_run_job(job);
-	/* skip ib schedule when vram is lost */
-	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
+
+	if (amdgpu_job_need_canceled(job)) {
+		/* skip ib schedule if looks needed, and set error */
  		dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
-		DRM_ERROR("Skip scheduling IBs!\n");
  	} else {
  		r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
  				       &fence);
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +1 -1
index 9cbeade..72ead8d 100644
@@ -525,7 +525,7 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
  					  r);
  			dma_fence_put(fence);
  		} else {
-			DRM_ERROR("Failed to run job!\n");
+			/* fake signal schedule fence here */
  			amd_sched_process_job(NULL, &s_fence->cb);
  		}
  		spin_lock(&sched->job_list_lock);

drivers/gpu/drm/amd/scheduler/gpu_scheduler.c static int amd_sched_main(void *param) · line 670

Quoted 8 lines
drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +1 -1
@@ -664,7 +664,7 @@ static int amd_sched_main(void *param)
  					  r);
  			dma_fence_put(fence);
  		} else {
-			DRM_ERROR("Failed to run job!\n");
+			DRM_INFO("job skipped\n");
  			amd_sched_process_job(NULL, &s_fence->cb);
  		}
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx