Skip to contentSkip to conversation

drm/amdgpu: Fix repeatly flr issue

PATCH In review Emily.Deng 2 participants 5 messages Aug 12, 2020 – Aug 18, 2020

Subsystems (2)
lore
Share & tools
lore

Emily.Deng

author
#0 lore raw

PATCH 1/1 drm/amdgpu: Fix repeatly flr issue

From: jqdeng <[email protected]>

Only for no job running test case need to do recover in flr notification.
For having job in mirror list, then let guest driver to hit job timeout, and then do recover.

Change-Id Ic6234fce46fa1655ba81c4149235eeac75e75868
---
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 20 +++++++++++++++++++-
 drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 22 ++++++++++++++++++++--
 2 files changed, 39 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +19 -1
index fe31cbeccfe9..12fe5164aaf3 100644
@@ -238,6 +238,9 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 	struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
 	struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
 	int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT;
+	int i;
+	bool need_do_recover = true;
+	struct drm_sched_job *job;
 
 	/* block amdgpu_gpu_recover till msg FLR COMPLETE received,
 	 * otherwise the mailbox msg will be ruined/reseted by
@@ -258,10 +261,25 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 
 flr_done:
 	up_read(&adev->reset_sem);
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+
+		spin_lock(&ring->sched.job_list_lock);
+		job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+		struct drm_sched_job, node);
+		spin_unlock(&ring->sched.job_list_lock);
+		if (job) {
+			need_do_recover = false;
+			break;
+		}
+	}
 
 	/* Trigger recovery for world switch failure if no TDR */
 	if (amdgpu_device_should_recover_gpu(adev)
-		&& adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)
+		&& (need_do_recover || adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT))
 		amdgpu_device_gpu_recover(adev, NULL);
 }
 
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +20 -2
index 6f55172e8337..fc92c494df0b 100644
@@ -259,6 +259,9 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
 	struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
 	struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
 	int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT;
+	int i;
+	bool need_do_recover = true;
+	struct drm_sched_job *job;
 
 	/* block amdgpu_gpu_recover till msg FLR COMPLETE received,
 	 * otherwise the mailbox msg will be ruined/reseted by
@@ -279,10 +282,25 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
 
 flr_done:
 	up_read(&adev->reset_sem);
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+
+		spin_lock(&ring->sched.job_list_lock);
+		job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+					struct drm_sched_job, node);
+		spin_unlock(&ring->sched.job_list_lock);
+		if (job) {
+			need_do_recover = false;
+			break;
+		}
+	}
 
 	/* Trigger recovery for world switch failure if no TDR */
-	if (amdgpu_device_should_recover_gpu(adev)
-		&& (adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
+	if (amdgpu_device_should_recover_gpu(adev) && (need_do_recover ||
+		adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
 		adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
 		adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
 		adev->video_timeout == MAX_SCHEDULE_TIMEOUT))
Signature · 6 lines
2.25.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#1 lore raw
Quoted 19 lines · Emily.Deng

From: jqdeng <[email protected]>

Only for no job running test case need to do recover in flr notification.
For having job in mirror list, then let guest driver to hit job timeout, and then do recover.

Change-Id Ic6234fce46fa1655ba81c4149235eeac75e75868
---
  drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 20 +++++++++++++++++++-
  drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 22 ++++++++++++++++++++--
  2 files changed, 39 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
index fe31cbeccfe9..12fe5164aaf3 100644

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 242

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -0
@@ -238,6 +238,9 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
  	struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
  	struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
  	int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT;
+	int i;
+	bool need_do_recover = true;

We should find a better name for "need_do_recover", may be "need_to_recover" ?

Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +6 -0
+	struct drm_sched_job *job;
 
  	/* block amdgpu_gpu_recover till msg FLR COMPLETE received,
  	 * otherwise the mailbox msg will be ruined/reseted by
@@ -258,10 +261,25 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 
  flr_done:
  	up_read(&adev->reset_sem);
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 278

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +10 -0
+
+		spin_lock(&ring->sched.job_list_lock);
+		job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+		struct drm_sched_job, node);
+		spin_unlock(&ring->sched.job_list_lock);
+		if (job) {
+			need_do_recover = false;
+			break;
+		}
+	}

This 1st job retrieval logic can move to a function as there are two instance of it.

Quoted 40 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -1
  	/* Trigger recovery for world switch failure if no TDR */
  	if (amdgpu_device_should_recover_gpu(adev)
-		&& adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)
+		&& (need_do_recover || adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT))
  		amdgpu_device_gpu_recover(adev, NULL);
  }
 
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +17 -0
index 6f55172e8337..fc92c494df0b 100644
@@ -259,6 +259,9 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
  	struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
  	struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
  	int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT;
+	int i;
+	bool need_do_recover = true;
+	struct drm_sched_job *job;
 
  	/* block amdgpu_gpu_recover till msg FLR COMPLETE received,
  	 * otherwise the mailbox msg will be ruined/reseted by
@@ -279,10 +282,25 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
 
  flr_done:
  	up_read(&adev->reset_sem);
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+
+		spin_lock(&ring->sched.job_list_lock);
+		job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+					struct drm_sched_job, node);
+		spin_unlock(&ring->sched.job_list_lock);
+		if (job) {
+			need_do_recover = false;
+			break;
+		}

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct work_struct *work) · line 306

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +3 -2
+	}
 
  	/* Trigger recovery for world switch failure if no TDR */
-	if (amdgpu_device_should_recover_gpu(adev)
-		&& (adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
+	if (amdgpu_device_should_recover_gpu(adev) && (need_do_recover ||
+		adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
  		adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
  		adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
  		adev->video_timeout == MAX_SCHEDULE_TIMEOUT))

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#2 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 33 lines

-----Original Message-----
From: Das, Nirmoy <[email protected]>
Sent: Wednesday, August 12, 2020 8:18 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix repeatly flr issue

Quoted 7 lines · Emily.Deng

From: jqdeng <[email protected]>

Only for no job running test case need to do recover in flr notification.
For having job in mirror list, then let guest driver to hit job timeout, and then do recover.

Quoted 5 lines
Change-Id Ic6234fce46fa1655ba81c4149235eeac75e75868
---
  drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 20 +++++++++++++++++++-
  drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 22 ++++++++++++++++++++-

-

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct

Quoted 8 lines
  2 files changed, 39 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
index fe31cbeccfe9..12fe5164aaf3 100644
@@ -238,6 +238,9 @@ static void xgpu_ai_mailbox_flr_work(struct

work_struct *work)

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct · line 238

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
  struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt,

flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct · line 239

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
  struct amdgpu_device *adev = container_of(virt, struct
Quoted 8 lines

amdgpu_device, virt);

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 242

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -0
  int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT;
+int i;
+bool need_do_recover = true;

We should find a better name for "need_do_recover", may be "need_to_recover" ?

Thanks, will modify later.

Quoted 80 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -0
+struct drm_sched_job *job;
 
  /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
   * otherwise the mailbox msg will be ruined/reseted by
@@ -258,10 +261,25 @@ static void xgpu_ai_mailbox_flr_work(struct

work_struct *work)

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +5 -0
  flr_done:
  up_read(&adev->reset_sem);
+for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+struct amdgpu_ring *ring = adev->rings[i];
+
+if (!ring || !ring->sched.thread)
+continue;

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 278

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +10 -0
+
+spin_lock(&ring->sched.job_list_lock);
+job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+struct drm_sched_job, node);
+spin_unlock(&ring->sched.job_list_lock);
+if (job) {
+need_do_recover = false;
+break;
+}
+}

This 1st job retrieval logic can move to a function as there are two instance of it.
Sorry, I didn't get your point.

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work)

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -1
  /* Trigger recovery for world switch failure if no TDR */
  if (amdgpu_device_should_recover_gpu(adev)
-&& adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)
+&& (need_do_recover || adev->sdma_timeout ==

MAX_SCHEDULE_TIMEOUT))

Review

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
  amdgpu_device_gpu_recover(adev, NULL);
  }
 
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct

Quoted 4 lines

index 6f55172e8337..fc92c494df0b 100644

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
@@ -259,6 +259,9 @@ static void xgpu_nv_mailbox_flr_work(struct

work_struct *work)

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct · line 259

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
  struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt,

flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct · line 260

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
  struct amdgpu_device *adev = container_of(virt, struct

amdgpu_device, virt);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +3 -0
  int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT;
+int i;
+bool need_do_recover = true;
+struct drm_sched_job *job;
 
  /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
   * otherwise the mailbox msg will be ruined/reseted by
@@ -279,10 +282,25 @@ static void xgpu_nv_mailbox_flr_work(struct

work_struct *work)

Quoted 19 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +15 -0
  flr_done:
  up_read(&adev->reset_sem);
+for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+struct amdgpu_ring *ring = adev->rings[i];
+
+if (!ring || !ring->sched.thread)
+continue;
+
+spin_lock(&ring->sched.job_list_lock);
+job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+struct drm_sched_job, node);
+spin_unlock(&ring->sched.job_list_lock);
+if (job) {
+need_do_recover = false;
+break;
+}
+}
 
Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct work_struct *work)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1 -2
  /* Trigger recovery for world switch failure if no TDR */
-if (amdgpu_device_should_recover_gpu(adev)
-&& (adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
+if (amdgpu_device_should_recover_gpu(adev) && (need_do_recover
||
> +adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
>   adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
>   adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
>   adev->video_timeout == MAX_SCHEDULE_TIMEOUT))

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#3 lore raw
Quoted 35 lines · Emily

[AMD Official Use Only - Internal Distribution Only]

Quoted 33 lines

-----Original Message-----
From: Das, Nirmoy <[email protected]>
Sent: Wednesday, August 12, 2020 8:18 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix repeatly flr issue

Quoted 7 lines · Emily.Deng

From: jqdeng <[email protected]>

Only for no job running test case need to do recover in flr notification.
For having job in mirror list, then let guest driver to hit job timeout, and then do recover.

Quoted 5 lines
Change-Id Ic6234fce46fa1655ba81c4149235eeac75e75868
---
   drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 20 +++++++++++++++++++-
   drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 22 ++++++++++++++++++++-

-

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct

Quoted 8 lines
   2 files changed, 39 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
index fe31cbeccfe9..12fe5164aaf3 100644
@@ -238,6 +238,9 @@ static void xgpu_ai_mailbox_flr_work(struct

work_struct *work)

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct · line 238

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
   struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt,

flr_work);

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
   struct amdgpu_device *adev = container_of(virt, struct
Quoted 36 lines
Quoted 7 lines

amdgpu_device, virt);

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 242

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -0
   int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT;
+int i;
+bool need_do_recover = true;

We should find a better name for "need_do_recover", may be "need_to_recover" ?

Thanks, will modify later.

Quoted 28 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -0
+struct drm_sched_job *job;
 
   /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
    * otherwise the mailbox msg will be ruined/reseted by
@@ -258,10 +261,25 @@ static void xgpu_ai_mailbox_flr_work(struct

work_struct *work)

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +5 -0
   flr_done:
   up_read(&adev->reset_sem);
+for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+struct amdgpu_ring *ring = adev->rings[i];
+
+if (!ring || !ring->sched.thread)
+continue;

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 278

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +10 -0
+
+spin_lock(&ring->sched.job_list_lock);
+job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+struct drm_sched_job, node);
+spin_unlock(&ring->sched.job_list_lock);
+if (job) {
+need_do_recover = false;
+break;
+}
+}

This 1st job retrieval logic can move to a function as there are two instance of it.
Sorry, I didn't get your point.

xgpu_ai_mailbox_flr_work() and xgpu_nv_mailbox_flr_work() are using same logic under "flr_done:" label trying to retrieve 1st job entry to determine if we should do recover or not.

We could move that logic into a function like:

bool function_name ()

{
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
		struct amdgpu_ring *ring = adev->rings[i];

		if (!ring || !ring->sched.thread)
			continue;

		spin_lock(&ring->sched.job_list_lock);
		job = list_first_entry_or_null(&ring->sched.ring_mirror_list, struct drm_sched_job, node);
		spin_unlock(&ring->sched.job_list_lock);
		if (job)
			return true;
			
	}

	return false;
}

and use that in xgpu_ai_mailbox_flr_work() and xgpu_nv_mailbox_flr_work() instead of

having two copy of that logic.

Nirmoy

Quoted 52 lines
Quoted 52 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work)

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +1 -1
   /* Trigger recovery for world switch failure if no TDR */
   if (amdgpu_device_should_recover_gpu(adev)
-&& adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)
+&& (need_do_recover || adev->sdma_timeout ==

MAX_SCHEDULE_TIMEOUT))

Review

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
   amdgpu_device_gpu_recover(adev, NULL);
   }
 
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct

Quoted 4 lines

index 6f55172e8337..fc92c494df0b 100644

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
@@ -259,6 +259,9 @@ static void xgpu_nv_mailbox_flr_work(struct

work_struct *work)

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct · line 259

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
   struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt,

flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct · line 260

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
   struct amdgpu_device *adev = container_of(virt, struct

amdgpu_device, virt);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +3 -0
   int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT;
+int i;
+bool need_do_recover = true;
+struct drm_sched_job *job;
 
   /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
    * otherwise the mailbox msg will be ruined/reseted by
@@ -279,10 +282,25 @@ static void xgpu_nv_mailbox_flr_work(struct

work_struct *work)

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +10 -0
   flr_done:
   up_read(&adev->reset_sem);
+for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+struct amdgpu_ring *ring = adev->rings[i];
+
+if (!ring || !ring->sched.thread)
+continue;
+
+spin_lock(&ring->sched.job_list_lock);
+job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+struct drm_sched_job, node);
+spin_unlock(&ring->sched.job_list_lock);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct work_struct *work)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +6 -2
+if (job) {
+need_do_recover = false;
+break;
+}
+}
 
   /* Trigger recovery for world switch failure if no TDR */
-if (amdgpu_device_should_recover_gpu(adev)
-&& (adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
+if (amdgpu_device_should_recover_gpu(adev) && (need_do_recover

||

Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct work_struct *work) · line 306

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +1 -0
+adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
   adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
   adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
   adev->video_timeout == MAX_SCHEDULE_TIMEOUT))

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#4 lore raw

[AMD Official Use Only - Internal Distribution Only]

Quoted 108 lines

-----Original Message-----
From: Das, Nirmoy <[email protected]>
Sent: Tuesday, August 18, 2020 4:22 PM
To: Deng, Emily <[email protected]>; Das, Nirmoy <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix repeatly flr issue

Quoted 21 lines · Emily

[AMD Official Use Only - Internal Distribution Only]

Quoted 19 lines

-----Original Message-----
From: Das, Nirmoy <[email protected]>
Sent: Wednesday, August 12, 2020 8:18 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix repeatly flr issue

Quoted 11 lines · Emily.Deng

From: jqdeng <[email protected]>

Only for no job running test case need to do recover in flr notification.
For having job in mirror list, then let guest driver to hit job timeout, and then do recover.

Change-Id Ic6234fce46fa1655ba81c4149235eeac75e75868
---
   drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 20

+++++++++++++++++++-

Quoted 1 line
Quoted 1 line
Quoted 1 line
   drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 22

++++++++++++++++++++-

Quoted 13 lines
Quoted 13 lines

-

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct

Quoted 8 lines
   2 files changed, 39 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
index fe31cbeccfe9..12fe5164aaf3 100644
@@ -238,6 +238,9 @@ static void xgpu_ai_mailbox_flr_work(struct

work_struct *work)

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct · line 238

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
   struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt,

flr_work);

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +0 -0
   struct amdgpu_device *adev = container_of(virt, struct
Quoted 36 lines
Quoted 7 lines

amdgpu_device, virt);

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 242

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -0
   int timeout = AI_MAILBOX_POLL_FLR_TIMEDOUT;
+int i;
+bool need_do_recover = true;

We should find a better name for "need_do_recover", may be "need_to_recover" ?

Thanks, will modify later.

Quoted 28 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work)

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +2 -0
+struct drm_sched_job *job;
 
   /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
    * otherwise the mailbox msg will be ruined/reseted by @@ -258,10
+261,25 @@ static void xgpu_ai_mailbox_flr_work(struct

work_struct *work)

Quoted 7 lines
   flr_done:
   up_read(&adev->reset_sem);
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +5 -0
+for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring =
+adev->rings[i];
+
+if (!ring || !ring->sched.thread)
+continue;

drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c static void xgpu_ai_mailbox_flr_work(struct work_struct *work) · line 278

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +10 -0
+
+spin_lock(&ring->sched.job_list_lock);
+job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+struct drm_sched_job, node);
+spin_unlock(&ring->sched.job_list_lock);
+if (job) {
+need_do_recover = false;
+break;
+}
+}

This 1st job retrieval logic can move to a function as there are two instance of it.
Sorry, I didn't get your point.

xgpu_ai_mailbox_flr_work() and xgpu_nv_mailbox_flr_work() are using same logic under "flr_done:" label trying to retrieve 1st job entry to determine if we should do recover or not.

We could move that logic into a function like:

bool function_name ()
{
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];

if (!ring || !ring->sched.thread)
continue;

spin_lock(&ring->sched.job_list_lock);
job = list_first_entry_or_null(&ring->sched.ring_mirror_list, struct drm_sched_job, node); spin_unlock(&ring->sched.job_list_lock);
if (job)
return true;

}

Quoted 7 lines

return false;
}

and use that in xgpu_ai_mailbox_flr_work() and
xgpu_nv_mailbox_flr_work() instead of

having two copy of that logic.

Understand completely now. Thanks.

Quoted 61 lines

Nirmoy

Quoted 56 lines
Quoted 56 lines
Quoted 5 lines
   /* Trigger recovery for world switch failure if no TDR */
   if (amdgpu_device_should_recover_gpu(adev)

-&& adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)
+&& (need_do_recover || adev->sdma_timeout ==

MAX_SCHEDULE_TIMEOUT))

Review

Quoted 4 lines
   amdgpu_device_gpu_recover(adev, NULL);
   }
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct

Quoted 4 lines

index 6f55172e8337..fc92c494df0b 100644

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
@@ -259,6 +259,9 @@ static void xgpu_nv_mailbox_flr_work(struct

work_struct *work)

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct · line 259

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
   struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt,

flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct · line 260

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +0 -0
   struct amdgpu_device *adev = container_of(virt, struct

amdgpu_device, virt);

drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c static void xgpu_nv_mailbox_flr_work(struct work_struct *work)

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +4 -0
   int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT;
+int i;
+bool need_do_recover = true;
+struct drm_sched_job *job;
 
   /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
    * otherwise the mailbox msg will be ruined/reseted by @@ -279,10
+282,25 @@ static void xgpu_nv_mailbox_flr_work(struct

work_struct *work)

Quoted 19 lines
   flr_done:
   up_read(&adev->reset_sem);
drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +15 -0
+for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring =
+adev->rings[i];
+
+if (!ring || !ring->sched.thread)
+continue;
+
+spin_lock(&ring->sched.job_list_lock);
+job = list_first_entry_or_null(&ring->sched.ring_mirror_list,
+struct drm_sched_job, node);
+spin_unlock(&ring->sched.job_list_lock);
+if (job) {
+need_do_recover = false;
+break;
+}
+}
 
   /* Trigger recovery for world switch failure if no TDR */ -if
Quoted 3 lines

(amdgpu_device_should_recover_gpu(adev)
-&& (adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT || +if (amdgpu_device_should_recover_gpu(adev) && (need_do_recover

||
> +adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
>    adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
>    adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
>    adev->video_timeout == MAX_SCHEDULE_TIMEOUT))

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx