Skip to contentSkip to conversation

drm/amdgpu: Fix the null pointer issue for tdr

PATCH NAK Emily Deng 4 participants 40 messages Nov 7, 2019 – Nov 18, 2019 35 messages not in the archive Page 1 of 2

Subsystems (4)
lore
Share & tools
lore

Emily Deng

author
#0 lore raw

PATCH 1/1 drm/amdgpu: Fix the null pointer issue for tdr

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
 drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	 *
 	 * job->base holds a reference to parent fence
 	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
 	    dma_fence_is_signaled(job->base.s_fence->parent))
 		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +6 -5
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct drm_sched_job *bad)
 
 			spin_lock(&rq->lock);
 			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence->scheduled.context ==
+				    entity->fence_context)) {
 					if (atomic_read(&bad->karma) >
 					    bad->sched->hang_limit)
 						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	 * This iteration is thread safe as sched thread is stopped.
 	 */
 	list_for_each_entry_safe_reverse(s_job, tmp, &sched->ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
 		    dma_fence_remove_callback(s_job->s_fence->parent,
 					      &s_job->cb)) {
 			atomic_dec(&sched->hw_rq_count);
@@ -395,7 +395,8 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 			 *
 			 * Job is still alive so fence refcount at least 1
 			 */
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished, false);
 
 			/*
 			 * We must keep bad job alive for later use during
@@ -438,7 +439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
 	 * GPU recovers can't run in parallel.
 	 */
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence->parent : NULL;
 
 		atomic_inc(&sched->hw_rq_count);
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1573122349-22080-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org

#2 lore raw

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

Quoted 55 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
  drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
  2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
  	 *
  	 * job->base holds a reference to parent fence
  	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
  	    dma_fence_is_signaled(job->base.s_fence->parent))
  		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +5 -4
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct drm_sched_job *bad)
 
  			spin_lock(&rq->lock);
  			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence->scheduled.context ==
+				    entity->fence_context)) {
  					if (atomic_read(&bad->karma) >
  					    bad->sched->hang_limit)
  						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
  	 * This iteration is thread safe as sched thread is stopped.
  	 */
  	list_for_each_entry_safe_reverse(s_job, tmp, &sched->ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
  		    dma_fence_remove_callback(s_job->s_fence->parent,
  					      &s_job->cb)) {
  			atomic_dec(&sched->hw_rq_count);
@@ -395,7 +395,8 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
  			 *
  			 * Job is still alive so fence refcount at least 1
  			 */
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished, false);
 
  			/*
  			 * We must keep bad job alive for later use during

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) · line 445

Quoted 8 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
@@ -438,7 +439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
  	 * GPU recovers can't run in parallel.
  	 */
  	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence->parent : NULL;
 
  		atomic_inc(&sched->hw_rq_count);
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 9de32e5b-69a2-f43f-629f-fef3c30bf5a1-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#4 lore raw
Quoted 6 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 6 lines

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

Quoted 78 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 13 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
  drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
  2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
  	 *
  	 * job->base holds a reference to parent fence
  	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
  	    dma_fence_is_signaled(job->base.s_fence->parent))
  		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct drm_sched_job *bad)

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
  			spin_lock(&rq->lock);
  			list_for_each_entry_safe(entity, tmp, &rq->entities,

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler

Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -2
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
  					if (atomic_read(&bad->karma) >
  					    bad->sched->hang_limit)
  						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler

Quoted 10 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
  	 * This iteration is thread safe as sched thread is stopped.
  	 */
  	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
  		    dma_fence_remove_callback(s_job->s_fence->parent,
  					      &s_job->cb)) {
  			atomic_dec(&sched->hw_rq_count);
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -395,7 +395,8 @@ void drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)

Quoted 6 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -1
  			 *
  			 * Job is still alive so fence refcount at least 1
  			 */
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished,

false);

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler · line 401

Quoted 3 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
  			/*
  			 * We must keep bad job alive for later use during @@

-438,7

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler · line 402

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +1 -0
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool

full_recovery)

Quoted 3 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node)

Quoted 8 lines

{

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence-

parent :
+NULL;

  		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB2975D4E26CED960B82305F308F7B0-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#6 lore raw

Ping.....

Best wishes
Emily Deng

Quoted 111 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 6 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 6 lines

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

Quoted 51 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 13 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
  drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
  2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
  	 *
  	 * job->base holds a reference to parent fence
  	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
  	    dma_fence_is_signaled(job->base.s_fence->parent))
  		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct drm_sched_job *bad)

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
  			spin_lock(&rq->lock);
  			list_for_each_entry_safe(entity, tmp, &rq->entities,

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler

Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -2
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
  					if (atomic_read(&bad->karma) >
  					    bad->sched->hang_limit)
  						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
  	 * This iteration is thread safe as sched thread is stopped.
  	 */
  	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
  		    dma_fence_remove_callback(s_job->s_fence->parent,
  					      &s_job->cb)) {

atomic_dec(&sched->hw_rq_count); @@ -395,7

+395,8 @@ void

Quoted 25 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)

Quoted 6 lines

*
* Job is still alive so fence refcount at least 1 */

drivers/gpu/drm/scheduler/sched_main.c +2 -1
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 3 lines

/*
* We must keep bad job alive for later use during @@

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool

full_recovery)

Quoted 4 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
node)

{

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence-

parent :
+NULL;

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
  		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB2975E26D8A8352863BA01FCA8F7B0-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#8 lore raw

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 31 lines

Best wishes
Emily Deng

Quoted 24 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 11 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 7 lines

        drm_sched_job_cleanup(s_job);

        amdgpu_ring_priority_put(ring, s_job->s_priority); dma_fence_put(job->fence); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); kfree(job);

The job itself is freed up directly after freeing the reference to the s_fence.

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 83 lines
Quoted 83 lines
Quoted 49 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
   drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
   2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
   	 *
   	 * job->base holds a reference to parent fence
   	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
   	    dma_fence_is_signaled(job->base.s_fence->parent))
   		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
   			spin_lock(&rq->lock);
   			list_for_each_entry_safe(entity, tmp, &rq->entities,

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler

Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -2
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
   					if (atomic_read(&bad->karma) >
   					    bad->sched->hang_limit)
   						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
   	 * This iteration is thread safe as sched thread is stopped.
   	 */
   	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
   		    dma_fence_remove_callback(s_job->s_fence->parent,
   					      &s_job->cb)) {
   			atomic_dec(&sched->hw_rq_count); @@ -395,7

+395,8 @@ void

Quoted 24 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 6 lines
   			 *
   			 * Job is still alive so fence refcount at least 1
   			 */
drivers/gpu/drm/scheduler/sched_main.c +2 -1
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use during @@

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool

full_recovery)

Quoted 4 lines
   	 * GPU recovers can't run in parallel.
   	 */
   	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,

node)

{

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence-

parent :
+NULL;

Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
   		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): c01acb29-72ce-a109-3ca5-166706327d61-5C7GfCeVMHo@public.gmane.org

#10 lore raw

Sorry, please take your time.

Best wishes
Emily Deng

Quoted 53 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 31 lines

Best wishes
Emily Deng

Quoted 24 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 11 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 7 lines

        drm_sched_job_cleanup(s_job);

        amdgpu_ring_priority_put(ring, s_job->s_priority); dma_fence_put(job->fence); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); kfree(job);

Quoted 98 lines

The job itself is freed up directly after freeing the reference to the s_fence.

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 90 lines
Quoted 90 lines
Quoted 49 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
   drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
   2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
   	 *
   	 * job->base holds a reference to parent fence
   	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
   	    dma_fence_is_signaled(job->base.s_fence->parent))
   		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
   			spin_lock(&rq->lock);
   			list_for_each_entry_safe(entity, tmp, &rq->entities,

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler

Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -2
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
   					if (atomic_read(&bad->karma) >
   					    bad->sched->hang_limit)
   						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
   	 * This iteration is thread safe as sched thread is stopped.
   	 */
   	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
   		    dma_fence_remove_callback(s_job->s_fence->parent,
   					      &s_job->cb)) {
   			atomic_dec(&sched->hw_rq_count); @@ -395,7

+395,8 @@ void

Quoted 19 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 6 lines
   			 *
   			 * Job is still alive so fence refcount at least 1
   			 */
drivers/gpu/drm/scheduler/sched_main.c +2 -1
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use during @@

-438,7

Quoted 2 lines

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, +bool

full_recovery)

Quoted 4 lines
   	 * GPU recovers can't run in parallel.
   	 */
   	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,

node)

Quoted 8 lines

{

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence-

parent :
+NULL;

   		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB29755CFCE09CEC0D9EB999D18F7B0-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#12 lore raw

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 153 lines

Best wishes
Emily Deng

Quoted 147 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 30 lines

Best wishes
Emily Deng

Quoted 24 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 11 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 7 lines

        drm_sched_job_cleanup(s_job);

        amdgpu_ring_priority_put(ring, s_job->s_priority); dma_fence_put(job->fence); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); kfree(job);

The job itself is freed up directly after freeing the reference to the s_fence.

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 90 lines
Quoted 90 lines
Quoted 49 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
    drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
    2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 14 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
    	 *
    	 * job->base holds a reference to parent fence
    	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent &&
    	    dma_fence_is_signaled(job->base.s_fence->parent))
    		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
    			spin_lock(&rq->lock);
    			list_for_each_entry_safe(entity, tmp, &rq->entities,

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct drm_gpu_scheduler

Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -2
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
    					if (atomic_read(&bad->karma) >
    					    bad->sched->hang_limit)
    						if (entity->guilty)
@@ -376,7 +376,7 @@ void drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
    	 * This iteration is thread safe as sched thread is stopped.
    	 */
    	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
    		    dma_fence_remove_callback(s_job->s_fence->parent,
    					      &s_job->cb)) {
    			atomic_dec(&sched->hw_rq_count); @@ -395,7

+395,8 @@ void

Quoted 19 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 6 lines
    			 *
    			 * Job is still alive so fence refcount at least 1
    			 */
drivers/gpu/drm/scheduler/sched_main.c +2 -1
-			dma_fence_wait(&s_job->s_fence->finished, false);
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use during @@

-438,7

Quoted 2 lines

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, +bool

full_recovery)

Quoted 4 lines
    	 * GPU recovers can't run in parallel.
    	 */
    	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,

node)

Quoted 8 lines

{

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job->s_fence-

parent :
+NULL;

    		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 70c2c1cc-40b8-30da-7aee-f59fbc4d0d42-5C7GfCeVMHo@public.gmane.org

#14 lore raw

Hi Christian,
Please refer to follow log, when it enter to amdgpu_device_gpu_recover function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence signal. But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=2481, emitted seq=2483
[  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:000000005086879e
[  449.794163] amdgpu 0000:00:08.0: GPU reset begin!
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:000000005086879e
[  449.794221] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:0000000066eb74ab
[  449.794222] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000d4438ad9
[  449.794255] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000b6d69c65
[  449.794257] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000ea85e922
[  449.794287] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6
[  449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0
[  449.800818] PGD 0 P4D 0
[  449.801040] Oops: 0000 [#1] SMP PTI
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE     4.18.0-15-generic #16~18.04.1-Ubuntu
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014
[  449.802944] Workqueue: events drm_sched_job_timedout [amd_sched]
[  449.803488] RIP: 0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff 45 85 e4 0f 85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b 98 c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286
[  449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[  449.806625] RDX: ffffb4c7c08f5ac0 RSI: 0000000fffffffe0 RDI: 0000000000000246
[  449.807224] RBP: ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000
[  449.807818] R10: 0000000000000000 R11: 0000000000000148 R12: 0000000000000000
[  449.808411] R13: ffffb4c7c08f7da0 R14: ffff8d82b8525d40 R15: ffff8d82b8525d40
[  449.809004] FS:  0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000
[  449.809674] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  449.810153] CR2: 00000000000000c0 CR3: 000000003cc0a001 CR4: 00000000003606e0
[  449.810747] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  449.811937] Call Trace:
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu]
[  449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched]
[  449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu]
[  449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched]
[  449.814077]  process_one_work+0x1fd/0x3f0
[  449.814417]  worker_thread+0x34/0x410
[  449.814728]  kthread+0x121/0x140
[  449.815004]  ? process_one_work+0x3f0/0x3f0
[  449.815374]  ? kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 157 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 84 lines

Best wishes
Emily Deng

Quoted 78 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 32 lines

Best wishes
Emily Deng

Quoted 26 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 8 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 5 lines

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 7 lines

        drm_sched_job_cleanup(s_job);

        amdgpu_ring_priority_put(ring, s_job->s_priority); dma_fence_put(job->fence); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); kfree(job);

The job itself is freed up directly after freeing the reference to the s_fence.

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
    drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
    2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct · line 4078

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
    	 *
    	 * job->base holds a reference to parent fence
    	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
    	    dma_fence_is_signaled(job->base.s_fence->parent))
    		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 7 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
    			spin_lock(&rq->lock);
    			list_for_each_entry_safe(entity, tmp, &rq-

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 338

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-				if (bad->s_fence->scheduled.context

==

Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
    					if (atomic_read(&bad-

karma) >

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct

Quoted 3 lines
    					    bad->sched->hang_limit)
    						if (entity->guilty)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -376,7 +376,7 @@ void drm_sched_stop(struct

drm_gpu_scheduler

Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct · line 381

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
    	 * This iteration is thread safe as sched thread is stopped.
    	 */
    	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
    		    dma_fence_remove_callback(s_job->s_fence-

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
    					      &s_job->cb)) {
    			atomic_dec(&sched->hw_rq_count); @@ -

395,7

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

+395,8 @@ void

Quoted 6 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 4 lines
    			 *
    			 * Job is still alive so fence refcount at least 1
    			 */
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-			dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 29 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use

during @@

Quoted 22 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-438,7

Quoted 2 lines

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, +bool

full_recovery)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

    		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): DM6PR12MB2971859C1BF16EE7E65B35B18F7B0-lmeGfMZKVrGd4IXjMPYtUQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#16 lore raw

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 34 lines

Hi Christian,
Please refer to follow log, when it enter to amdgpu_device_gpu_recover function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence signal. But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=2481, emitted seq=2483
[  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:000000005086879e
[  449.794163] amdgpu 0000:00:08.0: GPU reset begin!
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:000000005086879e
[  449.794221] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:0000000066eb74ab
[  449.794222] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000d4438ad9
[  449.794255] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000b6d69c65
[  449.794257] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000ea85e922
[  449.794287] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6
[  449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0
[  449.800818] PGD 0 P4D 0
[  449.801040] Oops: 0000 [#1] SMP PTI
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE     4.18.0-15-generic #16~18.04.1-Ubuntu
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014
[  449.802944] Workqueue: events drm_sched_job_timedout [amd_sched]
[  449.803488] RIP: 0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff 45 85 e4 0f 85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b 98 c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286
[  449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[  449.806625] RDX: ffffb4c7c08f5ac0 RSI: 0000000fffffffe0 RDI: 0000000000000246
[  449.807224] RBP: ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000
[  449.807818] R10: 0000000000000000 R11: 0000000000000148 R12: 0000000000000000
[  449.808411] R13: ffffb4c7c08f7da0 R14: ffff8d82b8525d40 R15: ffff8d82b8525d40
[  449.809004] FS:  0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000
[  449.809674] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  449.810153] CR2: 00000000000000c0 CR3: 000000003cc0a001 CR4: 00000000003606e0
[  449.810747] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  449.811937] Call Trace:
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu]
[  449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched]
Quoted 192 lines
[  449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu]
[  449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched]
[  449.814077]  process_one_work+0x1fd/0x3f0
[  449.814417]  worker_thread+0x34/0x410
[  449.814728]  kthread+0x121/0x140
[  449.815004]  ? process_one_work+0x3f0/0x3f0
[  449.815374]  ? kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 183 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 82 lines

Best wishes
Emily Deng

Quoted 77 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 31 lines

Best wishes
Emily Deng

Quoted 26 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 8 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 5 lines

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free job. But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 7 lines
          drm_sched_job_cleanup(s_job);

          amdgpu_ring_priority_put(ring, s_job->s_priority);
          dma_fence_put(job->fence);
          amdgpu_sync_free(&job->sync);
          amdgpu_sync_free(&job->sched_sync);
          kfree(job);

The job itself is freed up directly after freeing the reference to the s_fence.

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
     drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
     2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct · line 4078

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
     	 *
     	 * job->base holds a reference to parent fence
     	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
     	    dma_fence_is_signaled(job->base.s_fence->parent))
     		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 7 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
     			spin_lock(&rq->lock);
     			list_for_each_entry_safe(entity, tmp, &rq-

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 338

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-				if (bad->s_fence->scheduled.context

==

Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
     					if (atomic_read(&bad-

karma) >

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct

Quoted 3 lines
     					    bad->sched->hang_limit)
     						if (entity->guilty)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -376,7 +376,7 @@ void drm_sched_stop(struct

drm_gpu_scheduler

Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct · line 381

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
     	 * This iteration is thread safe as sched thread is stopped.
     	 */
     	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
     		    dma_fence_remove_callback(s_job->s_fence-

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
     					      &s_job->cb)) {
     			atomic_dec(&sched->hw_rq_count); @@ -

395,7

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

+395,8 @@ void

Quoted 6 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 4 lines
     			 *
     			 * Job is still alive so fence refcount at least 1
     			 */
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-			dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use

during @@

Quoted 22 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-438,7

Quoted 2 lines

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, +bool

full_recovery)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

     		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): d6f9c508-3c23-c797-1cbc-7502dc4c0b13-5C7GfCeVMHo@public.gmane.org

#18 lore raw

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it are freed by main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Best wishes
Emily Deng

Quoted 245 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence signal. But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

Quoted 4 lines
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu 0000:00:08.0: GPU reset begin!

Quoted 9 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process

pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6

[  449.794366] BUG: unable to handle kernel NULL pointer dereference at

00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops: 0000 [#1] SMP PTI

Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 3 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996),

BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [ 449.802944] Workqueue: events drm_sched_job_timedout [amd_sched] [ 449.803488] RIP:

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 1 line
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff 45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b 98
c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 12 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000 CR0:
0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:

0000000000000400 [ 449.811937] Call Trace:

Quoted 37 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 27 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 28 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 22 lines

Best wishes
Emily Deng

Quoted 17 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 8 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
          drm_sched_job_cleanup(s_job);

          amdgpu_ring_priority_put(ring, s_job->s_priority);
          dma_fence_put(job->fence);
          amdgpu_sync_free(&job->sync);
          amdgpu_sync_free(&job->sched_sync);
          kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 83 lines
Quoted 83 lines
Quoted 26 lines
Quoted 26 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
     drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
     drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
     2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct · line 4078

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
     	 *
     	 * job->base holds a reference to parent fence
     	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
     	    dma_fence_is_signaled(job->base.s_fence->parent))
     		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 7 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
     			spin_lock(&rq->lock);
     			list_for_each_entry_safe(entity, tmp, &rq-

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 338

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-				if (bad->s_fence->scheduled.context

==

Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
     					if (atomic_read(&bad-

karma) >

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct

Quoted 3 lines
     					    bad->sched->hang_limit)
     						if (entity->guilty)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -376,7 +376,7 @@ void drm_sched_stop(struct

drm_gpu_scheduler

Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct · line 381

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
     	 * This iteration is thread safe as sched thread is stopped.
     	 */
     	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
     		    dma_fence_remove_callback(s_job->s_fence-

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
     					      &s_job->cb)) {
     			atomic_dec(&sched->hw_rq_count); @@ -

395,7

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

+395,8 @@ void

Quoted 6 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 4 lines
     			 *
     			 * Job is still alive so fence refcount at least 1
     			 */
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-			dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence-

finished,

Quoted 27 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

     		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): DM6PR12MB29714AB9AD16FA3ABD7D62C28F7B0-lmeGfMZKVrGd4IXjMPYtUQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#20 lore raw

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 277 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it are freed by main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Best wishes
Emily Deng

Quoted 269 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence signal. But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

Quoted 3 lines
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu 0000:00:08.0: GPU reset begin!

Quoted 9 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process

pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6

[  449.794366] BUG: unable to handle kernel NULL pointer dereference at

00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops: 0000 [#1] SMP PTI

Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 3 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996),

BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [ 449.802944] Workqueue: events drm_sched_job_timedout [amd_sched] [ 449.803488] RIP:

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 1 line
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff 45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b 98
c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 12 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000 CR0:
0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[  449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:

0000000000000400 [ 449.811937] Call Trace:

Quoted 37 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 27 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 28 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 22 lines

Best wishes
Emily Deng

Quoted 17 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 8 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
           drm_sched_job_cleanup(s_job);

           amdgpu_ring_priority_put(ring, s_job->s_priority);
           dma_fence_put(job->fence);
           amdgpu_sync_free(&job->sync);
           amdgpu_sync_free(&job->sched_sync);
           kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 82 lines
Quoted 82 lines
Quoted 25 lines
Quoted 25 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct

Quoted 12 lines
---
      drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
      drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
      2 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int amdgpu_device_gpu_recover(struct

amdgpu_device *adev,

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int amdgpu_device_gpu_recover(struct · line 4078

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
      	 *
      	 * job->base holds a reference to parent fence
      	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
      	    dma_fence_is_signaled(job->base.s_fence->parent))
      		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 7 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
      			spin_lock(&rq->lock);
      			list_for_each_entry_safe(entity, tmp, &rq-

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 338

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-				if (bad->s_fence->scheduled.context

==

Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
      					if (atomic_read(&bad-

karma) >

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct

Quoted 3 lines
      					    bad->sched->hang_limit)
      						if (entity->guilty)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -376,7 +376,7 @@ void drm_sched_stop(struct

drm_gpu_scheduler

Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_stop(struct · line 381

Quoted 7 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
      	 * This iteration is thread safe as sched thread is stopped.
      	 */
      	list_for_each_entry_safe_reverse(s_job, tmp, &sched-
 ring_mirror_list, node) {
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
      		    dma_fence_remove_callback(s_job->s_fence-

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
      					      &s_job->cb)) {
      			atomic_dec(&sched->hw_rq_count); @@ -

395,7

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

+395,8 @@ void

Quoted 6 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

drivers/gpu/drm/scheduler/sched_main.c

Quoted 4 lines
      			 *
      			 * Job is still alive so fence refcount at least 1
      			 */
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-			dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence-

finished,

Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later use

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

      		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): dcc1124b-5e19-b018-7449-659a8b7d74ea-5C7GfCeVMHo@public.gmane.org

#22 lore raw

Hi Christian,

     The drm_sched_job_timedout-> amdgpu_job_timedout call amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop. 

Best wishes
Emily Deng

Quoted 287 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 34 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 21 lines
Quoted 21 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process

pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops: 0000 [#1] SMP PTI

Quoted 5 lines
Quoted 5 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 3 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [  449.803488]

RIP:

Quoted 4 lines
Quoted 4 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 2 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff

45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 26 lines
Quoted 26 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 12 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000 CR0:
0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 31 lines
Quoted 31 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 57 lines
Quoted 57 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
           drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
           amdgpu_ring_priority_put(ring, s_job->s_priority);
           dma_fence_put(job->fence);
           amdgpu_sync_free(&job->sync);
           amdgpu_sync_free(&job->sched_sync);
           kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 18 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

Quoted 6 lines
---
      drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
      drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
      2 files changed, 7 insertions(+), 6 deletions(-)
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int · line 4078

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
      	 *
      	 * job->base holds a reference to parent fence
      	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
      	    dma_fence_is_signaled(job->base.s_fence->parent))
      		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
      			spin_lock(&rq->lock);
      			list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 338

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-				if (bad->s_fence->scheduled.context

==

Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
      					if (atomic_read(&bad-

karma) >

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
      					    bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*sched, struct drm_sched_job *bad)

Quoted 1 line

* This iteration is thread safe as sched thread is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp, &sched-
ring_mirror_list, node) {

drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
      		    dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
      					      &s_job->cb)) {
      			atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

395,7

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+395,8 @@ void

Quoted 4 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 2 lines

*
* Job is still alive so fence refcount at

least 1

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
      			 */
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-			dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 31 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later

use

Quoted 24 lines
Quoted 24 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

      		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): DM6PR12MB29710DFE90F22F5903499AFE8F7B0-lmeGfMZKVrGd4IXjMPYtUQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#24 lore raw

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

        /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 325 lines

Hi Christian,

      The drm_sched_job_timedout-> amdgpu_job_timedout call amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Best wishes
Emily Deng

Quoted 317 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 21 lines
Quoted 21 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process

pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops: 0000 [#1] SMP PTI

Quoted 5 lines
Quoted 5 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 3 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [  449.803488]

RIP:

Quoted 4 lines
Quoted 4 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 2 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff

45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 26 lines
Quoted 26 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 12 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000 CR0:
0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 31 lines
Quoted 31 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 57 lines
Quoted 57 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
            drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
            amdgpu_ring_priority_put(ring, s_job->s_priority);
            dma_fence_put(job->fence);
            amdgpu_sync_free(&job->sync);
            amdgpu_sync_free(&job->sched_sync);
            kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 18 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

Quoted 6 lines
---
       drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
       drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
       2 files changed, 7 insertions(+), 6 deletions(-)
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int · line 4078

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
       	 *
       	 * job->base holds a reference to parent fence
       	 */
-	if (job && job->base.s_fence->parent &&
+	if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
       	    dma_fence_is_signaled(job->base.s_fence->parent))
       		job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
       			spin_lock(&rq->lock);
       			list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 338

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-				if (bad->s_fence->scheduled.context

==

Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-				    entity->fence_context) {
+				if (bad->s_fence && (bad->s_fence-

scheduled.context ==

drivers/gpu/drm/scheduler/sched_main.c +1 -0
+				    entity->fence_context)) {
       					if (atomic_read(&bad-

karma) >

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
       					    bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*sched, struct drm_sched_job *bad)

Quoted 1 line

* This iteration is thread safe as sched thread is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp, &sched-
ring_mirror_list, node) {

drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		if (s_job->s_fence->parent &&
+		if (s_job->s_fence && s_job->s_fence->parent &&
       		    dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
       					      &s_job->cb)) {
       			atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

395,7

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+395,8 @@ void

Quoted 4 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 2 lines

*
* Job is still alive so fence refcount at

least 1

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
       			 */
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-			dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+			if (s_job->s_fence)
+				dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later

use

Quoted 24 lines
Quoted 24 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-		struct dma_fence *fence = s_job->s_fence->parent;
+		struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

       		atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 91f4a0c4-23e3-a399-5cb1-fb01da922784-5C7GfCeVMHo@public.gmane.org

#26 lore raw

Hi Christian,
Sorry, seems I understand wrong. And from the print, the free job's thread is the same as job timeout thread. So seems have some issue in function amdgpu_device_gpu_recover.

Best wishes
Emily Deng

Quoted 353 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:35 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

        /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

      The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it
are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff

ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
            drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
            amdgpu_ring_priority_put(ring, s_job->s_priority);
            dma_fence_put(job->fence);
            amdgpu_sync_free(&job->sync);
            amdgpu_sync_free(&job->sched_sync);
            kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Quoted 4 lines

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11 ++++++---

--

Quoted 20 lines
Quoted 9 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

amdgpu_device *adev,

Quoted 5 lines

*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence && job->base.s_fence-

parent

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 24 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_job

Quoted 4 lines

*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

Quoted 1 line

- if (bad->s_fence-

scheduled.context

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

==

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- entity->fence_context) {
+ if (bad->s_fence && (bad-

s_fence-

Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 8 lines

8 lines not shown · raw message

37 lines not shown · raw message

Quoted 15 lines

15 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): DM6PR12MB2971D540D3000B67E44970AF8F7B0-lmeGfMZKVrGd4IXjMPYtUQdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#28 lore raw
Quoted 2 lines · Emily

Hi Christian,
Sorry, seems I understand wrong. And from the print, the free job's thread is the same as job timeout thread. So seems have some issue in function amdgpu_device_gpu_recover.

I don't think it's correct, seems your prints just don't print the pids because it's all zeros which cannot be true for any kernel or user mode thread. In fact looking at the OOPs i see the actual TDR job pid which is 55

CPU: 3 PID: 55 Comm: kworker/3:1 Tainted

Andrey

Quoted 369 lines

Best wishes
Emily Deng

Quoted 362 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:35 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

        /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 35 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it
are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff

ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
             drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
             amdgpu_ring_priority_put(ring, s_job->s_priority);
             dma_fence_put(job->fence);
             amdgpu_sync_free(&job->sync);
             amdgpu_sync_free(&job->sched_sync);
             kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11 ++++++---

--

Quoted 20 lines
Quoted 9 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence && job->base.s_fence-

parent

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 24 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {
- if (bad->s_fence-

scheduled.context

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

==

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- entity->fence_context) {
+ if (bad->s_fence && (bad-

s_fence-

Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread is

stopped.

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
- if (s_job->s_fence->parent &&
+ if (s_job->s_fence && s_job->s_fence->parent

&&

Quoted 36 lines
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

395,7

Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+395,8 @@ void

Quoted 4 lines

drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence refcount at

least 1

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ if (s_job->s_fence)
+ dma_fence_wait(&s_job-

s_fence-

Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);
/*
* We must keep bad job alive for later

use

Quoted 11 lines
Quoted 11 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

{
- struct dma_fence *fence = s_job->s_fence-

parent;

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

+ struct dma_fence *fence = s_job->s_fence ?

s_job-

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

s_fence-

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#29 lore raw
In reply to a message not in the archive
Quoted 10 lines · Christian

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

        /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

I don't think this protects us if drm_sched_cleanup_jobs is called for scheduler which didn't experience a timeout, in amdgpu_device_gpu_recover we access sched->ring_mirror_list for all the schedulers on a device so this condition above won't protect us. What in fact could help maybe is my recent patch 541c521 drm/sched: Avoid job cleanup if sched thread is parked. because we do park each of the scheduler threads during tdr job before trying to access sched->ring_mirror_list.

Emily - did you see this problem with that patch in place ? I only pushed it yesterday.

Andrey

Quoted 326 lines

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 321 lines

Hi Christian,

       The drm_sched_job_timedout-> amdgpu_job_timedout call amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Best wishes
Emily Deng

Quoted 313 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 21 lines
Quoted 21 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process

pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops: 0000 [#1] SMP PTI

Quoted 5 lines
Quoted 5 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 3 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [  449.803488]

RIP:

Quoted 4 lines
Quoted 4 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 2 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff

45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 26 lines
Quoted 26 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 12 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000 CR0:
0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 31 lines
Quoted 31 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 57 lines
Quoted 57 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then
it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
             drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
             amdgpu_ring_priority_put(ring, s_job->s_priority);
             dma_fence_put(job->fence);
             amdgpu_sync_free(&job->sync);
             amdgpu_sync_free(&job->sched_sync);
             kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 18 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

Quoted 6 lines

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11 ++++++-----
2 files changed, 7 insertions(+), 6 deletions(-)

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

amdgpu_device *adev,

Quoted 5 lines

*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

Quoted 1 line

- if (bad->s_fence->scheduled.context

==

Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

- entity->fence_context) {
+ if (bad->s_fence && (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

karma) >

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*sched, struct drm_sched_job *bad)

Quoted 1 line

* This iteration is thread safe as sched thread is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp, &sched-
ring_mirror_list, node) {
- if (s_job->s_fence->parent &&
+ if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

395,7

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+395,8 @@ void

Quoted 4 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 2 lines

*
* Job is still alive so fence refcount at

least 1

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later

use

Quoted 20 lines
Quoted 20 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

Quoted 2 lines

- struct dma_fence *fence = s_job->s_fence->parent;
+ struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 30ac4863-70e0-2b95-4819-e9431a6b4680-5C7GfCeVMHo@public.gmane.org

#31 lore raw

Hi Andrey,
I don’t think your patch will help for this. As it will may call kthread_should_park in drm_sched_cleanup_jobs first, and then call kcl_kthread_park. And then it still has a race between the 2 threads.

Best wishes
Emily Deng

Quoted 375 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Saturday, November 9, 2019 3:01 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 10 lines · Christian

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

        /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

I don't think this protects us if drm_sched_cleanup_jobs is called for scheduler which didn't experience a timeout, in amdgpu_device_gpu_recover we access sched->ring_mirror_list for all the schedulers on a device so this condition above won't protect us. What in fact could help maybe is my recent patch 541c521 drm/sched: Avoid job cleanup if sched thread is parked. because we do park each of the scheduler threads during tdr job before trying to access sched->ring_mirror_list.

Emily - did you see this problem with that patch in place ? I only pushed it yesterday.

Andrey

Quoted 7 lines

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 60 lines
Quoted 60 lines

Best wishes
Emily Deng

Quoted 54 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even
it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6

[ 449.794366] BUG: unable to handle kernel NULL pointer

dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff

ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000

CR0:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[

Quoted 105 lines
Quoted 105 lines
Quoted 105 lines
Quoted 12 lines
Quoted 12 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
             drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
             amdgpu_ring_priority_put(ring, s_job->s_priority);
             dma_fence_put(job->fence);
             amdgpu_sync_free(&job->sync);
             amdgpu_sync_free(&job->sched_sync);
             kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11 ++++++-

----

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 25 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {
- if (bad->s_fence-

scheduled.context

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

==

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- entity->fence_context) {
+ if (bad->s_fence && (bad-

s_fence-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

*sched, struct drm_sched_job *bad)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

2 lines not shown · raw message

6 lines not shown · raw message

37 lines not shown · raw message

Quoted 19 lines

19 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB2975652B5191BAC055C01BEC8F740-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#33 lore raw

Hi Christian and Andrey,
The issue I encountered is the bad job is freeing after entering to the amdgpu_device_gpu_recover. Don't know why, as per Christian said, it will call cancel_delayed_work in drm_sched_cleanup_jobs.

Best wishes
Emily Deng

Quoted 424 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Monday, November 11, 2019 3:19 PM
To: Grodzovsky, Andrey <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Andrey,
I don’t think your patch will help for this. As it will may call
kthread_should_park in drm_sched_cleanup_jobs first, and then call kcl_kthread_park. And then it still has a race between the 2 threads.

Best wishes
Emily Deng

Quoted 41 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Saturday, November 9, 2019 3:01 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 11 lines · Christian

Hi Emily,

exactly that can't happen. See here:

Quoted 5 lines

        /* Don't destroy jobs while the timeout worker is running
*/
        if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

I don't think this protects us if drm_sched_cleanup_jobs is called for scheduler which didn't experience a timeout, in amdgpu_device_gpu_recover we access
sched->ring_mirror_list for all the schedulers on a device so this sched->condition
above won't protect us. What in fact could help maybe is my recent patch
541c521 drm/sched: Avoid job cleanup if sched thread is parked. because we do park each of the scheduler threads during tdr job before trying to access
sched->ring_mirror_list.

Emily - did you see this problem with that patch in place ? I only pushed it yesterday.

Andrey

Quoted 1 line
Quoted 18 lines
Quoted 6 lines

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 10 lines
Quoted 10 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first

place.

Quoted 43 lines
Quoted 43 lines
Quoted 43 lines
Quoted 43 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even
it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

Quoted 12 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
Quoted 3 lines

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process

information:

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0,

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6

[ 449.794366] BUG: unable to handle kernel NULL pointer

dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G

OE

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 5 lines
Quoted 5 lines

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff

ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

98

Quoted 6 lines
Quoted 6 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 5 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000

[

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES:
0000

CR0:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2:

0000000000000000

Quoted 77 lines

[

Quoted 76 lines
Quoted 76 lines
Quoted 76 lines
Quoted 12 lines
Quoted 12 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer
issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:
When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is

destroyed.

Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in
one case, when it enter into the amdgpu_device_gpu_recover,
it already in drm_sched_job_cleanup, and at this time, it
will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
             drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
             amdgpu_ring_priority_put(ring, s_job->s_priority);
             dma_fence_put(job->fence);
             amdgpu_sync_free(&job->sync);
             amdgpu_sync_free(&job->sched_sync);
             kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11

++++++-

Quoted 55 lines

----

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 27 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {
- if (bad->s_fence-

scheduled.context

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

==

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

2 lines not shown · raw message

14 lines not shown · raw message

Quoted 35 lines

35 lines not shown · raw message

24 lines not shown · raw message

Quoted 8 lines

8 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB2975B736A666D9EEC5E5DB158F740-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#35 lore raw

Emily - is there a particular scenario to reproduce this ? I am trying with libdrm deadlock test and artificially delaying the GPU reset logic until after the guilty job is signaling but indeed nothing bad happens as drm_sched_cleanup_jobs returns early because there is a reset in progress and so the bad job is not getting released while GPU reset is running.

Can you provide event tracing for timer, dma_fence and gpu_scheduler for when the problem happens ?

Andrey

Quoted 436 lines · Emily

Hi Christian and Andrey,
The issue I encountered is the bad job is freeing after entering to the amdgpu_device_gpu_recover. Don't know why, as per Christian said, it will call cancel_delayed_work in drm_sched_cleanup_jobs.

Best wishes
Emily Deng

Quoted 428 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Monday, November 11, 2019 3:19 PM
To: Grodzovsky, Andrey <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Andrey,
I don’t think your patch will help for this. As it will may call
kthread_should_park in drm_sched_cleanup_jobs first, and then call kcl_kthread_park. And then it still has a race between the 2 threads.

Best wishes
Emily Deng

Quoted 39 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Saturday, November 9, 2019 3:01 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 11 lines · Christian

Hi Emily,

exactly that can't happen. See here:

Quoted 5 lines

        /* Don't destroy jobs while the timeout worker is running
*/
        if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

I don't think this protects us if drm_sched_cleanup_jobs is called for scheduler which didn't experience a timeout, in amdgpu_device_gpu_recover we access
sched->ring_mirror_list for all the schedulers on a device so this sched->condition
above won't protect us. What in fact could help maybe is my recent patch
541c521 drm/sched: Avoid job cleanup if sched thread is parked. because we do park each of the scheduler threads during tdr job before trying to access
sched->ring_mirror_list.

Emily - did you see this problem with that patch in place ? I only pushed it yesterday.

Andrey

Quoted 17 lines
Quoted 6 lines

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

        The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 9 lines
Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first

place.

Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even
it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

Quoted 12 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
Quoted 3 lines

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process

information:

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0,

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6

[ 449.794366] BUG: unable to handle kernel NULL pointer

dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G

OE

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 5 lines
Quoted 5 lines

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff

ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

98

Quoted 6 lines
Quoted 6 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 5 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000

[

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES:
0000

CR0:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2:

0000000000000000

Quoted 77 lines

[

Quoted 76 lines
Quoted 76 lines
Quoted 76 lines
Quoted 12 lines
Quoted 12 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer
issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:
When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is

destroyed.

Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in
one case, when it enter into the amdgpu_device_gpu_recover,
it already in drm_sched_job_cleanup, and at this time, it
will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines

        drm_sched_job_cleanup(s_job);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

        amdgpu_ring_priority_put(ring, s_job->s_priority);
        dma_fence_put(job->fence);
        amdgpu_sync_free(&job->sync);
        amdgpu_sync_free(&job->sched_sync);
        kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11

++++++-

Quoted 55 lines

----

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 27 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines

list) {
- if (bad->s_fence-

scheduled.context

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

==

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- entity->fence_context) {
+ if (bad->s_fence && (bad-

s_fence-

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
Quoted 3 lines

scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

karma) >

Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

3 lines not shown · raw message

Quoted 35 lines

35 lines not shown · raw message

32 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): 53130d01-da16-7cc0-55df-ea2532e6b3d0-5C7GfCeVMHo@public.gmane.org

#37 lore raw

Hi Andrey,
On my side, it doesn't need to a specific scenario, I only run the quark with slow job. Then sometimes, it will have fake hang and hardware fence will back. For this case, it will randomly occur the NULL pointer issue in amdgpu_device_gpu_recover.

Quoted 441 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 5:35 AM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Emily - is there a particular scenario to reproduce this ? I am trying with libdrm deadlock test and artificially delaying the GPU reset logic until after the guilty job is signaling but indeed nothing bad happens as drm_sched_cleanup_jobs returns early because there is a reset in progress and so the bad job is not getting released while GPU reset is running.

Can you provide event tracing for timer, dma_fence and gpu_scheduler for when the problem happens ?

Andrey

Quoted 2 lines · Emily

Hi Christian and Andrey,
The issue I encountered is the bad job is freeing after entering to the

amdgpu_device_gpu_recover. Don't know why, as per Christian said, it will call cancel_delayed_work in drm_sched_cleanup_jobs.

Quoted 11 lines

Best wishes
Emily Deng

Quoted 5 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Deng, Emily
Sent: Monday, November 11, 2019 3:19 PM
To: Grodzovsky, Andrey <[email protected]>; Koenig,

Christian

Quoted 52 lines
Quoted 52 lines

<[email protected]>; [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Andrey,
I don’t think your patch will help for this. As it will may call
kthread_should_park in drm_sched_cleanup_jobs first, and then call kcl_kthread_park. And then it still has a race between the 2 threads.

Best wishes
Emily Deng

Quoted 39 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Saturday, November 9, 2019 3:01 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 11 lines · Christian

Hi Emily,

exactly that can't happen. See here:

Quoted 5 lines

        /* Don't destroy jobs while the timeout worker is
running */
        if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

I don't think this protects us if drm_sched_cleanup_jobs is called for scheduler which didn't experience a timeout, in amdgpu_device_gpu_recover we access sched->ring_mirror_list for all the schedulers on a device so this sched->condition
above won't protect us. What in fact could help maybe is my recent patch
541c521 drm/sched: Avoid job cleanup if sched thread is parked. because we do park each of the scheduler threads during tdr job before trying to access
sched->ring_mirror_list.

Emily - did you see this problem with that patch in place ? I only pushed it yesterday.

Andrey

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

        The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling

drm_sched_stop.

Quoted 65 lines
Quoted 65 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first

place.

Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 45 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch.
Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 36 lines

Best wishes
Emily Deng

Quoted 31 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR*

ring

Quoted 17 lines
Quoted 17 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

sdma0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process

information:

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0,

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040]

Oops:

Quoted 13 lines
Quoted 13 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

0000 [#1] SMP PTI

Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G

OE

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
Quoted 5 lines

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 4 lines
Quoted 4 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff

ff ff
45 85 e4 0f

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 3 lines
Quoted 3 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 2 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000

RCX:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09:

0000000000000000

Quoted 2 lines
Quoted 2 lines

[

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.807818] R10: 0000000000000000 R11: 0000000000000148

R12:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES:
0000

CR0:

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

Quoted 8 lines
Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

0000000000000000 DR1: 0000000000000000 DR2:

0000000000000000

Quoted 6 lines

[

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 3 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [

449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139] ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu]

[

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.813609] ? drm_sched_job_timedout+0x44/0x90 [amd_sched]

[

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

449.814077] process_one_work+0x1fd/0x3f0 [ 449.814417] worker_thread+0x34/0x410 [ 449.814728] kthread+0x121/0x140

[

Quoted 41 lines
Quoted 41 lines
Quoted 41 lines
Quoted 41 lines
Quoted 41 lines
Quoted 41 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

449.815004] ? process_one_work+0x3f0/0x3f0 [ 449.815374] ?

kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <[email protected]>

On

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

Behalf

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer
issue for tdr

2 lines not shown · raw message

141 lines not shown · raw message

Quoted 44 lines

44 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

#38 lore raw
In reply to a message not in the archive

Note that kthread_park waits for kthread->parked to be signaled before proceeding - so in the scenario you described it meas main thread is running (not parked and so kthread->parked is not signaled) and so kthread_park will not proceed until the sched thread finish current loop (including removing any signaled jobs from ring_mirror_list) and is back to wait_event_interruptible->drm_sched_blocked->kthread_parkme->complete(&self->parked) to park itself - so looks to me it should be OK.

Andrey

Quoted 399 lines · Emily

Hi Andrey,
I don’t think your patch will help for this. As it will may call kthread_should_park in drm_sched_cleanup_jobs first, and then call kcl_kthread_park. And then it still has a race between the 2 threads.

Best wishes
Emily Deng

Quoted 391 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Saturday, November 9, 2019 3:01 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 10 lines · Christian

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

        /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

I don't think this protects us if drm_sched_cleanup_jobs is called for scheduler which didn't experience a timeout, in amdgpu_device_gpu_recover we access sched->ring_mirror_list for all the schedulers on a device so this condition above won't protect us. What in fact could help maybe is my recent patch 541c521 drm/sched: Avoid job cleanup if sched thread is parked. because we do park each of the scheduler threads during tdr job before trying to access sched->ring_mirror_list.

Emily - did you see this problem with that patch in place ? I only pushed it yesterday.

Andrey

Quoted 6 lines

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

        The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 59 lines
Quoted 59 lines

Best wishes
Emily Deng

Quoted 54 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even
it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

    drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6

[ 449.794366] BUG: unable to handle kernel NULL pointer

dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff

ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000

CR0:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000

[

Quoted 105 lines
Quoted 105 lines
Quoted 105 lines
Quoted 12 lines
Quoted 12 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in
one case, when it enter into the amdgpu_device_gpu_recover,
it already in drm_sched_job_cleanup, and at this time, it
will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
              drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
              amdgpu_ring_priority_put(ring, s_job->s_priority);
              dma_fence_put(job->fence);
              amdgpu_sync_free(&job->sync);
              amdgpu_sync_free(&job->sched_sync);
              kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11 ++++++-

----

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 25 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {
- if (bad->s_fence-

scheduled.context

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

==

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- entity->fence_context) {
+ if (bad->s_fence && (bad-

s_fence-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

*sched, struct drm_sched_job *bad)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

* This iteration is thread safe as sched thread
is

stopped.

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
- if (s_job->s_fence->parent &&
+ if (s_job->s_fence && s_job->s_fence->parent

&&

Quoted 36 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

395,7

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence refcount at

least 1

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ if (s_job->s_fence)
+ dma_fence_wait(&s_job-

s_fence-

Quoted 16 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);
/*
* We must keep bad job alive for later

use

Quoted 11 lines
Quoted 11 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

{
- struct dma_fence *fence = s_job->s_fence-

parent;

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

+ struct dma_fence *fence = s_job->s_fence ?

s_job-

Quoted 14 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#39 lore raw
In reply to a message not in the archive

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress. Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <[email protected]> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines
        /* Don't destroy jobs while the timeout worker is running */
        if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
            !cancel_delayed_work(&sched->work_tdr))
                return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 325 lines

Hi Christian,

      The drm_sched_job_timedout-> amdgpu_job_timedout call amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Best wishes
Emily Deng

Quoted 317 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 33 lines

Best wishes
Emily Deng

Quoted 28 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e

Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 3 lines
Quoted 3 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue occurs.

Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 21 lines
Quoted 21 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information: process

pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops: 0000 [#1] SMP PTI

Quoted 5 lines
Quoted 5 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 3 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [  449.803488]

RIP:

Quoted 4 lines
Quoted 4 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 2 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff

45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 26 lines
Quoted 26 lines

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 12 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000 CR0:
0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 31 lines
Quoted 31 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 57 lines
Quoted 57 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

When the job is already signaled, the s_fence is freed. Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
            drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
            amdgpu_ring_priority_put(ring, s_job->s_priority);
            dma_fence_put(job->fence);
            amdgpu_sync_free(&job->sync);
            amdgpu_sync_free(&job->sched_sync);
            kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 18 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Regards,
Christian.

Quoted 6 lines
---
       drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
       drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++-----
       2 files changed, 7 insertions(+), 6 deletions(-)
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c int

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index e6ce949..5a8f08e 100644
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1 -1
            *
            * job->base holds a reference to parent fence
            */
-  if (job && job->base.s_fence->parent &&
+  if (job && job->base.s_fence && job->base.s_fence->parent

&&

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
               dma_fence_is_signaled(job->base.s_fence->parent))
                   job_signaled = true;
 
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index 31809ca..56cc10e 100644
@@ -334,8 +334,8 @@ void drm_sched_increase_karma(struct

drm_sched_job

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/scheduler/sched_main.c void drm_sched_increase_karma(struct · line 337

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +0 -0
 *bad)
 
                           spin_lock(&rq->lock);
                           list_for_each_entry_safe(entity, tmp,

&rq-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +0 -1
-                          if (bad->s_fence->scheduled.context

==

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-                              entity->fence_context) {
+                          if (bad->s_fence && (bad->s_fence-

scheduled.context ==
+ entity->fence_context)) {
if (atomic_read(&bad-

Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
                                               bad->sched-

hang_limit)

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

if (entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*sched, struct drm_sched_job *bad)

Quoted 1 line

* This iteration is thread safe as sched thread is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp, &sched-
ring_mirror_list, node) {

drivers/gpu/drm/scheduler/sched_main.c +1 -1
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
                       dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
                                                 &s_job->cb)) {
                           atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

395,7

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+395,8 @@ void

Quoted 4 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 2 lines

*
* Job is still alive so fence refcount at

least 1

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
                            */

- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/scheduler/sched_main.c

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +2 -0
+                  if (s_job->s_fence)
+                          dma_fence_wait(&s_job->s_fence-

finished,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

false);

Quoted 2 lines

/*
* We must keep bad job alive for later

use

Quoted 24 lines
Quoted 24 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

Quoted 2 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -1
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ? s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

parent :
+NULL;

                   atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): MWHPR12MB1453817C6F05A57FD431E159EA770-Gy0DoCVfaSWZBIDmKHdw+wdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#41 lore raw

Hi Christian,
I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don't know exactly where it is wrong about the driver. Do you have any suggestion about this?

+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n", current->tgid, current->pid);

        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
            !cancel_delayed_work(&sched->work_tdr)))
                return;

+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n", current->tgid, current->pid);

Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 394 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines
        /* Don't destroy jobs while the timeout worker is running */
        if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
            !cancel_delayed_work(&sched->work_tdr))
                return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,

      The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
No, I am with the new branch and also has the patch. Even it
are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e

Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [ 449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines
[  449.794175] Emily:amdgpu_job_free_cb,Process information:

process pid 0 thread pid 0, s_job:000000005086879e [ 449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [ 449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [ 449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [ 449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [ 449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines
[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff

ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00 00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [

449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [ 449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [ 449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS: 0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [ 449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines
[  449.812206]  amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635]  drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077]  process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799]  ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.

NAK, the s_fence is only set to NULL when the job is destroyed. See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
            drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
            amdgpu_ring_priority_put(ring, s_job->s_priority);
            dma_fence_put(job->fence);
            amdgpu_sync_free(&job->sync);
            amdgpu_sync_free(&job->sched_sync);
            kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Quoted 4 lines

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +-
drivers/gpu/drm/scheduler/sched_main.c | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,

Quoted 6 lines

*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+ job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 79 lines
Quoted 79 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job

Quoted 5 lines

*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {

Quoted 1 line

- if (bad->s_fence->scheduled.context

==

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

- entity->fence_context) {
+ if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

*sched, struct drm_sched_job *bad)

Quoted 2 lines

* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
- if (s_job->s_fence->parent &&
+ if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

+395,8 @@ void

Quoted 5 lines
Quoted 1 line

drm_sched_stop(struct drm_gpu_scheduler

*sched, struct drm_sched_job *bad)

Quoted 3 lines

*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-

finished,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

false);

Quoted 3 lines

/*
* We must keep bad job alive
for later

use

Quoted 19 lines
Quoted 19 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7

Quoted 1 line

+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 1 line

+bool

full_recovery)

Quoted 5 lines

* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)

{

Quoted 3 lines

- struct dma_fence *fence = s_job->s_fence->parent;
+ struct dma_fence *fence = s_job->s_fence ?
+ s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent :
+NULL;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB29750EDB35E27DF9CD63152C8F770-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#43 lore raw

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 420 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+       printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 387 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread  pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread  pid 0, s_job:000000005086879e [  449.794221] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:0000000066eb74ab [  449.794222] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000d4438ad9 [  449.794255] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000b6d69c65 [  449.794257] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process  pid 0 thread pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [  449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]

Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [ 449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [  449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [  449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [  449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [  449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [ 449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [ 449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]

worker_thread+0x34/0x410 [ 449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd- [email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Quoted 9 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().

I know it is set to NULL in drm_sched_job_cleanup. But in one case, when it enter into the amdgpu_device_gpu_recover, it already in drm_sched_job_cleanup, and at this time, it will go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
           drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
           amdgpu_ring_priority_put(ring, s_job->s_priority);
           dma_fence_put(job->fence);
           amdgpu_sync_free(&job->sync);
           amdgpu_sync_free(&job->sched_sync);
           kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+ job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 79 lines
Quoted 79 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {
-                          if (bad->s_fence->scheduled.context

==

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

-                              entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+                              entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

+395,8 @@ void

Quoted 5 lines

drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-

finished,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

false);
/*
* We must keep bad job alive
for later

use

Quoted 19 lines
Quoted 19 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 15 lines
Quoted 15 lines
Quoted 15 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent :
+NULL;

Quoted 12 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 2f035f22-4057-dd9e-27ef-0f5612113e29-5C7GfCeVMHo@public.gmane.org

#45 lore raw

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 443 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 432 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+       printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 301 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread  pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread  pid 0, s_job:000000005086879e [  449.794221] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:0000000066eb74ab [  449.794222] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000d4438ad9 [  449.794255] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000b6d69c65 [  449.794257] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [  449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]

Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [ 449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [  449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [ 449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [ 449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]

worker_thread+0x34/0x410 [ 449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

Quoted 2 lines

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in one
case, when it enter into the amdgpu_device_gpu_recover, it
already in drm_sched_job_cleanup, and at this time, it will
go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines
           drm_sched_job_cleanup(s_job);
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
           amdgpu_ring_priority_put(ring, s_job->s_priority);
           dma_fence_put(job->fence);
           amdgpu_sync_free(&job->sync);
           amdgpu_sync_free(&job->sched_sync);
           kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines
Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
- if (job && job->base.s_fence->parent &&
+ if (job && job->base.s_fence &&
+ job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 92 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 85 lines
Quoted 85 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

list) {
-                          if (bad->s_fence->scheduled.context

==

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

-                              entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+                              entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >

Quoted 1 line
Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-

finished,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

false);
/*
* We must keep bad job alive
for later

use

Quoted 25 lines
Quoted 25 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 21 lines
Quoted 21 lines
Quoted 21 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

@@ -647,14 +653,28 @@ static void drm_sched_cleanup_jobs(struct drm_gpu_scheduler *sched)

 {
        unsigned long flags;

+

38 lines not shown · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -0
@@ -4007,6 +4007,8 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
                device_list_handle = &device_list;
        }
 
+       DRM_ERROR("Waiting for bad job to finish %p", &job->base);
+       dma_fence_wait(&job->base.s_fence->finished, false);
        /* block all schedulers and reset given job's ring */
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

69 lines not shown · raw message

Parts of this message are not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): 9269d447-ed32-81f7-ab43-cb16139096e2-5C7GfCeVMHo@public.gmane.org

#47 lore raw

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 453 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 443 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 432 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+       printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 301 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread  pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread  pid 0, s_job:000000005086879e [  449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [  449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [  449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [  449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [  449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]

Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [ 449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [  449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [ 449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [ 449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]

worker_thread+0x34/0x410 [ 449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:

Quoted 1 line

Ping.....

You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in one
case, when it enter into the amdgpu_device_gpu_recover, it
already in drm_sched_job_cleanup, and at this time, it will
go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.

No, that case can't happen. See here:

Quoted 2 lines

drm_sched_job_cleanup(s_job);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

amdgpu_ring_priority_put(ring, s_job->s_priority);
dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
kfree(job);

The job itself is freed up directly after freeing the reference to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This patch is a clear NAK.

Regards,
Christian.

Quoted 3 lines

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
-  if (job && job->base.s_fence->parent &&
+  if (job && job->base.s_fence &&
+ job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 92 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 85 lines
Quoted 85 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 3 lines

entities,

Quoted 2 lines
Quoted 2 lines

list) {
-                          if (bad->s_fence->scheduled.context

==

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

-                              entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >

Quoted 1 line
Quoted 1 line

bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines
Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines

+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-

finished,

Quoted 4 lines
Quoted 4 lines

false);
/*
* We must keep bad job alive
for later

use

Quoted 25 lines
Quoted 25 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 21 lines
Quoted 21 lines
Quoted 21 lines
Quoted 11 lines
Quoted 11 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-

s_fence-

Quoted 9 lines
Quoted 9 lines

parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 33ffe2f1-32b6-a238-3752-cee67cd9e141-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#49 lore raw

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 464 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 457 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 443 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 432 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 301 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread  pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread  pid 0, s_job:000000005086879e [  449.794221] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:0000000066eb74ab [  449.794222] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000d4438ad9 [  449.794255] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0, s_job:00000000b6d69c65 [  449.794257] Emily:amdgpu_job_free_cb,Process information: process pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]

Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [ 449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [ 449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [ 449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [ 449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]

worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for
tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in one
case, when it enter into the amdgpu_device_gpu_recover, it
already in drm_sched_job_cleanup, and at this time, it will
go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.
No, that case can't happen. See here:

drm_sched_job_cleanup(s_job);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_ring_priority_put(ring, s_job->s_priority);
dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
kfree(job);
The job itself is freed up directly after freeing the reference
to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This
patch is a clear NAK.

Regards,
Christian.

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
-  if (job && job->base.s_fence->parent &&
+  if (job && job->base.s_fence &&
+ job->base.s_fence->parent

&&

Quoted 3 lines
Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 92 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 85 lines
Quoted 85 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 3 lines

entities,

Quoted 2 lines

list) {
-                          if (bad->s_fence->scheduled.context

==

Quoted 7 lines
Quoted 7 lines

- entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >

Quoted 1 line

bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines
Quoted 3 lines

*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,

Quoted 2 lines

&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7

Quoted 6 lines
Quoted 6 lines

+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 7 lines
Quoted 2 lines

+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-

finished,

Quoted 4 lines

false);
/*
* We must keep bad job alive
for later

use

Quoted 25 lines
Quoted 25 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines
Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 21 lines
Quoted 21 lines
Quoted 21 lines
Quoted 11 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-

s_fence-

Quoted 9 lines

parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 40bb3114-d996-10af-3140-51a4f7c212d6-5C7GfCeVMHo@public.gmane.org

#51 lore raw

Another more fundamental question: Could we get rid of the timeout job at all?

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 470 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 464 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 457 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 443 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 432 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 301 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread  pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:000000005086879e [  449.794221] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:0000000066eb74ab [  449.794222] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000d4438ad9 [  449.794255] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000b6d69c65 [  449.794257] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]

Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [ 449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [  449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [ 449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [ 449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]

worker_thread+0x34/0x410 [  449.814728]  kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:

Quoted 1 line

Sorry, please take your time.

Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Quoted 5 lines

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for
tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in one
case, when it enter into the amdgpu_device_gpu_recover, it
already in drm_sched_job_cleanup, and at this time, it will
go to free

job.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.
No, that case can't happen. See here:

drm_sched_job_cleanup(s_job);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

amdgpu_ring_priority_put(ring, s_job->s_priority);
dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
kfree(job);
The job itself is freed up directly after freeing the reference
to the

s_fence.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This
patch is a clear NAK.

Regards,
Christian.

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
-  if (job && job->base.s_fence->parent &&
+  if (job && job->base.s_fence &&
+ job->base.s_fence->parent

&&

Quoted 3 lines

dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 92 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 85 lines
Quoted 85 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 3 lines

entities,
list) {
-                          if (bad->s_fence->scheduled.context

==

Quoted 7 lines

- entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >
bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line

drm_sched_stop(struct

drm_gpu_scheduler

Quoted 3 lines

*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,
&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7

Quoted 6 lines

+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 2 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,

false);

Quoted 7 lines

+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-
finished,
false);
/*
* We must keep bad job alive
for later

use

Quoted 25 lines
Quoted 25 lines
Quoted 3 lines
Quoted 3 lines

during @@

Quoted 2 lines

-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 21 lines
Quoted 21 lines
Quoted 21 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-
s_fence-
parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 0858ea1b-d205-006d-a6ec-24b78b33e45b-5C7GfCeVMHo@public.gmane.org

#53 lore raw
Quoted 1 line · Christian König

Another more fundamental question: Could we get rid of the timeout job at all?

There are other stuff there besides picking the first unfinished job which is common for all the drivers - such as freeing guilty signaled job and rearming the timeout work timer.

Quoted 8 lines

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Should be ok since we have the extra check for __kthread_should_park in drm_sched_cleanup_jobs which will protect us in this case from a wakeup of sched thread and execution of in drm_sched_cleanup_jobs after we already parked it. The problem here is we need the drm_sched_job to access the private data for each client driver (see amdgpu_job_timedout for example). What about instead of peeking at the job to actually remove it from ring_mirror_list right there, go ahead with it through the reset routine, if it's signaled in the meanwhile that great - release it, otherwise put it back into ring_mirror_list in drm_sched_resubmit_jobs.

Andrey

Quoted 476 lines

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 470 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 464 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 457 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 443 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 432 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 301 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]> Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread  pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread pid 0, s_job:000000005086879e [  449.794221] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:0000000066eb74ab [  449.794222] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000d4438ad9 [  449.794255] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0, s_job:00000000b6d69c65 [  449.794257] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread pid 0,

s_job:00000000ea85e922 [ 449.794287] Emily:amdgpu_job_free_cb,Process
information: process  pid 0 thread  pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G           OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]

Workqueue: events drm_sched_job_timedout [amd_sched] [

449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [ 449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [ 449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000) knlGS:0000000000000000 [  449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [ 449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [ 449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [ 449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]

worker_thread+0x34/0x410 [  449.814728] kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 20 lines
Quoted 20 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for
tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:
Sorry, please take your time.
Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be
NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Best wishes
Emily Deng

Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines
Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for
tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 16 lines
Quoted 16 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:

Quoted 8 lines
Quoted 8 lines

When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in one
case, when it enter into the amdgpu_device_gpu_recover, it
already in drm_sched_job_cleanup, and at this time, it will
go to free

job.

Quoted 6 lines
Quoted 6 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.
No, that case can't happen. See here:

drm_sched_job_cleanup(s_job);

Quoted 7 lines
Quoted 7 lines

amdgpu_ring_priority_put(ring, s_job->s_priority);
dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
kfree(job);
The job itself is freed up directly after freeing the reference
to the

s_fence.

Quoted 9 lines
Quoted 9 lines

So you are just papering over a much bigger problem here. This
patch is a clear NAK.

Regards,
Christian.

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
-  if (job && job->base.s_fence->parent &&
+  if (job && job->base.s_fence &&
+ job->base.s_fence->parent
&&
dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 92 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 85 lines
Quoted 85 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

entities,
list) {
-                          if (bad->s_fence->scheduled.context
==
- entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

karma) >
bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drm_sched_stop(struct
drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

parent,
&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7
+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,
false);
+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-
finished,
false);
/*
* We must keep bad job alive
for later

use

Quoted 25 lines
Quoted 25 lines
Quoted 3 lines
Quoted 3 lines

during @@
-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 21 lines
Quoted 21 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-
s_fence-
parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): c784ef0a-2cd7-d4b1-0581-356d8c401102-5C7GfCeVMHo@public.gmane.org

#55 lore raw
Quoted 1 line

What about instead of peeking at the job to actually remove it from ring_mirror_list right there,

Also an interesting idea. We would need to protect the mirror list with a lock again, but that should be the lesser evil.

Maybe prototype that and see if it works or not.

Regards,
Christian.

Am 13.11.19 um 17:00 schrieb Andrey Grodzovsky:

Quoted 501 lines · Christian König
Quoted 1 line

Another more fundamental question: Could we get rid of the timeout job at all?

There are other stuff there besides picking the first unfinished job which is common for all the drivers - such as freeing guilty signaled job and rearming the timeout work timer.

Quoted 8 lines

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Should be ok since we have the extra check for __kthread_should_park in drm_sched_cleanup_jobs which will protect us in this case from a wakeup of sched thread and execution of in drm_sched_cleanup_jobs after we already parked it. The problem here is we need the drm_sched_job to access the private data for each client driver (see amdgpu_job_timedout for example). What about instead of peeking at the job to actually remove it from ring_mirror_list right there, go ahead with it through the reset routine, if it's signaled in the meanwhile that great - release it, otherwise put it back into ring_mirror_list in drm_sched_resubmit_jobs.

Andrey

Quoted 476 lines

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 470 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 464 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 457 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 443 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 432 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu,
pid:%lu\n", current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process pid 0 thread  pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 301 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Thinking more about this claim - we assume here that if cancel_delayed_work returned true it guarantees that timeout work is not running but, it merely means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a chance to be dequeued and executed, it doesn't cover already executing work. So there is a possibility where while timeout work started executing another timeout work already got enqueued (maybe through earlier cleanup jobs or through drm_sched_fault) and if at this point another drm_sched_cleanup_jobs runs cancel_delayed_work(&sched->work_tdr) will return true even while there is a timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout work itself waits for schedule thread  to be parked again when calling park_thread.

Andrey

________________________________________
From: amd-gfx <[email protected]> on behalf of Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

exactly that can't happen. See here:

Quoted 4 lines

         /* Don't destroy jobs while the timeout worker is running */ if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&

           !cancel_delayed_work(&sched->work_tdr))

                 return NULL;

We never free jobs while the timeout working is running to prevent exactly that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the jobs while in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 36 lines

Best wishes
Emily Deng

Quoted 30 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job in the first place.

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 3 lines

Hi Chrisitan,
        No, I am with the new branch and also has the patch. Even it are freed by

main scheduler, how we could avoid main scheduler to free jobs while enter to function amdgpu_device_gpu_recover?

Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only if no timeout handler is running.

See this patch here:

Quoted 5 lines

commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

Hi Christian,
         Please refer to follow log, when it enter to
amdgpu_device_gpu_recover

function, the bad job 000000005086879e is freeing in function amdgpu_job_free_cb  at the same time, because of the hardware fence

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the s_fence is already freed, but job is not freed in time. Then this issue

occurs.

Quoted 39 lines
Quoted 39 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring

sdma0

Quoted 22 lines
Quoted 22 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202]
[drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information:

process  pid 0 thread pid 0, s_job:000000005086879e [  449.794163] amdgpu
0000:00:08.0: GPU reset begin!

Quoted 10 lines

[  449.794175] Emily:amdgpu_job_free_cb,Process information:
process pid 0 thread  pid 0, s_job:000000005086879e [ 449.794221]
Emily:amdgpu_job_free_cb,Process information: process  pid 0
thread pid 0, s_job:0000000066eb74ab [  449.794222]
Emily:amdgpu_job_free_cb,Process information: process  pid 0
thread pid 0, s_job:00000000d4438ad9 [  449.794255]
Emily:amdgpu_job_free_cb,Process information: process  pid 0
thread pid 0, s_job:00000000b6d69c65 [  449.794257]
Emily:amdgpu_job_free_cb,Process information: process  pid 0
thread pid 0,

s_job:00000000ea85e922 [  449.794287] Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread  pid 0, s_job:00000000ed3a5ac6 [ 449.794366] BUG: unable to handle kernel NULL pointer dereference at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [  449.801040] Oops:
0000 [#1] SMP PTI

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted: G OE

4.18.0-15-generic #16~18.04.1-Ubuntu

Quoted 4 lines

[  449.802157] Hardware name: QEMU Standard PC (i440FX + PIIX,
1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [
449.803488]

RIP:

Quoted 6 lines
Quoted 6 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]

Quoted 3 lines

[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85 56 ff ff
ff
45 85 e4 0f

85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48 8b 40 10 <48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43 48 a8 01

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS: 00010286 [
449.806032] RAX: 0000000000000000 RBX: 0000000000000000 RCX:
0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09: 0000000000000000 [
449.807818] R10: 0000000000000000 R11: 0000000000000148 R12:
0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000)
knlGS:0000000000000000 [  449.809674] CS:  0010 DS: 0000 ES: 0000

CR0:

Quoted 108 lines
Quoted 108 lines
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:

0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:

Quoted 10 lines

[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10 [amdgpu] [
449.813609]  ? drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [  449.814728] kthread+0x121/0x140 [
449.815004]  ? process_one_work+0x3f0/0x3f0 [  449.815374]  ?
kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 32 lines
Quoted 32 lines
Quoted 24 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for
tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:
Sorry, please take your time.
Have you seen my other response a bit below?

I can't follow how it would be possible for job->s_fence to be
NULL without the job also being freed.

So it looks like this patch is just papering over some bigger issues.

Regards,
Christian.

Best wishes
Emily Deng

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 7 lines

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for
tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx <[email protected]> On

Behalf

Quoted 48 lines
Quoted 48 lines
Quoted 24 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

-----Original Message-----
From: Christian König <[email protected]>
Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue
for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:
When the job is already signaled, the s_fence is freed.
Then it will has null pointer in amdgpu_device_gpu_recover.
NAK, the s_fence is only set to NULL when the job is destroyed.
See drm_sched_job_cleanup().
I know it is set to NULL in drm_sched_job_cleanup. But in one
case, when it enter into the amdgpu_device_gpu_recover, it
already in drm_sched_job_cleanup, and at this time, it will
go to free

job.

Quoted 13 lines

But the amdgpu_device_gpu_recover sometimes is faster. At
that time, job is not freed, but s_fence is already NULL.
No, that case can't happen. See here:

drm_sched_job_cleanup(s_job);

amdgpu_ring_priority_put(ring, s_job->s_priority);
dma_fence_put(job->fence);
amdgpu_sync_free(&job->sync);
amdgpu_sync_free(&job->sched_sync);
kfree(job);
The job itself is freed up directly after freeing the reference
to the

s_fence.

Quoted 9 lines

So you are just papering over a much bigger problem here. This
patch is a clear NAK.

Regards,
Christian.

When you see a job without an s_fence then that means the
problem is somewhere else.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Signed-off-by: Emily Deng <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  2 +-
drivers/gpu/drm/scheduler/sched_main.c     | 11 ++++++---

--

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e6ce949..5a8f08e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4075,7 +4075,7 @@ int

amdgpu_device_gpu_recover(struct

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

amdgpu_device *adev,
*
* job->base holds a reference to parent fence
*/
-  if (job && job->base.s_fence->parent &&
+  if (job && job->base.s_fence &&
+ job->base.s_fence->parent
&&
dma_fence_is_signaled(job->base.s_fence->parent))
job_signaled = true;

Quoted 92 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

diff --git a/drivers/gpu/drm/scheduler/sched_main.c
b/drivers/gpu/drm/scheduler/sched_main.c
index 31809ca..56cc10e 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -334,8 +334,8 @@ void

drm_sched_increase_karma(struct

Quoted 85 lines
Quoted 85 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

drm_sched_job
*bad)

spin_lock(&rq->lock);
list_for_each_entry_safe(entity,
tmp,

&rq-

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

entities,
list) {
-                          if (bad->s_fence->scheduled.context
==
- entity->fence_context) {
+                          if (bad->s_fence &&
+ (bad->s_fence-
scheduled.context ==
+ entity->fence_context)) {
if
(atomic_read(&bad-

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines

karma) >
bad->sched-

hang_limit)

Quoted 2 lines
Quoted 2 lines

if
(entity-

guilty) @@ -376,7 +376,7 @@ void

Quoted 5 lines
Quoted 5 lines

drm_sched_stop(struct
drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
* This iteration is thread safe as sched thread
is

stopped.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines

*/
list_for_each_entry_safe_reverse(s_job, tmp,
&sched- ring_mirror_list, node) {
-          if (s_job->s_fence->parent &&
+          if (s_job->s_fence && s_job->s_fence->parent &&
dma_fence_remove_callback(s_job-

s_fence-

Quoted 3 lines
Quoted 3 lines

parent,
&s_job->cb)) {
atomic_dec(&sched->hw_rq_count);

@@ -

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

395,7
+395,8 @@ void
drm_sched_stop(struct drm_gpu_scheduler
*sched, struct drm_sched_job *bad)
*
* Job is still alive so fence
refcount at

least 1

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

*/
- dma_fence_wait(&s_job->s_fence->finished,
false);
+                  if (s_job->s_fence)
+ dma_fence_wait(&s_job->s_fence-
finished,
false);
/*
* We must keep bad job alive
for later

use

Quoted 25 lines
Quoted 25 lines
Quoted 3 lines

during @@
-438,7
+439,7 @@ void drm_sched_start(struct drm_gpu_scheduler

*sched,

Quoted 21 lines

+bool
full_recovery)
* GPU recovers can't run in parallel.
*/
list_for_each_entry_safe(s_job, tmp,
&sched->ring_mirror_list,
node)
{
-          struct dma_fence *fence = s_job->s_fence->parent;
+          struct dma_fence *fence = s_job->s_fence ?
+ s_job-
s_fence-
parent :
+NULL;

atomic_inc(&sched->hw_rq_count);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>

Quoted 5 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 088fb2bc-b401-17cc-4d7c-001705ee1eb9-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#57 lore raw

ok

Andrey

Quoted 597 lines · Christian König
Quoted 1 line

What about instead of peeking at the job to actually remove it from ring_mirror_list right there,

Also an interesting idea. We would need to protect the mirror list with a lock again, but that should be the lesser evil.

Maybe prototype that and see if it works or not.

Regards,
Christian.

Am 13.11.19 um 17:00 schrieb Andrey Grodzovsky:

Quoted 587 lines · Christian König
Quoted 1 line

Another more fundamental question: Could we get rid of the timeout job at all?

There are other stuff there besides picking the first unfinished job which is common for all the drivers - such as freeing guilty signaled job and rearming the timeout work timer.

Quoted 8 lines

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Should be ok since we have the extra check for __kthread_should_park in drm_sched_cleanup_jobs which will protect us in this case from a wakeup of sched thread and execution of in drm_sched_cleanup_jobs after we already parked it. The problem here is we need the drm_sched_job to access the private data for each client driver (see amdgpu_job_timedout for example). What about instead of peeking at the job to actually remove it from ring_mirror_list right there, go ahead with it through the reset routine, if it's signaled in the meanwhile that great - release it, otherwise put it back into ring_mirror_list in drm_sched_resubmit_jobs.

Andrey

Quoted 558 lines

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 552 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 546 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 539 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 525 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 514 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu,
pid:%lu\n", current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process pid 0 thread pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 6 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 2 lines

Thinking more about this claim - we assume here that if

cancel_delayed_work

Quoted 1 line

returned true it guarantees that timeout work is not running

but, it merely

Quoted 2 lines

means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a

chance to be

Quoted 1 line

dequeued and executed, it doesn't cover already executing

work. So there is a

Quoted 1 line

possibility where while timeout work started executing another

timeout work

Quoted 1 line

already got enqueued (maybe through earlier cleanup jobs or

through

Quoted 1 line

drm_sched_fault) and if at this point another

drm_sched_cleanup_jobs runs

Quoted 1 line

cancel_delayed_work(&sched->work_tdr) will return true even

while there is a

Quoted 3 lines

timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout

work itself

Quoted 1 line

waits for schedule thread  to be parked again when calling

park_thread.

Quoted 5 lines

Andrey

________________________________________
From: amd-gfx <[email protected]> on

behalf of

Quoted 4 lines

Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 6 lines

Hi Emily,

exactly that can't happen. See here:

Quoted 1 line

         /* Don't destroy jobs while the timeout worker is

running */

Quoted 5 lines
Quoted 3 lines

         if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to

prevent exactly

Quoted 9 lines

that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the

jobs while

Quoted 11 lines

in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 10 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>;

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job

in the first place.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
        No, I am with the new branch and also has the

patch. Even it

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line

are freed by

main scheduler, how we could avoid main scheduler to free

jobs while

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

enter to function amdgpu_device_gpu_recover?

Quoted 5 lines

Best wishes
Emily Deng

Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for tdr

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only

if no

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

timeout handler is running.

See this patch here:
commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:
Hi Christian,
         Please refer to follow log, when it enter to
amdgpu_device_gpu_recover
function, the bad job 000000005086879e is freeing in

function

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

amdgpu_job_free_cb  at the same time, because of the

hardware fence

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the
s_fence is already freed, but job is not freed in time.

Then this issue

Quoted 2 lines

occurs.

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]]

*ERROR* ring

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

sdma0

Quoted 2 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202]
[drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

process  pid 0 thread pid 0, s_job:000000005086879e [

449.794163]

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

amdgpu
0000:00:08.0: GPU reset begin!
[  449.794175] Emily:amdgpu_job_free_cb,Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

process pid 0 thread  pid 0, s_job:000000005086879e [

449.794221]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:0000000066eb74ab [  449.794222]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:00000000d4438ad9 [  449.794255]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:00000000b6d69c65 [  449.794257]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

thread pid 0,
s_job:00000000ea85e922 [ 449.794287]
Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread  pid 0,

s_job:00000000ed3a5ac6 [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.794366] BUG: unable to handle kernel NULL pointer

dereference

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [  449.801040]

Oops:

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

0000 [#1] SMP PTI
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted:

G OE

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

4.18.0-15-generic #16~18.04.1-Ubuntu
[  449.802157] Hardware name: QEMU Standard PC (i440FX +

PIIX,

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines

1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [
449.803488]

RIP:

Quoted 2 lines
Quoted 2 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85

56 ff ff

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

ff
45 85 e4 0f
85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48

8b 40 10

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

<48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43

48 a8 01

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS:

00010286 [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.806032] RAX: 0000000000000000 RBX: 0000000000000000

RCX:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09:

0000000000000000 [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.807818] R10: 0000000000000000 R11: 0000000000000148

R12:

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000)
knlGS:0000000000000000 [ 449.809674] CS:  0010 DS: 0000

ES: 0000

Quoted 4 lines

CR0:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:
0000000000000000 DR1: 0000000000000000 DR2:

0000000000000000 [

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:
[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10

[amdgpu] [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.813609]  ? drm_sched_job_timedout+0x44/0x90

[amd_sched] [

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [ 449.814728]

kthread+0x121/0x140 [
>>>>>> 449.815004]  ? process_one_work+0x3f0/0x3f0 [  

449.815374]  ?

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 7 lines
Quoted 7 lines

kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:
Sorry, please take your time.
Have you seen my other response a bit below?

I can't follow how it would be possible for

job->s_fence to be

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

NULL without the job also being freed.

So it looks like this patch is just papering over some

bigger issues.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

Regards,
Christian.

Best wishes
Emily Deng

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx

<[email protected]> On

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Behalf

Quoted 5 lines
Quoted 5 lines

Of Deng, Emily
Sent: Friday, November 8, 2019 10:56 AM
To: Koenig, Christian <[email protected]>; amd-
[email protected]
Subject: RE: [PATCH] drm/amdgpu: Fix the null

pointer issue

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

for tdr

-----Original Message-----
From: Christian König

<[email protected]>

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

Sent: Thursday, November 7, 2019 7:28 PM
To: Deng, Emily <[email protected]>;
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null

pointer issue

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

for tdr

Am 07.11.19 um 11:25 schrieb Emily Deng:
When the job is already signaled, the s_fence is

freed.

177 lines not shown · raw message

4 lines not shown · raw message

4 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

#58 lore raw
In reply to a message not in the archive

Attached.

Emily - can you give it a try ?

Andrey

Quoted 592 lines · Christian König
Quoted 1 line

What about instead of peeking at the job to actually remove it from ring_mirror_list right there,

Also an interesting idea. We would need to protect the mirror list with a lock again, but that should be the lesser evil.

Maybe prototype that and see if it works or not.

Regards,
Christian.

Am 13.11.19 um 17:00 schrieb Andrey Grodzovsky:

Quoted 583 lines · Christian König
Quoted 1 line

Another more fundamental question: Could we get rid of the timeout job at all?

There are other stuff there besides picking the first unfinished job which is common for all the drivers - such as freeing guilty signaled job and rearming the timeout work timer.

Quoted 8 lines

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Should be ok since we have the extra check for __kthread_should_park in drm_sched_cleanup_jobs which will protect us in this case from a wakeup of sched thread and execution of in drm_sched_cleanup_jobs after we already parked it. The problem here is we need the drm_sched_job to access the private data for each client driver (see amdgpu_job_timedout for example). What about instead of peeking at the job to actually remove it from ring_mirror_list right there, go ahead with it through the reset routine, if it's signaled in the meanwhile that great - release it, otherwise put it back into ring_mirror_list in drm_sched_resubmit_jobs.

Andrey

Quoted 558 lines

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 552 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 546 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 539 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously  this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 525 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 514 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu,
pid:%lu\n", current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n",
current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring sdma0 timeout, signaled seq=78585, emitted seq=78587
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process pid 0 thread pid 0, s_job:00000000fe75ab36,tid=15603, pid=15603
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225428] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253, pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel: [11381.225494] Emily:amdgpu_job_free_cb,Process information: process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262
Quoted 6 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 2 lines

Thinking more about this claim - we assume here that if

cancel_delayed_work

Quoted 1 line

returned true it guarantees that timeout work is not running

but, it merely

Quoted 2 lines

means there was a pending timeout work which was removed from the workqueue before it's timer elapsed and so it didn't have a

chance to be

Quoted 1 line

dequeued and executed, it doesn't cover already executing

work. So there is a

Quoted 1 line

possibility where while timeout work started executing another

timeout work

Quoted 1 line

already got enqueued (maybe through earlier cleanup jobs or

through

Quoted 1 line

drm_sched_fault) and if at this point another

drm_sched_cleanup_jobs runs

Quoted 1 line

cancel_delayed_work(&sched->work_tdr) will return true even

while there is a

Quoted 3 lines

timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout

work itself

Quoted 1 line

waits for schedule thread  to be parked again when calling

park_thread.

Quoted 5 lines

Andrey

________________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on

behalf of

Quoted 4 lines

Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 6 lines

Hi Emily,

exactly that can't happen. See here:

Quoted 1 line

         /* Don't destroy jobs while the timeout worker is

running */

Quoted 5 lines
Quoted 3 lines

         if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to

prevent exactly

Quoted 9 lines

that issue.

Regards,
Christian.

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the

jobs while

Quoted 11 lines

in amdgpu_device_gpu_recover, and before calling drm_sched_stop.

Quoted 10 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>;

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job

in the first place.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
        No, I am with the new branch and also has the

patch. Even it

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line

are freed by

main scheduler, how we could avoid main scheduler to free

jobs while

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

enter to function amdgpu_device_gpu_recover?

Quoted 5 lines

Best wishes
Emily Deng

Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for tdr

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only

if no

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

timeout handler is running.

See this patch here:
commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:
Hi Christian,
         Please refer to follow log, when it enter to
amdgpu_device_gpu_recover
function, the bad job 000000005086879e is freeing in

function

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

amdgpu_job_free_cb  at the same time, because of the

hardware fence

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

signal.

Quoted 2 lines
Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case, the
s_fence is already freed, but job is not freed in time.

Then this issue

Quoted 2 lines

occurs.

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]]

*ERROR* ring

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

sdma0

Quoted 2 lines
Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [  449.793202]
[drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

process  pid 0 thread pid 0, s_job:000000005086879e [

449.794163]

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

amdgpu
0000:00:08.0: GPU reset begin!
[  449.794175] Emily:amdgpu_job_free_cb,Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

process pid 0 thread  pid 0, s_job:000000005086879e [

449.794221]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:0000000066eb74ab [  449.794222]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:00000000d4438ad9 [  449.794255]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:00000000b6d69c65 [  449.794257]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

thread pid 0,
s_job:00000000ea85e922 [ 449.794287]
Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread  pid 0,

s_job:00000000ed3a5ac6 [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.794366] BUG: unable to handle kernel NULL pointer

dereference

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

at
00000000000000c0 [ 449.800818] PGD 0 P4D 0 [  449.801040]

Oops:

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

0000 [#1] SMP PTI
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted:

G OE

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

4.18.0-15-generic #16~18.04.1-Ubuntu
[  449.802157] Hardware name: QEMU Standard PC (i440FX +

PIIX,

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines

1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [  449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched] [
449.803488]

RIP:

Quoted 2 lines
Quoted 2 lines

0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85

56 ff ff

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

ff
45 85 e4 0f
85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48

8b 40 10

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line

<48> 8b

98

Quoted 1 line
Quoted 1 line

c0 00         00 00 48 85 db 0f 84 4c 01 00 00 48 8b 43

48 a8 01

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.805593] RSP: 0018:ffffb4c7c08f7d68 EFLAGS:

00010286 [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.806032] RAX: 0000000000000000 RBX: 0000000000000000

RCX:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

0000000000000000 [ 449.806625] RDX: ffffb4c7c08f5ac0 RSI:
0000000fffffffe0 RDI: 0000000000000246 [  449.807224] RBP:
ffffb4c7c08f7de0 R08: 00000068b9d54000 R09:

0000000000000000 [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.807818] R10: 0000000000000000 R11: 0000000000000148

R12:

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

0000000000000000 [ 449.808411] R13: ffffb4c7c08f7da0 R14:
ffff8d82b8525d40 R15: ffff8d82b8525d40 [  449.809004] FS:
0000000000000000(0000) GS:ffff8d82bfd80000(0000)
knlGS:0000000000000000 [ 449.809674] CS:  0010 DS: 0000

ES: 0000

Quoted 4 lines

CR0:

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

0000000080050033 [ 449.810153] CR2: 00000000000000c0 CR3:
000000003cc0a001 CR4: 00000000003606e0 [  449.810747] DR0:
0000000000000000 DR1: 0000000000000000 DR2:

0000000000000000 [

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

449.811344] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7:
0000000000000400 [ 449.811937] Call Trace:
[  449.812206] amdgpu_job_timedout+0x114/0x140 [amdgpu] [
449.812635] drm_sched_job_timedout+0x44/0x90 [amd_sched] [
449.813139]  ? amdgpu_cgs_destroy_device+0x10/0x10

[amdgpu] [

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

449.813609]  ? drm_sched_job_timedout+0x44/0x90

[amd_sched] [

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

449.814077] process_one_work+0x1fd/0x3f0 [  449.814417]
worker_thread+0x34/0x410 [ 449.814728]

kthread+0x121/0x140 [
>>>>>> 449.815004]  ? process_one_work+0x3f0/0x3f0 [  

449.815374]  ?

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 7 lines
Quoted 7 lines

kthread_create_worker_on_cpu+0x70/0x70
[  449.815799] ret_from_fork+0x35/0x40

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:43 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

tdr

Am 08.11.19 um 10:39 schrieb Deng, Emily:
Sorry, please take your time.
Have you seen my other response a bit below?

I can't follow how it would be possible for

job->s_fence to be

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

NULL without the job also being freed.

So it looks like this patch is just papering over some

bigger issues.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

Regards,
Christian.

Best wishes
Emily Deng

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 5:08 PM
To: Deng, Emily <[email protected]>; amd-
[email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

tdr

Am 08.11.19 um 09:52 schrieb Deng, Emily:
Ping.....
You need to give me at least enough time to wake up :)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Best wishes
Emily Deng

-----Original Message-----
From: amd-gfx

<amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> On

199 lines not shown · raw message

4 lines not shown · raw message

Quoted 6 lines

5 lines not shown · raw message

17 lines not shown · raw message

drivers/gpu/drm/scheduler/sched_main.c +30 -1
index 80ddbdf..c2a6108 100644
@@ -287,10 +287,24 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	unsigned long flags;
 
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
+
+	/*
+	 * Protects against concurrent deletion in drm_sched_cleanup_jobs that
+	 * is already in progress.
+	 */
+	spin_lock_irqsave(&sched->job_list_lock, flags);
 	job = list_first_entry_or_null(&sched->ring_mirror_list,
 				       struct drm_sched_job, node);
 
 	if (job) {
+		/*
+		 * Remove the bad job so it cannot be freed by already in progress
+		 * drm_sched_cleanup_jobs. It will be reinsrted back after sched->thread
+		 * is parked at which point it's safe.
+		 */
+		list_del_init(&job->node);
+		spin_unlock_irqrestore(&sched->job_list_lock, flags);
+
 		job->sched->ops->timedout_job(job);
 
 		/*
@@ -302,6 +316,8 @@ static void drm_sched_job_timedout(struct work_struct *work)
 			sched->free_guilty = false;
 		}
 	}
+	else
+		spin_unlock_irqrestore(&sched->job_list_lock, flags);
 
 	spin_lock_irqsave(&sched->job_list_lock, flags);
 	drm_sched_start_timeout(sched);
@@ -373,6 +389,19 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

5 lines not shown · raw message

Parts of this message are not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): e267429b-9c80-a9e7-7ffd-75ec439ed759-5C7GfCeVMHo@public.gmane.org

#60 lore raw

Hi Andrey,
Currently, I am busying with another issue, maybe will try next week.

Best wishes
Emily Deng

Quoted 683 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Friday, November 15, 2019 6:14 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Attached.

Emily - can you give it a try ?

Andrey

Quoted 5 lines · Christian König
Quoted 2 lines

What about instead of peeking at the job to actually remove it from ring_mirror_list right there,

Also an interesting idea. We would need to protect the mirror list with a lock again, but that should be the lesser evil.

Quoted 138 lines

Maybe prototype that and see if it works or not.

Regards,
Christian.

Am 13.11.19 um 17:00 schrieb Andrey Grodzovsky:

Quoted 132 lines · Christian König
Quoted 2 lines

Another more fundamental question: Could we get rid of the timeout job at all?

There are other stuff there besides picking the first unfinished job which is common for all the drivers - such as freeing guilty signaled job and rearming the timeout work timer.

Quoted 11 lines

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Should be ok since we have the extra check for __kthread_should_park in drm_sched_cleanup_jobs which will protect us in this case from a wakeup of sched thread and execution of in drm_sched_cleanup_jobs after we already parked it. The problem here is we need the drm_sched_job to access the private data for each client driver (see amdgpu_job_timedout for example). What about instead of peeking at the job to actually remove it from ring_mirror_list right there, go ahead with it through the reset routine, if it's signaled in the meanwhile that great - release it, otherwise put it back into ring_mirror_list in drm_sched_resubmit_jobs.

Andrey

Quoted 93 lines

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 88 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 79 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 72 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 50 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 36 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n", current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is running  OR thread
         * is being parked and hence assumed to not touch ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n", current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR*

ring

Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines

sdma0 timeout, signaled seq=78585, emitted seq=78587 Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:

[11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR*
Process information: process pid 0 thread pid 0,
s_job:00000000fe75ab36,tid=15603, pid=15603 Nov 12 12:58:20
ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225428] Emily:amdgpu_job_free_cb,Process information:
process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253,
pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225494] Emily:amdgpu_job_free_cb,Process information:

process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262

Quoted 6 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

for tdr

Quoted 2 lines

Thinking more about this claim - we assume here that if

cancel_delayed_work

Quoted 1 line

returned true it guarantees that timeout work is not running

but, it merely

Quoted 1 line

means there was a pending timeout work which was removed

from

Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
Quoted 48 lines
Quoted 2 lines

the workqueue before it's timer elapsed and so it didn't have a

chance to be

Quoted 1 line

dequeued and executed, it doesn't cover already executing

work. So there is a

Quoted 1 line

possibility where while timeout work started executing another

timeout work

Quoted 1 line

already got enqueued (maybe through earlier cleanup jobs or

through

Quoted 1 line

drm_sched_fault) and if at this point another

drm_sched_cleanup_jobs runs

Quoted 1 line

cancel_delayed_work(&sched->work_tdr) will return true even

while there is a

Quoted 3 lines

timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout

work itself

Quoted 1 line

waits for schedule thread  to be parked again when calling

park_thread.

Quoted 5 lines

Andrey

________________________________________
From: amd-gfx <[email protected]> on

behalf of

Quoted 4 lines

Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 6 lines

Hi Emily,

exactly that can't happen. See here:

Quoted 1 line

         /* Don't destroy jobs while the timeout worker is

running */

Quoted 5 lines
Quoted 3 lines

         if (sched->timeout != MAX_SCHEDULE_TIMEOUT && !cancel_delayed_work(&sched->work_tdr)) return NULL;

We never free jobs while the timeout working is running to

prevent exactly

Quoted 5 lines

that issue.

Regards,
Christian.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the

jobs while

Quoted 1 line

in amdgpu_device_gpu_recover, and before calling

drm_sched_stop.

Quoted 73 lines
Quoted 73 lines
Quoted 73 lines
Quoted 73 lines
Quoted 73 lines
Quoted 73 lines
Quoted 73 lines
Quoted 73 lines
Quoted 10 lines
Quoted 10 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>;

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job

in the first place.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:

Quoted 2 lines

Hi Chrisitan,
        No, I am with the new branch and also has the

patch. Even it

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line

are freed by

main scheduler, how we could avoid main scheduler to free

jobs while

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

enter to function amdgpu_device_gpu_recover?

Quoted 5 lines

Best wishes
Emily Deng

Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for tdr

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only

if no

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

timeout handler is running.

See this patch here:
commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:
Hi Christian,
         Please refer to follow log, when it enter to
amdgpu_device_gpu_recover
function, the bad job 000000005086879e is freeing in

function

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

amdgpu_job_free_cb  at the same time, because of the

hardware fence

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

signal.

Quoted 2 lines

But amdgpu_device_gpu_recover goes faster, at this case,
the s_fence is already freed, but job is not freed in time.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Then this issue

Quoted 2 lines

occurs.

Quoted 1 line
Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]]

*ERROR* ring

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

sdma0

Quoted 2 lines

timeout, signaled seq=2481, emitted seq=2483 [
449.793202] [drm:amdgpu_job_timedout [amdgpu]]

*ERROR*

Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

process  pid 0 thread pid 0, s_job:000000005086879e [

449.794163]

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

amdgpu
0000:00:08.0: GPU reset begin!
[  449.794175] Emily:amdgpu_job_free_cb,Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

process pid 0 thread  pid 0, s_job:000000005086879e [

449.794221]

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:0000000066eb74ab [  449.794222]
Emily:amdgpu_job_free_cb,Process information: process

8 lines not shown · raw message

Quoted 8 lines

8 lines not shown · raw message

309 lines not shown · raw message

Quoted 31 lines

31 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): MN2PR12MB29754C96F982E8C4F5ACC4C08F700-rweVpJHSKToFlvJWC7EAqwdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#62 lore raw

Thanks Emily.

Christan - ping for review.

Andrey

Quoted 695 lines · Emily

Hi Andrey,
Currently, I am busying with another issue, maybe will try next week.

Best wishes
Emily Deng

Quoted 687 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Friday, November 15, 2019 6:14 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue for tdr

Attached.

Emily - can you give it a try ?

Andrey

Quoted 5 lines · Christian König
Quoted 2 lines

What about instead of peeking at the job to actually remove it from ring_mirror_list right there,

Also an interesting idea. We would need to protect the mirror list with a lock again, but that should be the lesser evil.

Quoted 130 lines

Maybe prototype that and see if it works or not.

Regards,
Christian.

Am 13.11.19 um 17:00 schrieb Andrey Grodzovsky:

Quoted 124 lines · Christian König
Quoted 2 lines

Another more fundamental question: Could we get rid of the timeout job at all?

There are other stuff there besides picking the first unfinished job which is common for all the drivers - such as freeing guilty signaled job and rearming the timeout work timer.

Quoted 9 lines

I mean we used to give this as parameter to the scheduler callback because we had the timeout worker in the job, but that is no longer the case.

E.g. in drm_sched_job_timedout() we do the following:

Quoted 2 lines

        job = list_first_entry_or_null(&sched->ring_mirror_list, struct drm_sched_job, node);

Why don't we just remove that here and only get the first job after we have stopped the scheduler?

Should be ok since we have the extra check for __kthread_should_park in drm_sched_cleanup_jobs which will protect us in this case from a wakeup of sched thread and execution of in drm_sched_cleanup_jobs after we already parked it. The problem here is we need the drm_sched_job to access the private data for each client driver (see amdgpu_job_timedout for example). What about instead of peeking at the job to actually remove it from ring_mirror_list right there, go ahead with it through the reset routine, if it's signaled in the meanwhile that great - release it, otherwise put it back into ring_mirror_list in drm_sched_resubmit_jobs.

Andrey

Quoted 90 lines

Regards,
Christian.

Am 13.11.19 um 15:12 schrieb Andrey Grodzovsky:

Quoted 86 lines

This why I asked for a trace with timer enabled, but since there is a finite number of places we touch the timer Emily can just put prints there. Also, I wonder if this temp fix helps her with the issue or not.

Andrey

Quoted 78 lines · Christian König

The question is where do we rearm the timer for this problem to occur?

Regards,
Christian.

Am 12.11.19 um 20:21 schrieb Andrey Grodzovsky:

Quoted 71 lines

I was able to reproduce the crash by using the attached simulate_crash.patch - waiting on guilty job to signal in reset work and artificially rearming the timeout timer just before the check for !cancel_delayed_work(&sched->work_tdr)  in drm_sched_cleanup_jobs - crash log attached in crash.log. This I think confirms my theory i described earlier in this thread.

basic_fix.patch handles this by testing whether another timer already armed ob this scheduler or is there a timeout work in execution right now (see documentation for work_busy) - obviously this is not a full solution as this will not protect from races if for example there is immediate work scheduling such as in drm_sched_fault -  so we probably need to account for this by making drm_sched_cleanup_jobs (at least in the part where it iterates ring mirror list and frees jobs) and GPU reset really mutually exclusive and not like now.

Andrey

Quoted 50 lines · Christian König

Hi Emily,

you need to print which scheduler instance is freeing the jobs and which one is triggering the reset. The TID and PID is completely meaningless here since we are called from different worker threads and the TID/PID can change on each call.

Apart from that I will look into this a bit deeper when I have time.

Regards,
Christian.

Am 12.11.19 um 07:02 schrieb Deng, Emily:

Quoted 36 lines

Hi Christian,
    I add the follow print in function drm_sched_cleanup_jobs. From the log it shows that only use cancel_delayed_work could
not avoid to free job when the sched is in reset. But don’t know exactly where it is wrong about the driver. Do you have any suggestion about this?
+ printk("Emily:drm_sched_cleanup_jobs:begin,tid:%lu, pid:%lu\n", current->tgid, current->pid);
        /*
         * Don't destroy jobs while the timeout worker is
running  OR thread
         * is being parked and hence assumed to not touch
ring_mirror_list
         */
         if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr)))
                return;
+ printk("Emily:drm_sched_cleanup_jobs,tid:%lu, pid:%lu\n", current->tgid, current->pid);
Best wishes
Emily Deng

Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695091] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695104] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695105] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695107] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11380.695107] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.222954] [drm:amdgpu_job_timedout [amdgpu]] *ERROR*

ring

Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines
Quoted 42 lines

sdma0 timeout, signaled seq=78585, emitted seq=78587 Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:

[11381.224275] [drm:amdgpu_job_timedout [amdgpu]] *ERROR*
Process information: process pid 0 thread pid 0,
s_job:00000000fe75ab36,tid=15603, pid=15603 Nov 12 12:58:20
ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225413] amdgpu 0000:00:08.0: GPU reset begin!
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225417] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225425] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225425] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225428] Emily:amdgpu_job_free_cb,Process information:
process  pid 0 thread  pid 0, s_job:00000000fe75ab36, tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225429] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225430] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225473] Emily:drm_sched_cleanup_jobs:begin,tid:2253,
pid:2253
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225486] Emily:drm_sched_cleanup_jobs:begin,tid:2262,
pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225489] Emily:drm_sched_cleanup_jobs,tid:2262, pid:2262
Nov 12 12:58:20 ubuntu-drop-August-2018-rc2-gpu0-vf02 kernel:
[11381.225494] Emily:amdgpu_job_free_cb,Process information:

process  pid 0 thread  pid 0, s_job:00000000f086ec84, tid:2262, pid:2262

Quoted 6 lines

-----Original Message-----
From: Grodzovsky, Andrey <[email protected]>
Sent: Tuesday, November 12, 2019 11:28 AM
To: Koenig, Christian <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

for tdr

Quoted 1 line

Thinking more about this claim - we assume here that if

cancel_delayed_work

Quoted 1 line

returned true it guarantees that timeout work is not running

but, it merely

Quoted 1 line

means there was a pending timeout work which was removed

from

Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 45 lines
Quoted 2 lines

the workqueue before it's timer elapsed and so it didn't have a

chance to be

Quoted 1 line

dequeued and executed, it doesn't cover already executing

work. So there is a

Quoted 1 line

possibility where while timeout work started executing another

timeout work

Quoted 1 line

already got enqueued (maybe through earlier cleanup jobs or

through

Quoted 1 line

drm_sched_fault) and if at this point another

drm_sched_cleanup_jobs runs

Quoted 1 line

cancel_delayed_work(&sched->work_tdr) will return true even

while there is a

Quoted 3 lines

timeout job in progress.
Unfortunately we cannot change cancel_delayed_work to cancel_delayed_work_sync to flush the timeout work as timeout

work itself

Quoted 1 line

waits for schedule thread  to be parked again when calling

park_thread.

Quoted 4 lines

Andrey

________________________________________
From: amd-gfx <[email protected]> on

behalf of

Quoted 4 lines

Koenig, Christian <[email protected]>
Sent: 08 November 2019 05:35:18
To: Deng, Emily; [email protected]
Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 5 lines

Hi Emily,

exactly that can't happen. See here:

Quoted 1 line

         /* Don't destroy jobs while the timeout worker is

running */

Quoted 4 lines
Quoted 3 lines

         if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
!cancel_delayed_work(&sched->work_tdr))
                 return NULL;

We never free jobs while the timeout working is running to

prevent exactly

Quoted 5 lines

that issue.

Regards,
Christian.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines

Am 08.11.19 um 11:32 schrieb Deng, Emily:

Quoted 2 lines

Hi Christian,
       The drm_sched_job_timedout-> amdgpu_job_timedout call

amdgpu_device_gpu_recover. I mean the main scheduler free the

jobs while

Quoted 1 line

in amdgpu_device_gpu_recover, and before calling

drm_sched_stop.

Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 9 lines
Quoted 9 lines

Best wishes
Emily Deng

Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:26 PM
To: Deng, Emily <[email protected]>;

[email protected]

Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer issue

for tdr

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

Hi Emily,

well who is calling amdgpu_device_gpu_recover() in this case?

When it's not the scheduler we shouldn't have a guilty job

in the first place.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Regards,
Christian.

Am 08.11.19 um 11:22 schrieb Deng, Emily:
Hi Chrisitan,
        No, I am with the new branch and also has the

patch. Even it

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

are freed by
main scheduler, how we could avoid main scheduler to free

jobs while

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

enter to function amdgpu_device_gpu_recover?
Best wishes
Emily Deng

Quoted 6 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Koenig, Christian <[email protected]>
Sent: Friday, November 8, 2019 6:15 PM
To: Deng, Emily <[email protected]>; amd-

[email protected]

Quoted 1 line
Quoted 1 line

Subject: Re: [PATCH] drm/amdgpu: Fix the null pointer

issue for tdr

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

Hi Emily,

in this case you are on an old code branch.

Jobs are freed now by the main scheduler thread and only

if no

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

timeout handler is running.

See this patch here:
commit 5918045c4ed492fb5813f980dcf89a90fefd0a4e
Author: Christian König <[email protected]>
Date:   Thu Apr 18 11:00:21 2019 -0400

       drm/scheduler: rework job destruction

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

Regards,
Christian.

Am 08.11.19 um 11:11 schrieb Deng, Emily:
Hi Christian,
         Please refer to follow log, when it enter to
amdgpu_device_gpu_recover
function, the bad job 000000005086879e is freeing in

function

Quoted 1 line
Quoted 1 line
Quoted 1 line

amdgpu_job_free_cb  at the same time, because of the

hardware fence

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

signal.
But amdgpu_device_gpu_recover goes faster, at this case,
the s_fence is already freed, but job is not freed in time.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Then this issue

Quoted 2 lines

occurs.

Quoted 1 line
Quoted 1 line

[  449.792189] [drm:amdgpu_job_timedout [amdgpu]]

*ERROR* ring

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

sdma0
timeout, signaled seq=2481, emitted seq=2483 [
449.793202] [drm:amdgpu_job_timedout [amdgpu]]

*ERROR*

Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line

process  pid 0 thread pid 0, s_job:000000005086879e [

449.794163]

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

amdgpu
0000:00:08.0: GPU reset begin!
[  449.794175] Emily:amdgpu_job_free_cb,Process

information:

Quoted 1 line
Quoted 1 line
Quoted 1 line

process pid 0 thread  pid 0, s_job:000000005086879e [

449.794221]

Quoted 1 line
Quoted 1 line
Quoted 1 line

Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:0000000066eb74ab [  449.794222]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:00000000d4438ad9 [  449.794255]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

thread pid 0, s_job:00000000b6d69c65 [  449.794257]
Emily:amdgpu_job_free_cb,Process information: process

pid 0

Quoted 1 line
Quoted 1 line
Quoted 1 line

thread pid 0,

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

s_job:00000000ea85e922 [ 449.794287]
Emily:amdgpu_job_free_cb,Process
information: process pid 0 thread  pid 0,

s_job:00000000ed3a5ac6 [

Quoted 1 line
Quoted 1 line
Quoted 1 line

449.794366] BUG: unable to handle kernel NULL pointer

dereference

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

at
00000000000000c0 [ 449.800818] PGD 0 P4D 0

[  449.801040]

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Oops:

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

0000 [#1] SMP PTI
[  449.801338] CPU: 3 PID: 55 Comm: kworker/3:1 Tainted:

G OE

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

4.18.0-15-generic #16~18.04.1-Ubuntu
[  449.802157] Hardware name: QEMU Standard PC (i440FX

+

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

PIIX,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 [
449.802944]
Workqueue: events drm_sched_job_timedout [amd_sched]

[

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

449.803488]
RIP:
0010:amdgpu_device_gpu_recover+0x1da/0xb60 [amdgpu]
[  449.804020] Code: dd ff ff 49 39 c5 48 89 55 a8 0f 85

56 ff ff

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

ff
45 85 e4 0f
85 a1 00 00 00 48 8b 45 b0 48 85 c0 0f 84 60 01 00 00 48

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

8b 40 10

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

3 lines not shown · raw message

4 lines not shown · raw message

300 lines not shown · raw message

4 lines not shown · raw message

Part of this message is not shown here. View the raw message.

A message that is not in the archive (sent off-list or not yet received): c4791437-d42d-31fd-972f-cd2cdb26e951-5C7GfCeVMHo@public.gmane.org