Skip to contentSkip to conversation

drm/sched: fix the bug of time out calculation(v3)

PATCH In review Monk Liu 6 participants 27 messages Aug 31, 2021 – Nov 11, 2021

Subsystems (3)
lore
Share & tools
lore

Monk Liu

author
#0 lore raw

PATCH 1/2 drm/sched: fix the bug of time out calculation(v3)

issue:
in cleanup_job the cancle_delayed_work will cancel a TO timer even the its corresponding job is still running.

fix:
do not cancel the timer in cleanup_job, instead do the cancelling only when the heading job is signaled, and if there is a "next" job we start_timeout again.

v2:
further cleanup the logic, and do the TDR timer cancelling if the signaled job is the last one in its scheduler.

v3:
change the issue description
remove the cancel_delayed_work in the begining of the cleanup_job recover the implement of drm_sched_job_begin.

TODO:
1)introduce pause/resume scheduler in job_timeout to serial the handling of scheduler and job_timeout.
2)drop the bad job's del and insert in scheduler due to above serialization (no race issue anymore with the serialization)

---
 drivers/gpu/drm/scheduler/sched_main.c | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +10 -15
index a2a9536..ecf8140 100644
@@ -676,13 +676,7 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 {
 	struct drm_sched_job *job, *next;
 
-	/*
-	 * Don't destroy jobs while the timeout worker is running  OR thread
-	 * is being parked and hence assumed to not touch pending_list
-	 */
-	if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
-	    !cancel_delayed_work(&sched->work_tdr)) ||
-	    kthread_should_park())
+	if (kthread_should_park())
 		return NULL;
 
 	spin_lock(&sched->job_list_lock);
@@ -693,17 +687,21 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 	if (job && dma_fence_is_signaled(&job->s_fence->finished)) {
 		/* remove job from pending_list */
 		list_del_init(&job->list);
+
+		/* cancel this job's TO timer */
+		cancel_delayed_work(&sched->work_tdr);
 		/* make the scheduled timestamp more accurate */
 		next = list_first_entry_or_null(&sched->pending_list,
 						typeof(*next), list);
-		if (next)
+
+		if (next) {
 			next->s_fence->scheduled.timestamp =
 				job->s_fence->finished.timestamp;
-
+			/* start TO timer for next job */
+			drm_sched_start_timeout(sched);
+		}
 	} else {
 		job = NULL;
-		/* queue timeout for next job */
-		drm_sched_start_timeout(sched);
 	}
 
 	spin_unlock(&sched->job_list_lock);
@@ -791,11 +789,8 @@ static int drm_sched_main(void *param)
 					  (entity = drm_sched_select_entity(sched))) ||
 					 kthread_should_stop());
 
-		if (cleanup_job) {
+		if (cleanup_job)
 			sched->ops->free_job(cleanup_job);
-			/* queue timeout for next job */
-			drm_sched_start_timeout(sched);
-		}
 
 		if (!entity)
 			continue;
Signature · 1 line
2.7.4
author
#1 lore raw

PATCH 2/2 drm/sched: serialize job_timeout and scheduler

---
 drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +4 -20
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
 	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
Signature · 1 line
2.7.4
#2 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 12 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 322

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
 	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.
-Daniel

Quoted 43 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -20
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
Signature · 1 line
2.7.4
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#3 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 25 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 12 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 322

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
 	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much.
-Daniel

Quoted 48 lines
Quoted 43 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -20
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
Signature · 1 line
2.7.4
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
author
#4 lore raw

RE: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

[AMD Official Use Only]

Quoted 1 line
Quoted 1 line

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much.

Are you referring to drm_sched_stop ?

That's different, we don't need the logic from it, see that it go through pending list and remove all callbacks , etc... meanwhile vendor's timeout callback will call drm_sched_stop in a proper way, All we want in my patch is to simply park scheduler,
Besides, even you call drm_sched_stop in job_timeout you still run into the warning issue I hit.

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 19 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24 
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, 
 work_tdr.work);
 
 	/* Protects against concurrent deletion in 

drm_sched_get_cleanup_job */

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 322

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +1 -0
+	if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

author
#5 lore raw

RE: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

In reply to Daniel Vetter

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 19 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24 
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, 
 work_tdr.work);
 
 	/* Protects against concurrent deletion in 

drm_sched_get_cleanup_job */

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 322

Quoted 1 line
drivers/gpu/drm/scheduler/sched_main.c +1 -0
+	if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

#6 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

---------------------

(quoted diff) +0 -0

b/drivers/gpu/drm/scheduler/sched_main.c
index 67382621b429..4d6ce775c316 100644

drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct

drm_gpu_scheduler *sched)

        return false;

}

+static bool handle_timeout(struct drm_gpu_scheduler *sched) +{
+ struct drm_sched_job *bad;
+
+ if (!sched->has_timeout)
+ return false;
+
+ sched->has_timeout = false;
+
+ spin_lock(&sched->job_list_lock);
+ bad = list_first_entry_or_null(&sched->pending_list,
+ struct drm_sched_job, list);
+
+ if (!bad) {
+ spin_unlock(&sched->job_list_lock);
+ return false;
+ }
+
+ spin_unlock(&sched->job_list_lock);
+
+ if (sched->timeout_wq == system_wq) {
+ /*

+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */

+ sched->ops->timedout_job(job);
+ } else {
+ /*
+ * Otherwise queue it on timeout_wq and wait for it to complete
+ */
+ ... more typing needed here ...
+ }
+
+ if (sched->free_guilty) {
+ sched->ops->free_job(job);
+ sched->free_guilty = false;
+ }
+}
+
/**
* drm_sched_main - main scheduler thread
*
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)

                wait_event_interruptible(sched->wake_up_worker,
                                         (cleanup_job =
drm_sched_get_cleanup_job(sched)) ||
+                                        handle_timeout(sched) ||
                                         (!drm_sched_blocked(sched) &&
                                          (entity =
drm_sched_select_entity(sched))) ||
                                         kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

BR,
-R

Quoted 113 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
    /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
    spin_lock(&sched->job_list_lock);
    job = list_first_entry_or_null(&sched->pending_list,
                                   struct drm_sched_job, list);
 
    if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
            spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
            status = job->sched->ops->timedout_job(job);
 
            /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    kthread_park(sched->thread);
 
    /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
     * Iterate the job list from later to  earlier one and either deactive
     * their HW callbacks or remove them from pending list if they already
     * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

#7 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 11 lines · Rob Clark

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.
-Daniel

Quoted 191 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
        return false;
 }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
 /**
  * drm_sched_main - main scheduler thread

*

drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                wait_event_interruptible(sched->wake_up_worker,
                                         (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                         (!drm_sched_blocked(sched) &&
                                          (entity =
drm_sched_select_entity(sched))) ||
                                         kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

BR,
-R

Quoted 113 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
    /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
    spin_lock(&sched->job_list_lock);
    job = list_first_entry_or_null(&sched->pending_list,
                                   struct drm_sched_job, list);
 
    if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
            spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
            status = job->sched->ops->timedout_job(job);
 
            /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    kthread_park(sched->thread);
 
    /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
     * Iterate the job list from later to  earlier one and either deactive
     * their HW callbacks or remove them from pending list if they already
     * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#8 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 31 lines · Daniel Vetter
Quoted 11 lines · Rob Clark

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

Quoted 4 lines

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

BR,
-R

Quoted 198 lines

-Daniel

Quoted 74 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
        return false;
 }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
 /**
  * drm_sched_main - main scheduler thread

*

drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                wait_event_interruptible(sched->wake_up_worker,
                                         (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                         (!drm_sched_blocked(sched) &&
                                          (entity =
drm_sched_select_entity(sched))) ||
                                         kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

Quoted 117 lines

BR,
-R

Quoted 113 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
    /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
    spin_lock(&sched->job_list_lock);
    job = list_first_entry_or_null(&sched->pending_list,
                                   struct drm_sched_job, list);
 
    if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
            spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
            status = job->sched->ops->timedout_job(job);
 
            /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    kthread_park(sched->thread);
 
    /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
     * Iterate the job list from later to  earlier one and either deactive
     * their HW callbacks or remove them from pending list if they already
     * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

#9 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 32 lines · Rob Clark
Quoted 31 lines · Daniel Vetter
Quoted 11 lines · Rob Clark

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

Quoted 8 lines
Quoted 4 lines

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

Maybe this was because you worked on an old kernel? Boris did update the kerneldoc as part of making gpu reset work for panfrost, which has this multi-engine reset problem. If that's not yet clear then we need to improve the docs further.

AMD's problem is even worse, because their reset domain is the entire xgmi hive, so multiple pci devices.

Also there might more issues in drm/sched ofc, e.g. I've looked a bit at ordering/barriers and I'm pretty sure a lot are still missing. Or at least we should have comments in the code explaining why it all works. -Daniel

Quoted 202 lines

BR,
-R

Quoted 198 lines

-Daniel

Quoted 74 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
        return false;
 }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
 /**
  * drm_sched_main - main scheduler thread

*

drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                wait_event_interruptible(sched->wake_up_worker,
                                         (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                         (!drm_sched_blocked(sched) &&
                                          (entity =
drm_sched_select_entity(sched))) ||
                                         kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

Quoted 117 lines

BR,
-R

Quoted 113 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
    /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
    spin_lock(&sched->job_list_lock);
    job = list_first_entry_or_null(&sched->pending_list,
                                   struct drm_sched_job, list);
 
    if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
            spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
            status = job->sched->ops->timedout_job(job);
 
            /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    kthread_park(sched->thread);
 
    /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
     * Iterate the job list from later to  earlier one and either deactive
     * their HW callbacks or remove them from pending list if they already
     * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#10 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Am 10.11.21 um 10:50 schrieb Daniel Vetter:

Quoted 38 lines · Rob Clark
Quoted 30 lines · Daniel Vetter
Quoted 29 lines · Rob Clark
Quoted 11 lines

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

Quoted 7 lines
Quoted 4 lines

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

Quoted 7 lines

Maybe this was because you worked on an old kernel? Boris did update the kerneldoc as part of making gpu reset work for panfrost, which has this multi-engine reset problem. If that's not yet clear then we need to improve the docs further.

AMD's problem is even worse, because their reset domain is the entire xgmi hive, so multiple pci devices.

I'm pushing for quite a while that we get something like an amdgpu_reset_domain structure or similar for this, but we unfortunately don't have that yet.

Maybe it should be a good idea to have something like a drm_sched_domain or similar with all the necessary information for the inter scheduler handling.

E.g. a workqueue for reset etc...

Regards,
Christian.

Quoted 200 lines

Also there might more issues in drm/sched ofc, e.g. I've looked a bit at ordering/barriers and I'm pretty sure a lot are still missing. Or at least we should have comments in the code explaining why it all works. -Daniel

Quoted 194 lines

BR,
-R

Quoted 191 lines

-Daniel

Quoted 73 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
         return false;
  }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
  /**
   * drm_sched_main - main scheduler thread
   *
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                 wait_event_interruptible(sched->wake_up_worker,
                                          (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                          (!drm_sched_blocked(sched) &&
                                           (entity =
drm_sched_select_entity(sched))) ||
                                          kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

Quoted 112 lines

BR,
-R

Quoted 108 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 28 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
  drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
  1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
     sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
     /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 44 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
     spin_lock(&sched->job_list_lock);
     job = list_first_entry_or_null(&sched->pending_list,
                                    struct drm_sched_job, list);
 
     if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
             spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
             status = job->sched->ops->timedout_job(job);
 
             /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
     kthread_park(sched->thread);
 
     /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
      * Iterate the job list from later to  earlier one and either deactive
      * their HW callbacks or remove them from pending list if they already
      * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

#11 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 50 lines · Christian König

Am 10.11.21 um 10:50 schrieb Daniel Vetter:

Quoted 41 lines · Rob Clark
Quoted 33 lines · Daniel Vetter
Quoted 32 lines · Rob Clark
Quoted 11 lines

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs).  We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics.  Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler
   threads.

- You get deadlocks if 2 schedulers time out at the same time, and both
   want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in
   the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

Quoted 7 lines
Quoted 4 lines

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

Quoted 7 lines

Maybe this was because you worked on an old kernel? Boris did update the kerneldoc as part of making gpu reset work for panfrost, which has this multi-engine reset problem. If that's not yet clear then we need to improve the docs further.

AMD's problem is even worse, because their reset domain is the entire xgmi hive, so multiple pci devices.

Quoted 8 lines

I'm pushing for quite a while that we get something like an amdgpu_reset_domain structure or similar for this, but we unfortunately don't have that yet.

Maybe it should be a good idea to have something like a drm_sched_domain or similar with all the necessary information for the inter scheduler handling.

E.g. a workqueue for reset etc...

Regards,
Christian.

I think Monk and Jingwen already switched SRIOV case to using Boris's single threaded queue interface for SRIOV, we can try to expand this to general bare metal case for AMD and on the way to add drm_sched_domain to the scheduler.

Andrey

Quoted 221 lines
Quoted 220 lines

Also there might more issues in drm/sched ofc, e.g. I've looked a bit at ordering/barriers and I'm pretty sure a lot are still missing. Or at least we should have comments in the code explaining why it all works. -Daniel

Quoted 213 lines

BR,
-R

Quoted 210 lines

-Daniel

Quoted 203 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +23 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
          return false;
   }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about 

serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }

drivers/gpu/drm/scheduler/sched_main.c +7 -0
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+

  /**
   * drm_sched_main - main scheduler thread
   *

drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
 wait_event_interruptible(sched->wake_up_worker,

                                          (cleanup_job = drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) || (!drm_sched_blocked(sched) &&
                                           (entity = drm_sched_select_entity(sched))) || kthread_should_stop());
---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

BR,
-R

Quoted 123 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 19 lines · Monk Liu

---
  drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------

  1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct 
 work_struct *work)
      sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
      /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+   if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 65 lines
Quoted 51 lines

+ kthread_park(sched->thread);
+
     spin_lock(&sched->job_list_lock);
     job = list_first_entry_or_null(&sched->pending_list, struct drm_sched_job, list);

     if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe. -            */
-           list_del_init(&job->list); spin_unlock(&sched->job_list_lock);

+           /* vendor's timeout_job should call drm_sched_start() */
             status = job->sched->ops->timedout_job(job);

             /*

drivers/gpu/drm/scheduler/sched_main.c +0 -2
@@ -393,20 +391,6 @@ void drm_sched_stop(struct 
 drm_gpu_scheduler *sched, struct drm_sched_job *bad)
      kthread_park(sched->thread);
 
      /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and 

release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
      * Iterate the job list from later to  earlier one and either deactive
      * their HW callbacks or remove them from pending list if they already
      * signaled.

Signature · 1 line
2.7.4
Signature · 13 lines
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog. 

ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 

b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 

51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL 

CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg 

KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C3c31e06d94674e61f3a008d9a4323fb1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637721358003695123%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=FMFNbfIxwNG3Tv1p1iHLI%2BpepJgwvsT%2FxhL%2FKZc0eVE%3D&amp;reserved=0 
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C3c31e06d94674e61f3a008d9a4323fb1%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637721358003695123%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=FMFNbfIxwNG3Tv1p1iHLI%2BpepJgwvsT%2FxhL%2FKZc0eVE%3D&amp;reserved=0 
#12 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

In reply to Christian König
Quoted 47 lines · Christian König

Am 10.11.21 um 10:50 schrieb Daniel Vetter:

Quoted 38 lines · Rob Clark
Quoted 30 lines · Daniel Vetter
Quoted 29 lines · Rob Clark
Quoted 11 lines

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

Quoted 7 lines
Quoted 4 lines

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

Quoted 7 lines

Maybe this was because you worked on an old kernel? Boris did update the kerneldoc as part of making gpu reset work for panfrost, which has this multi-engine reset problem. If that's not yet clear then we need to improve the docs further.

AMD's problem is even worse, because their reset domain is the entire xgmi hive, so multiple pci devices.

Quoted 8 lines

I'm pushing for quite a while that we get something like an amdgpu_reset_domain structure or similar for this, but we unfortunately don't have that yet.

Maybe it should be a good idea to have something like a drm_sched_domain or similar with all the necessary information for the inter scheduler handling.

E.g. a workqueue for reset etc...

Yeah I think as soon as we have more stuff than just the wq then a drm_sched_reset_domain sounds good.

But if it's just driver stuff (e.g. the xgmi locking you have in amdgpu reset comes to mind) then I think just a driver_reset_domain struct that bundles all that stuff up seems good enough.

E.g. on i915 I'm also pondering whether some of the fw requests should be processed by the reset wq, to avoid locking headaches, so I don't think hiding that work too much in abstractions is a good idea. -Daniel

Quoted 205 lines

Regards,
Christian.

Quoted 200 lines

Also there might more issues in drm/sched ofc, e.g. I've looked a bit at ordering/barriers and I'm pretty sure a lot are still missing. Or at least we should have comments in the code explaining why it all works. -Daniel

Quoted 194 lines

BR,
-R

Quoted 191 lines

-Daniel

Quoted 73 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
         return false;
  }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
  /**
   * drm_sched_main - main scheduler thread
   *
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                 wait_event_interruptible(sched->wake_up_worker,
                                          (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                          (!drm_sched_blocked(sched) &&
                                           (entity =
drm_sched_select_entity(sched))) ||
                                          kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

Quoted 112 lines

BR,
-R

Quoted 108 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 28 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
  drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
  1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
     sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
     /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 44 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
     spin_lock(&sched->job_list_lock);
     job = list_first_entry_or_null(&sched->pending_list,
                                    struct drm_sched_job, list);
 
     if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
             spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
             status = job->sched->ops->timedout_job(job);
 
             /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
     kthread_park(sched->thread);
 
     /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
      * Iterate the job list from later to  earlier one and either deactive
      * their HW callbacks or remove them from pending list if they already
      * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#13 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 58 lines · Daniel Vetter
Quoted 46 lines · Christian König

Am 10.11.21 um 10:50 schrieb Daniel Vetter:

Quoted 38 lines · Rob Clark
Quoted 30 lines · Daniel Vetter
Quoted 29 lines · Rob Clark
Quoted 11 lines

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

Quoted 7 lines
Quoted 4 lines

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

Quoted 7 lines

Maybe this was because you worked on an old kernel? Boris did update the kerneldoc as part of making gpu reset work for panfrost, which has this multi-engine reset problem. If that's not yet clear then we need to improve the docs further.

AMD's problem is even worse, because their reset domain is the entire xgmi hive, so multiple pci devices.

Quoted 8 lines

I'm pushing for quite a while that we get something like an amdgpu_reset_domain structure or similar for this, but we unfortunately don't have that yet.

Maybe it should be a good idea to have something like a drm_sched_domain or similar with all the necessary information for the inter scheduler handling.

E.g. a workqueue for reset etc...

Yeah I think as soon as we have more stuff than just the wq then a drm_sched_reset_domain sounds good.

Quoted 8 lines

But if it's just driver stuff (e.g. the xgmi locking you have in amdgpu reset comes to mind) then I think just a driver_reset_domain struct that bundles all that stuff up seems good enough.

E.g. on i915 I'm also pondering whether some of the fw requests should be processed by the reset wq, to avoid locking headaches, so I don't think hiding that work too much in abstractions is a good idea. -Daniel

I suggest we keep the drm_sched_reset_domain as a base struct to hold the wq (and possible something else cross drivers in the future) and then embed it in a derived driver specific struct to hold driver specific stuff like
the XGMI lock you mentioned.

Andrey

Quoted 203 lines
Quoted 203 lines

Regards,
Christian.

Quoted 199 lines

Also there might more issues in drm/sched ofc, e.g. I've looked a bit at ordering/barriers and I'm pretty sure a lot are still missing. Or at least we should have comments in the code explaining why it all works. -Daniel

Quoted 194 lines

BR,
-R

Quoted 191 lines

-Daniel

Quoted 73 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
          return false;
   }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
   /**
    * drm_sched_main - main scheduler thread
    *
drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                  wait_event_interruptible(sched->wake_up_worker,
                                           (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                           (!drm_sched_blocked(sched) &&
                                            (entity =
drm_sched_select_entity(sched))) ||
                                           kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

Quoted 112 lines

BR,
-R

Quoted 108 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 28 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
   drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
   1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
      sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
      /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 44 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
      spin_lock(&sched->job_list_lock);
      job = list_first_entry_or_null(&sched->pending_list,
                                     struct drm_sched_job, list);
 
      if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
              spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
              status = job->sched->ops->timedout_job(job);
 
              /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
      kthread_park(sched->thread);
 
      /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
       * Iterate the job list from later to  earlier one and either deactive
       * their HW callbacks or remove them from pending list if they already
       * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7Cbf8af1e8a797474bd5c108d9a44d664b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637721474618053495%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=irN0l%2F8L7X9A8BRNAIYmOO4jMI1ZLeFGHPLYanVOMOA%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7Cbf8af1e8a797474bd5c108d9a44d664b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637721474618053495%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=irN0l%2F8L7X9A8BRNAIYmOO4jMI1ZLeFGHPLYanVOMOA%3D&amp;reserved=0

#14 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

In reply to Daniel Vetter
Quoted 39 lines · Daniel Vetter
Quoted 37 lines · Rob Clark
Quoted 35 lines · Daniel Vetter
Quoted 11 lines · Rob Clark

I stumbled across this thread when I ran into the same issue, while working out how to move drm/msm to use scheduler's retire + timeout/recovery (and get rid of our own mirror list of in-flight jobs). We already have hw error detection enabled, and it can signal quite fast, so assuming the first job on the list is the guilty job just won't work.

But I was considering a slightly different approach to fixing this, instead just handling it all in drm_sched_main() and getting rid of the complicated kthread parking gymnastics. Ie. something along the lines of:

So handling timeouts in the main sched thread wont work as soon as you have multiple engines and reset that impacts across engines:

- Nothing is simplified since you still need to stop the other scheduler threads.

- You get deadlocks if 2 schedulers time out at the same time, and both want to stop the other one.

Hence workqueue. Now the rule for the wq is that you can only have one per reset domain, so
- single engine you just take the one drm/sched provides
- if reset affects all your engines in the chip, then you allocate on in the drm_device and pass that to all
- if you have a complex of gpus all interconnected (e.g. xgmi hive for amd), then it's one wq for the entire hive

_All_ reset related things must be run on that workqueue or things breaks, which means if you get hw fault that also needs to be run there. I guess we should either patch drm/sched to check you call that function from the right workqueue, or just handle it internally.

Quoted 8 lines
Quoted 3 lines

Hmm, ok.. I guess it would be useful to better document the reasoning for the current design, that would have steered me more towards the approach taken in this patch.

Maybe this was because you worked on an old kernel? Boris did update the kerneldoc as part of making gpu reset work for panfrost, which has this multi-engine reset problem. If that's not yet clear then we need to improve the docs further.

I saw that, and understood the ordered wq.. but missed the implication regarding having to park other scheduler kthreads

BR,
-R

Quoted 215 lines

AMD's problem is even worse, because their reset domain is the entire xgmi hive, so multiple pci devices.

Also there might more issues in drm/sched ofc, e.g. I've looked a bit at ordering/barriers and I'm pretty sure a lot are still missing. Or at least we should have comments in the code explaining why it all works. -Daniel

Quoted 202 lines

BR,
-R

Quoted 198 lines

-Daniel

Quoted 74 lines

---------------------

drivers/gpu/drm/scheduler/sched_main.c +39 -0
index 67382621b429..4d6ce775c316 100644
@@ -764,6 +764,45 @@ static bool drm_sched_blocked(struct
 drm_gpu_scheduler *sched)
        return false;
 }
 
+static bool handle_timeout(struct drm_gpu_scheduler *sched)
+{
+       struct drm_sched_job *bad;
+
+       if (!sched->has_timeout)
+               return false;
+
+       sched->has_timeout = false;
+
+       spin_lock(&sched->job_list_lock);
+       bad = list_first_entry_or_null(&sched->pending_list,
+                                      struct drm_sched_job, list);
+
+       if (!bad) {
+               spin_unlock(&sched->job_list_lock);
+               return false;
+       }
+
+       spin_unlock(&sched->job_list_lock);
+
+       if (sched->timeout_wq == system_wq) {
+               /*
+                * If driver has no specific requirements about serializing
+                * reset wrt. other engines, just call timedout_job() directly
+                */
+               sched->ops->timedout_job(job);
+       } else {
+               /*
+                * Otherwise queue it on timeout_wq and wait for it to complete
+                */
+               ... more typing needed here ...
+       }
+
+       if (sched->free_guilty) {
+               sched->ops->free_job(job);
+               sched->free_guilty = false;
+       }
+}
+
 /**
  * drm_sched_main - main scheduler thread

*

drivers/gpu/drm/scheduler/sched_main.c +0 -0
@@ -787,6 +826,7 @@ static int drm_sched_main(void *param)
 
                wait_event_interruptible(sched->wake_up_worker,
                                         (cleanup_job =

drm_sched_get_cleanup_job(sched)) ||
+ handle_timeout(sched) ||

                                         (!drm_sched_blocked(sched) &&
                                          (entity =
drm_sched_select_entity(sched))) ||
                                         kthread_should_stop());

---------------------

drm_sched_fault() and the sw timeout handler would just set sched->has_timeout and kick sched->wake_up_worker.

And since we handle the timeout case after drm_sched_get_cleanup_job(), we know that all of the successfully completed jobs have already been popped off the list, and won't be unfairly maligned.

Quoted 117 lines

BR,
-R

Quoted 113 lines · Monk

[AMD Official Use Only]

Okay, I will reprepare this patch

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 9:02 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 29 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 18 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24
++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    sched = container_of(work, struct drm_gpu_scheduler,
 work_tdr.work);
 
    /* Protects against concurrent deletion in
Quoted 2 lines

drm_sched_get_cleanup_job */
+ if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Quoted 3 lines

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.

Also why don't we reuse the function drivers already have to stop a scheduler thread? We seem to have two kthread_park now, that's probably one too much. -Daniel

Quoted 45 lines
Quoted 41 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -21
+           kthread_park(sched->thread);
+
    spin_lock(&sched->job_list_lock);
    job = list_first_entry_or_null(&sched->pending_list,
                                   struct drm_sched_job, list);
 
    if (job) {
-           /*
-            * Remove the bad job so it cannot be freed by concurrent
-            * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-            * is parked at which point it's safe.
-            */
-           list_del_init(&job->list);
            spin_unlock(&sched->job_list_lock);
 
+           /* vendor's timeout_job should call drm_sched_start() */
            status = job->sched->ops->timedout_job(job);
 
            /*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    kthread_park(sched->thread);
 
    /*
-    * Reinsert back the bad job here - now it's safe as
-    * drm_sched_get_cleanup_job cannot race against us and release the
-    * bad job at this point - we parked (waited for) any in progress
-    * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-    * now until the scheduler thread is unparked.
-    */
-   if (bad && bad->sched == sched)
-           /*
-            * Add at the head of the queue to reflect it was the earliest
-            * job extracted.
-            */
-           list_add(&bad->list, &sched->pending_list);
-
-   /*
     * Iterate the job list from later to  earlier one and either deactive
     * their HW callbacks or remove them from pending list if they already
     * signaled.
--
Quoted 2 lines

2.7.4

--

Quoted 8 lines

Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.
ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76 b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C6376601170 51194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiL CJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLg KeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C298815bea18f4fbf76b308d96c7f7a8b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660117051194614%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=QzgCU7%2BPdA0aWL5%2BJLgKeKbGaMMGqeGI9KE0P0LXlN4%3D&amp;reserved=0

--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

--
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

#15 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

In reply to Daniel Vetter
Quoted 3 lines · Daniel Vetter

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

I agree with Daniel--a narrative form of a commit message is so much easier for humans to digest. The "[what]"/"[why]"/"[how]" and "issue"/"fix" format is somewhat dry and uninformative, and leaves much to be desired.

Regards,
Luben

Quoted 69 lines · Monk Liu
Quoted 12 lines
---
 drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 322

Quoted 4 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
 	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure.
-Daniel

Quoted 43 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -20
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
Signature · 1 line
2.7.4
author
#16 lore raw

RE: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

In reply to Daniel Vetter

[AMD Official Use Only]

Quoted 1 line
Quoted 1 line

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

It's not a traditional habit from what I can see that put explain in code, but we can do that in mails ,
We want to park the scheduler in job_timeout to serialize the job accessing from both sched and TO handler , but inside vendor's callback timeout_job at least both panfrost and amd They both call drm_sched_stop() on all schedulers.

If we unconditionally call "kthread_park" in job_timedout then the bailing job's timedout will try to call "kthread_park" again on its scheduler and introduce "warning"

The scenario is :
1,Job1 on sched1 triggers timedout, and sched1 is parked,
2,vendor callback runs, it will usually stop all schedulers.
3,Job2 on sched2 triggers timedout, so the job_timedout also try to park sched2, but sched2 was stopped already by above step. (job2's timeout is introduced by job1, or by other VF)
---So there will be "warning" in kernel log from above step... after this "__kthread_should_park()" here we can avoid the warning, that's the only reason I need this __function.
4,Before vendor callback exit, it will unpark all schedulers.

From another hand if we don't do the kthread_park() and still delete the job here (drop deleting/reinserting the job from pending_list is what we want), we still have a small windows to hit the race issue:
That cleanup_job from sched thread is freeing the job while job is under processing by job_timedout or vendor's call back.

And the reason we want to avoid deleting/reinserting the timedout job is because we (amd vendor) have our own way to do a diagnostic on all jobs in pending list from all scheduler, we want to cherry-pick the real bad job From all scheduler's pending list that causes this JOB TIMEOUT.

Besides, it is also much reasonable to park scheduler when job_timedout is running there, they should exclusively access those common members like sched_job. (due to spin_lock is off before running into vendor's calback)

Hope I explained ourselves well enough.

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Daniel Vetter <[email protected]>
Sent: Tuesday, August 31, 2021 8:59 PM
To: Liu, Monk <[email protected]>
Cc: [email protected]; [email protected]; Chen, Jingwen <[email protected]>
Subject: Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Can we please have some actual commit message here, with detailed explanation of the race/bug/whatever, how you fix it and why this is the best option?

Quoted 13 lines · Monk Liu
---
 drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
 	/* Protects against concurrent deletion in drm_sched_get_cleanup_job 
 */
+	if (!__kthread_should_park(sched->thread))

This is a __ function, i.e. considered internal, and it's lockless atomic, i.e. unordered. And you're not explaining why this works.

Iow it's probably buggy, and an just unconditionally parking the kthread is probably the right thing to do. If it's not the right thing to do, there's a bug here for sure. -Daniel

Quoted 43 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -6
+		kthread_park(sched->thread);
+
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
 	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
 		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
 		status = job->sched->ops->timedout_job(job);
 
drivers/gpu/drm/scheduler/sched_main.c +0 -15
 		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 	kthread_park(sched->thread);
 
 	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
 	 * Iterate the job list from later to  earlier one and either deactive
 	 * their HW callbacks or remove them from pending list if they already
 	 * signaled.
--

2.7.4

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fblog.ffwll.ch%2F&amp;data=04%7C01%7CMonk.Liu%40amd.com%7C4af6e233f48348677d5f08d96c7f1db2%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660115493853883%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=RJsFMBCIveO5pJvhkEdV6CBbP4VRiJKqb62Py8U44tw%3D&amp;reserved=0
#17 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

In reply to Monk Liu

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
  drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
  1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 324

Quoted 6 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
  	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
  	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other schedulers you just race here against them, e.g. you parked it now but another one in progress will unpark it as part of calling  drm_sched_start for other rings[1] Unless I am missing something since I haven't found patch [1/2]

[1] - https://elixir.bootlin.com/linux/latest/source/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c#L5041

Andrey

Quoted 38 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
  	spin_lock(&sched->job_list_lock);
  	job = list_first_entry_or_null(&sched->pending_list,
  				       struct drm_sched_job, list);
 
  	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
  		spin_unlock(&sched->job_list_lock);
 
+		/* vendor's timeout_job should call drm_sched_start() */
  		status = job->sched->ops->timedout_job(job);
 
  		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
  	kthread_park(sched->thread);
 
  	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
  	 * Iterate the job list from later to  earlier one and either deactive
  	 * their HW callbacks or remove them from pending list if they already
  	 * signaled.
#18 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 30 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
  drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
  1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
  	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
  	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 1 line

[1] - https://elixir.bootlin.com/linux/latest/source/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c#L5041

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

See

https://dri.freedesktop.org/docs/drm/gpu/drm-mm.html#c.drm_sched_backend_ops

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Quoted 38 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
  	spin_lock(&sched->job_list_lock);
  	job = list_first_entry_or_null(&sched->pending_list,
  				       struct drm_sched_job, list);
  	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
  		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
  		status = job->sched->ops->timedout_job(job);
  		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
  	kthread_park(sched->thread);
  	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
  	 * Iterate the job list from later to  earlier one and either deactive
  	 * their HW callbacks or remove them from pending list if they already
  	 * signaled.
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#19 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 30 lines · Daniel Vetter
Quoted 29 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
   drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
   1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
   	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
   	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 3 lines
Quoted 1 line

[1] - https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L5041&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7Cc697c75898664f678f4b08d96c8820e7%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660154199259544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=1Y8Tuh2fLtexYsGrmQD2ITTSIfUVJmqTylwgMryDjxw%3D&amp;reserved=0

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

IMHO what is problematic in serializing vs. locking (with trylock and bail out like we do in [1]) is for multiple TO events arising from same reason like maybe one job just waits for another and once first is hanged the second will also appear to be hanged triggering it's own TO event. In this case multiple TOs event will trigger multiple resets if we serialize but if we use lock with trylock the second one will quietly bail out.

[1] https://elixir.bootlin.com/linux/latest/source/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c#L4903

Andrey

Quoted 7 lines

See

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdri.freedesktop.org%2Fdocs%2Fdrm%2Fgpu%2Fdrm-mm.html%23c.drm_sched_backend_ops&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7Cc697c75898664f678f4b08d96c8820e7%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660154199259544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=tLjFaN7mREYjjydxHszbQlTk3lwH4bQtBDVzHFHvPJg%3D&amp;reserved=0

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Yes, this discussion is a continuation of your comment about serializing, I mentioned before that you proposed it.

Andrey

Quoted 38 lines
Quoted 38 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
   	spin_lock(&sched->job_list_lock);
   	job = list_first_entry_or_null(&sched->pending_list,
   				       struct drm_sched_job, list);
   	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
   		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
   		status = job->sched->ops->timedout_job(job);
   		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
   	kthread_park(sched->thread);
   	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
   	 * Iterate the job list from later to  earlier one and either deactive
   	 * their HW callbacks or remove them from pending list if they already
   	 * signaled.
#20 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 37 lines · Andrey Grodzovsky
Quoted 30 lines · Daniel Vetter
Quoted 29 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
   drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
   1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
   	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
   	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 3 lines
Quoted 1 line

[1] - https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L5041&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7Cc697c75898664f678f4b08d96c8820e7%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660154199259544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=1Y8Tuh2fLtexYsGrmQD2ITTSIfUVJmqTylwgMryDjxw%3D&amp;reserved=0

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

Quoted 8 lines

IMHO what is problematic in serializing vs. locking (with trylock and bail out like we do in [1]) is for multiple TO events arising from same reason like maybe one job just waits for another and once first is hanged the second will also appear to be hanged triggering it's own TO event. In this case multiple TOs event will trigger multiple resets if we serialize but if we use lock with trylock the second one will quietly bail out.

[1] https://elixir.bootlin.com/linux/latest/source/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c#L4903

Hm so I guess a single wq here, that will hold up all other TO. And they should recheck whether the job is moving meanwhile.

Also unless you use hw semaphores the job shouldn't even start before the deps are singalled, so not sure how this goes wrong?

The vm_id flush stuff can make things a bit more fun for your specific case, but in your specific case you have to run all TO handlers on the same ordered workqueue anyway (because trying to paper over this in other ways doesn't work imo).

So I think this should all work, no need for tricky cross-scheduler locking.
-Daniel

Quoted 57 lines

Andrey

Quoted 7 lines

See

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdri.freedesktop.org%2Fdocs%2Fdrm%2Fgpu%2Fdrm-mm.html%23c.drm_sched_backend_ops&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7Cc697c75898664f678f4b08d96c8820e7%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660154199259544%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=tLjFaN7mREYjjydxHszbQlTk3lwH4bQtBDVzHFHvPJg%3D&amp;reserved=0

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Yes, this discussion is a continuation of your comment about serializing, I mentioned before that you proposed it.

Andrey

Quoted 38 lines
Quoted 38 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
   	spin_lock(&sched->job_list_lock);
   	job = list_first_entry_or_null(&sched->pending_list,
   				       struct drm_sched_job, list);
   	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
   		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
   		status = job->sched->ops->timedout_job(job);
   		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
   	kthread_park(sched->thread);
   	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
   	 * Iterate the job list from later to  earlier one and either deactive
   	 * their HW callbacks or remove them from pending list if they already
   	 * signaled.
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#21 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 42 lines · Daniel Vetter
Quoted 41 lines · Andrey Grodzovsky
Quoted 29 lines · Daniel Vetter
Quoted 28 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
    drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
    1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
    	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 3 lines
Quoted 1 line

[1] - https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L5041&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C86b39a7bbcd34a18c6e908d96c8cf862%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660174991641911%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=tz7lxvL%2BR6NrpcdfIg1Mjw5lZ55%2F5HTPF%2Fkwu7wqvqE%3D&amp;reserved=0

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

IMHO what is problematic in serializing vs. locking (with trylock and bail out like we do in [1]) is for multiple TO events arising from same reason like maybe one job just waits for another and once first is hanged the second will also appear to be hanged triggering it's own TO event. In this case multiple TOs event will trigger multiple resets if we serialize but if we use lock with trylock the second one will quietly bail out.

Quoted 3 lines
Quoted 1 line

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L4903&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C86b39a7bbcd34a18c6e908d96c8cf862%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660174991651903%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=SpirDOLVdw5kIZAq0LHjnB0Qy6apwPLDPFjm61Wc2ko%3D&amp;reserved=0

Hm so I guess a single wq here, that will hold up all other TO. And they should recheck whether the job is moving meanwhile.

Can you clarify about this ? What job should be moving ? The dependent job ?

Quoted 3 lines

Also unless you use hw semaphores the job shouldn't even start before the deps are singalled, so not sure how this goes wrong?

What about a simple example where
we actually can submit a shader on one ring and a simple WAIT_REG_MEM packet on another to wait for the shader to write
a specific value to specific memory location. Here you have both of them started in close proximity and no explicit dependencies involved (at the scheduler level) and yet if the shader hangs also the WAIT_REG_MEM job will hang.

Quoted 5 lines

The vm_id flush stuff can make things a bit more fun for your specific case, but in your specific case you have to run all TO handlers on the same ordered workqueue anyway (because trying to paper over this in other ways doesn't work imo).

I didn't get this one.

Andrey

Quoted 58 lines

So I think this should all work, no need for tricky cross-scheduler locking.
-Daniel

Quoted 53 lines

Andrey

Quoted 6 lines

See

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdri.freedesktop.org%2Fdocs%2Fdrm%2Fgpu%2Fdrm-mm.html%23c.drm_sched_backend_ops&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C86b39a7bbcd34a18c6e908d96c8cf862%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660174991651903%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=uV4s%2Fsu%2FKabZvsRMd1PAyd36JRSz91aPfYEn8PlvFlM%3D&amp;reserved=0

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Yes, this discussion is a continuation of your comment about serializing, I mentioned before that you proposed it.

Andrey

Quoted 37 lines
Quoted 37 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
    	spin_lock(&sched->job_list_lock);
    	job = list_first_entry_or_null(&sched->pending_list,
    				       struct drm_sched_job, list);
    	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
    		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
    		status = job->sched->ops->timedout_job(job);
    		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    	kthread_park(sched->thread);
    	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
    	 * Iterate the job list from later to  earlier one and either deactive
    	 * their HW callbacks or remove them from pending list if they already
    	 * signaled.
#22 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 62 lines · Andrey Grodzovsky
Quoted 41 lines · Daniel Vetter
Quoted 40 lines · Andrey Grodzovsky
Quoted 29 lines · Daniel Vetter
Quoted 28 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
    drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
    1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
    	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
    	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 3 lines
Quoted 1 line

[1] - https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L5041&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=PrrvFHAwDeLlbcOctlKHsCFs9%2F56XNVzoLVcT1RoJgI%3D&amp;reserved=0

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

IMHO what is problematic in serializing vs. locking (with trylock and bail out like we do in [1]) is for multiple TO events arising from same reason like maybe one job just waits for another and once first is hanged the second will also appear to be hanged triggering it's own TO event. In this case multiple TOs event will trigger multiple resets if we serialize but if we use lock with trylock the second one will quietly bail out.

Quoted 3 lines
Quoted 1 line

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L4903&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=kxSWBoshVTLMMFIFZdPsP4MBgUAoC%2F3szo9GUemSRLY%3D&amp;reserved=0

Hm so I guess a single wq here, that will hold up all other TO. And they should recheck whether the job is moving meanwhile.

Can you clarify about this ? What job should be moving ? The dependent job ?

Quoted 2 lines

Also unless you use hw semaphores the job shouldn't even start before the deps are singalled, so not sure how this goes wrong?

What about a simple example where
we actually can submit a shader on one ring and a simple WAIT_REG_MEM packet on another to wait for the shader to write a specific value to specific memory location. Here you have both of them started
in close proximity and no explicit dependencies involved (at the scheduler level)
and yet if the shader hangs also the WAIT_REG_MEM job will hang.

Quoted 6 lines
Quoted 4 lines

The vm_id flush stuff can make things a bit more fun for your specific case, but in your specific case you have to run all TO handlers on the same ordered workqueue anyway (because trying to paper over this in other ways doesn't work imo).

I didn't get this one.

So, awhile back I tried to "serialize" this by moving timed-out jobs into their own timed-out-dedicated list, then freeing them asynchronously, but I never got it to work reliably due to races with low-level drivers and assumptions made way back.

My idea was to atomic-move timed-out jobs into their own list, at the time of timeout, and later asynchronously to free them (or better yet, inquire about their state, and free them or move them back--ideally the inquiry is atomic and done at timeout time before being moved to the timeout list). Anyway...

But I found out that all these knobs and levers weren't in place and I was getting problems with it and it never materialized.

The paradigm was loosely "let someone else do it", like, "on an event, move it to a list, and let someone else handle it", or "on an event, mark it, and let someone else handle it". (loosely borrowed from an iSCSI target I did many many years ago--it worked well and there were no races, even with out-of-order executions.)

If you guys have any ideas to that end, maybe we can try it out.

Regards,
Luben

Quoted 60 lines

Andrey

Quoted 56 lines

So I think this should all work, no need for tricky cross-scheduler locking.
-Daniel

Quoted 52 lines

Andrey

Quoted 6 lines

See

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdri.freedesktop.org%2Fdocs%2Fdrm%2Fgpu%2Fdrm-mm.html%23c.drm_sched_backend_ops&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=Fpt%2Btho2W4woHKQ861cEbBzoOidS6zuDhFi%2B1UTwWJg%3D&amp;reserved=0

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Yes, this discussion is a continuation of your comment about serializing, I mentioned before that you proposed it.

Andrey

Quoted 37 lines
Quoted 37 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
    	spin_lock(&sched->job_list_lock);
    	job = list_first_entry_or_null(&sched->pending_list,
    				       struct drm_sched_job, list);
    	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
    		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
    		status = job->sched->ops->timedout_job(job);
    		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
    	kthread_park(sched->thread);
    	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
    	 * Iterate the job list from later to  earlier one and either deactive
    	 * their HW callbacks or remove them from pending list if they already
    	 * signaled.
#23 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 85 lines · Luben Tuikov
Quoted 60 lines · Andrey Grodzovsky
Quoted 41 lines · Daniel Vetter
Quoted 40 lines · Andrey Grodzovsky
Quoted 29 lines · Daniel Vetter
Quoted 28 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
     drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
     1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
     	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
     	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 3 lines
Quoted 1 line

[1] - https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L5041&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=PrrvFHAwDeLlbcOctlKHsCFs9%2F56XNVzoLVcT1RoJgI%3D&amp;reserved=0

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

IMHO what is problematic in serializing vs. locking (with trylock and bail out like we do in [1]) is for multiple TO events arising from same reason like maybe one job just waits for another and once first is hanged the second will also appear to be hanged triggering it's own TO event. In this case multiple TOs event will trigger multiple resets if we serialize but if we use lock with trylock the second one will quietly bail out.

Quoted 3 lines
Quoted 1 line

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L4903&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=kxSWBoshVTLMMFIFZdPsP4MBgUAoC%2F3szo9GUemSRLY%3D&amp;reserved=0

Hm so I guess a single wq here, that will hold up all other TO. And they should recheck whether the job is moving meanwhile.

Can you clarify about this ? What job should be moving ? The dependent job ?

Quoted 2 lines

Also unless you use hw semaphores the job shouldn't even start before the deps are singalled, so not sure how this goes wrong?

What about a simple example where
we actually can submit a shader on one ring and a simple WAIT_REG_MEM packet on another to wait for the shader to write a specific value to specific memory location. Here you have both of them started
in close proximity and no explicit dependencies involved (at the scheduler level)
and yet if the shader hangs also the WAIT_REG_MEM job will hang.

Quoted 5 lines
Quoted 4 lines

The vm_id flush stuff can make things a bit more fun for your specific case, but in your specific case you have to run all TO handlers on the same ordered workqueue anyway (because trying to paper over this in other ways doesn't work imo).

I didn't get this one.

So, awhile back I tried to "serialize" this by moving timed-out jobs into their own timed-out-dedicated list, then freeing them asynchronously, but I never got it to work reliably due to races with low-level drivers and assumptions made way back.

My idea was to atomic-move timed-out jobs into their own list, at the time of timeout, and later asynchronously to free them (or better yet, inquire about their state, and free them or move them back--ideally the inquiry is atomic and done at timeout time before being moved to the timeout list). Anyway...

But I found out that all these knobs and levers weren't in place and I was getting problems with it and it never materialized.

The paradigm was loosely "let someone else do it", like, "on an event, move it to a list, and let someone else handle it", or "on an event, mark it, and let someone else handle it". (loosely borrowed from an iSCSI target I did many many years ago--it worked well and there were no races, even with out-of-order executions.)

Quoted 4 lines

If you guys have any ideas to that end, maybe we can try it out.

Regards,
Luben

I wonder if we really need this serialization at all, if we do HW fence embedding at the drm_sched_job level instead of doing it only for amdgpu, and modifying all the drivers to support this we can both remove this hack and solve the race against concurrent drm_sched_cleanup_jobs job freeing just by taking reference to the hw fence of the job at the beginning of drm_sched_job_timedout

Andrey

Quoted 61 lines
Quoted 61 lines

Andrey

Quoted 56 lines

So I think this should all work, no need for tricky cross-scheduler locking.
-Daniel

Quoted 52 lines

Andrey

Quoted 6 lines

See

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdri.freedesktop.org%2Fdocs%2Fdrm%2Fgpu%2Fdrm-mm.html%23c.drm_sched_backend_ops&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=Fpt%2Btho2W4woHKQ861cEbBzoOidS6zuDhFi%2B1UTwWJg%3D&amp;reserved=0

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Yes, this discussion is a continuation of your comment about serializing, I mentioned before that you proposed it.

Andrey

Quoted 37 lines
Quoted 37 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
     	spin_lock(&sched->job_list_lock);
     	job = list_first_entry_or_null(&sched->pending_list,
     				       struct drm_sched_job, list);
     	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
     		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
     		status = job->sched->ops->timedout_job(job);
     		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
     	kthread_park(sched->thread);
     	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
     	 * Iterate the job list from later to  earlier one and either deactive
     	 * their HW callbacks or remove them from pending list if they already
     	 * signaled.
#24 lore raw

Re: [PATCH 2/2] drm/sched: serialize job_timeout and scheduler

Quoted 100 lines · Andrey Grodzovsky
Quoted 85 lines · Luben Tuikov
Quoted 60 lines · Andrey Grodzovsky
Quoted 41 lines · Daniel Vetter
Quoted 40 lines · Andrey Grodzovsky
Quoted 29 lines · Daniel Vetter
Quoted 28 lines · Andrey Grodzovsky

It's says patch [2/2] but i can't find patch 1

Quoted 12 lines · Monk Liu
---
     drivers/gpu/drm/scheduler/sched_main.c | 24 ++++--------------------
     1 file changed, 4 insertions(+), 20 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +0 -0
index ecf8140..894fdb24 100644

drivers/gpu/drm/scheduler/sched_main.c static void drm_sched_job_timedout(struct work_struct *work) · line 323

Quoted 5 lines
drivers/gpu/drm/scheduler/sched_main.c +3 -0
@@ -319,19 +319,17 @@ static void drm_sched_job_timedout(struct work_struct *work)
     	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
     	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	if (!__kthread_should_park(sched->thread))
+		kthread_park(sched->thread);
+

As mentioned before, without serializing against other TDR handlers from other
schedulers you just race here against them, e.g. you parked it now but another
one in progress will unpark it as part of calling  drm_sched_start for other rings[1]
Unless I am missing something since I haven't found patch [1/2]

Quoted 3 lines
Quoted 1 line

[1] - https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L5041&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=PrrvFHAwDeLlbcOctlKHsCFs9%2F56XNVzoLVcT1RoJgI%3D&amp;reserved=0

You need to have your own wq and run all your tdr work on the same wq if your reset has any cross-engine impact.

IMHO what is problematic in serializing vs. locking (with trylock and bail out like we do in [1]) is for multiple TO events arising from same reason like maybe one job just waits for another and once first is hanged the second will also appear to be hanged triggering it's own TO event. In this case multiple TOs event will trigger multiple resets if we serialize but if we use lock with trylock the second one will quietly bail out.

Quoted 3 lines
Quoted 1 line

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Felixir.bootlin.com%2Flinux%2Flatest%2Fsource%2Fdrivers%2Fgpu%2Fdrm%2Famd%2Famdgpu%2Famdgpu_device.c%23L4903&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=kxSWBoshVTLMMFIFZdPsP4MBgUAoC%2F3szo9GUemSRLY%3D&amp;reserved=0

Hm so I guess a single wq here, that will hold up all other TO. And they should recheck whether the job is moving meanwhile.

Can you clarify about this ? What job should be moving ? The dependent job ?

Quoted 2 lines

Also unless you use hw semaphores the job shouldn't even start before the deps are singalled, so not sure how this goes wrong?

What about a simple example where
we actually can submit a shader on one ring and a simple WAIT_REG_MEM packet on another to wait for the shader to write a specific value to specific memory location. Here you have both of them started
in close proximity and no explicit dependencies involved (at the scheduler level)
and yet if the shader hangs also the WAIT_REG_MEM job will hang.

Quoted 5 lines
Quoted 4 lines

The vm_id flush stuff can make things a bit more fun for your specific case, but in your specific case you have to run all TO handlers on the same ordered workqueue anyway (because trying to paper over this in other ways doesn't work imo).

I didn't get this one.

So, awhile back I tried to "serialize" this by moving timed-out jobs into their own timed-out-dedicated list, then freeing them asynchronously, but I never got it to work reliably due to races with low-level drivers and assumptions made way back.

My idea was to atomic-move timed-out jobs into their own list, at the time of timeout, and later asynchronously to free them (or better yet, inquire about their state, and free them or move them back--ideally the inquiry is atomic and done at timeout time before being moved to the timeout list). Anyway...

But I found out that all these knobs and levers weren't in place and I was getting problems with it and it never materialized.

The paradigm was loosely "let someone else do it", like, "on an event, move it to a list, and let someone else handle it", or "on an event, mark it, and let someone else handle it". (loosely borrowed from an iSCSI target I did many many years ago--it worked well and there were no races, even with out-of-order executions.)

Quoted 4 lines

If you guys have any ideas to that end, maybe we can try it out.

Regards,
Luben

I wonder if we really need this serialization at all, if we do HW fence embedding
at the drm_sched_job level instead of doing it only for amdgpu, and modifying all
the drivers to support this we can both remove this hack and solve the race against concurrent drm_sched_cleanup_jobs job freeing just by taking reference
to the hw fence of the job at the beginning of drm_sched_job_timedout

Quoted 1 line

Andrey

This sounds like the right approach to me.

(Convincing the low-level drivers of this might might be a big task.)

Regards,
Luben

Quoted 62 lines
Quoted 62 lines
Quoted 62 lines

Andrey

Quoted 56 lines

So I think this should all work, no need for tricky cross-scheduler locking.
-Daniel

Quoted 52 lines

Andrey

Quoted 6 lines

See

https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdri.freedesktop.org%2Fdocs%2Fdrm%2Fgpu%2Fdrm-mm.html%23c.drm_sched_backend_ops&amp;data=04%7C01%7Cluben.tuikov%40amd.com%7C228bd1600c914efe24aa08d96c934bbb%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637660202148713283%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=Fpt%2Btho2W4woHKQ861cEbBzoOidS6zuDhFi%2B1UTwWJg%3D&amp;reserved=0

for the ->timeout_job callback docs. I thought I brought this up already? -Daniel

Yes, this discussion is a continuation of your comment about serializing, I mentioned before that you proposed it.

Andrey

Quoted 37 lines
Quoted 37 lines

Andrey

Quoted 34 lines
drivers/gpu/drm/scheduler/sched_main.c +1 -20
     	spin_lock(&sched->job_list_lock);
     	job = list_first_entry_or_null(&sched->pending_list,
     				       struct drm_sched_job, list);
     	if (job) {
-		/*
-		 * Remove the bad job so it cannot be freed by concurrent
-		 * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
-		 * is parked at which point it's safe.
-		 */
-		list_del_init(&job->list);
     		spin_unlock(&sched->job_list_lock);
+		/* vendor's timeout_job should call drm_sched_start() */
     		status = job->sched->ops->timedout_job(job);
     		/*
@@ -393,20 +391,6 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
     	kthread_park(sched->thread);
     	/*
-	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
-	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
-	 * now until the scheduler thread is unparked.
-	 */
-	if (bad && bad->sched == sched)
-		/*
-		 * Add at the head of the queue to reflect it was the earliest
-		 * job extracted.
-		 */
-		list_add(&bad->list, &sched->pending_list);
-
-	/*
     	 * Iterate the job list from later to  earlier one and either deactive
     	 * their HW callbacks or remove them from pending list if they already
     	 * signaled.
author Tested-by jingwen on v1 1/2
#25 lore raw
In reply to Monk Liu

[AMD Official Use Only]

Ping Christian, Andrey

Can we merge this patch first ? this is a standalone patch for the timer

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Monk Liu <[email protected]>
Sent: Tuesday, August 31, 2021 6:36 PM
To: [email protected]
Cc: [email protected]; Liu, Monk <[email protected]>
Subject: [PATCH 1/2] drm/sched: fix the bug of time out calculation(v3)

issue:
in cleanup_job the cancle_delayed_work will cancel a TO timer even the its corresponding job is still running.

fix:
do not cancel the timer in cleanup_job, instead do the cancelling only when the heading job is signaled, and if there is a "next" job we start_timeout again.

v2:
further cleanup the logic, and do the TDR timer cancelling if the signaled job is the last one in its scheduler.

v3:
change the issue description
remove the cancel_delayed_work in the begining of the cleanup_job recover the implement of drm_sched_job_begin.

TODO:
1)introduce pause/resume scheduler in job_timeout to serial the handling of scheduler and job_timeout. 2)drop the bad job's del and insert in scheduler due to above serialization (no race issue anymore with the serialization)

---
 drivers/gpu/drm/scheduler/sched_main.c | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +10 -15
index a2a9536..ecf8140 100644
@@ -676,13 +676,7 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)  {
 	struct drm_sched_job *job, *next;
 
-	/*
-	 * Don't destroy jobs while the timeout worker is running  OR thread
-	 * is being parked and hence assumed to not touch pending_list
-	 */
-	if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
-	    !cancel_delayed_work(&sched->work_tdr)) ||
-	    kthread_should_park())
+	if (kthread_should_park())
 		return NULL;
 
 	spin_lock(&sched->job_list_lock);
@@ -693,17 +687,21 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 	if (job && dma_fence_is_signaled(&job->s_fence->finished)) {
 		/* remove job from pending_list */
 		list_del_init(&job->list);
+
+		/* cancel this job's TO timer */
+		cancel_delayed_work(&sched->work_tdr);
 		/* make the scheduled timestamp more accurate */
 		next = list_first_entry_or_null(&sched->pending_list,
 						typeof(*next), list);
-		if (next)
+
+		if (next) {
 			next->s_fence->scheduled.timestamp =
 				job->s_fence->finished.timestamp;
-
+			/* start TO timer for next job */
+			drm_sched_start_timeout(sched);
+		}
 	} else {
 		job = NULL;
-		/* queue timeout for next job */
-		drm_sched_start_timeout(sched);
 	}
 
 	spin_unlock(&sched->job_list_lock);
@@ -791,11 +789,8 @@ static int drm_sched_main(void *param)
 					  (entity = drm_sched_select_entity(sched))) ||
 					 kthread_should_stop());
 
-		if (cleanup_job) {
+		if (cleanup_job)
 			sched->ops->free_job(cleanup_job);
-			/* queue timeout for next job */
-			drm_sched_start_timeout(sched);
-		}
 
 		if (!entity)
 			continue;

--
2.7.4

Tested-by jingwen
#26 lore raw

What about removing

(kthread_should_park()) ? We decided it's useless as far as I remember.

Andrey
________________________________
From: amd-gfx <[email protected]> on behalf of Liu, Monk <[email protected]>
Sent: 31 August 2021 20:24
To: Liu, Monk <[email protected]>; [email protected] <[email protected]>
Cc: [email protected] <[email protected]>
Subject: RE: [PATCH 1/2] drm/sched: fix the bug of time out calculation(v3)

[AMD Official Use Only]

Ping Christian, Andrey

Can we merge this patch first ? this is a standalone patch for the timer

Thanks

------------------------------------------
Monk Liu | Cloud-GPU Core team
------------------------------------------

-----Original Message-----
From: Monk Liu <[email protected]>
Sent: Tuesday, August 31, 2021 6:36 PM
To: [email protected]
Cc: [email protected]; Liu, Monk <[email protected]>
Subject: [PATCH 1/2] drm/sched: fix the bug of time out calculation(v3)

issue:
in cleanup_job the cancle_delayed_work will cancel a TO timer even the its corresponding job is still running.

fix:
do not cancel the timer in cleanup_job, instead do the cancelling only when the heading job is signaled, and if there is a "next" job we start_timeout again.

v2:
further cleanup the logic, and do the TDR timer cancelling if the signaled job is the last one in its scheduler.

v3:
change the issue description
remove the cancel_delayed_work in the begining of the cleanup_job recover the implement of drm_sched_job_begin.

TODO:
1)introduce pause/resume scheduler in job_timeout to serial the handling of scheduler and job_timeout. 2)drop the bad job's del and insert in scheduler due to above serialization (no race issue anymore with the serialization)

---
 drivers/gpu/drm/scheduler/sched_main.c | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)
drivers/gpu/drm/scheduler/sched_main.c +10 -15
index a2a9536..ecf8140 100644
@@ -676,13 +676,7 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)  {
         struct drm_sched_job *job, *next;
 
-       /*
-        * Don't destroy jobs while the timeout worker is running  OR thread
-        * is being parked and hence assumed to not touch pending_list
-        */
-       if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
-           !cancel_delayed_work(&sched->work_tdr)) ||
-           kthread_should_park())
+       if (kthread_should_park())
                 return NULL;
 
         spin_lock(&sched->job_list_lock);
@@ -693,17 +687,21 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
         if (job && dma_fence_is_signaled(&job->s_fence->finished)) {
                 /* remove job from pending_list */
                 list_del_init(&job->list);
+
+               /* cancel this job's TO timer */
+               cancel_delayed_work(&sched->work_tdr);
                 /* make the scheduled timestamp more accurate */
                 next = list_first_entry_or_null(&sched->pending_list,
                                                 typeof(*next), list);
-               if (next)
+
+               if (next) {
                         next->s_fence->scheduled.timestamp =
                                 job->s_fence->finished.timestamp;
-
+                       /* start TO timer for next job */
+                       drm_sched_start_timeout(sched);
+               }
         } else {
                 job = NULL;
-               /* queue timeout for next job */
-               drm_sched_start_timeout(sched);
         }
 
         spin_unlock(&sched->job_list_lock);
@@ -791,11 +789,8 @@ static int drm_sched_main(void *param)
                                           (entity = drm_sched_select_entity(sched))) ||
                                          kthread_should_stop());
 
-               if (cleanup_job) {
+               if (cleanup_job)
                         sched->ops->free_job(cleanup_job);
-                       /* queue timeout for next job */
-                       drm_sched_start_timeout(sched);
-               }
 
                 if (!entity)
                         continue;

--
2.7.4