Skip to contentSkip to conversation

drm/msm/gpu: More system suspend fixes

PATCH In review Rob Clark 8 participants 27 messages Mar 10, 2022 – Mar 18, 2022

Subsystems (4)
lore
Share & tools
lore

Rob Clark

author
#0 lore raw

cover drm/msm/gpu: More system suspend fixes

From: Rob Clark <[email protected]>

In particular, we want to park the scheduler threads so that suspend is not racing with the kthread pushing more jobs to the driver.

Rob Clark (3):
drm/msm/gpu: Rename runtime suspend/resume functions drm/msm/gpu: Park scheduler threads for system suspend drm/msm/gpu: Remove mutex from wait_event condition

 drivers/gpu/drm/msm/adreno/adreno_device.c | 79 ++++++++++++++++++----
 1 file changed, 65 insertions(+), 14 deletions(-)
Signature · 1 line
2.35.1
author
#1 lore raw

PATCH 1/3 drm/msm/gpu: Rename runtime suspend/resume functions

From: Rob Clark <[email protected]>

---
 drivers/gpu/drm/msm/adreno/adreno_device.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +3 -3
index 89cfd84760d7..8859834b51b8 100644
@@ -600,7 +600,7 @@ static const struct of_device_id dt_match[] = {
 };
 
 #ifdef CONFIG_PM
-static int adreno_resume(struct device *dev)
+static int adreno_runtime_resume(struct device *dev)
 {
 	struct msm_gpu *gpu = dev_to_gpu(dev);
 
@@ -616,7 +616,7 @@ static int active_submits(struct msm_gpu *gpu)
 	return active_submits;
 }
 
-static int adreno_suspend(struct device *dev)
+static int adreno_runtime_suspend(struct device *dev)
 {
 	struct msm_gpu *gpu = dev_to_gpu(dev);
 	int remaining;
@@ -635,7 +635,7 @@ static int adreno_suspend(struct device *dev)
 
 static const struct dev_pm_ops adreno_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
-	SET_RUNTIME_PM_OPS(adreno_suspend, adreno_resume, NULL)
+	SET_RUNTIME_PM_OPS(adreno_runtime_suspend, adreno_runtime_resume, NULL)
 };
 
 static struct platform_driver adreno_driver = {
Signature · 1 line
2.35.1
#2 lore raw

Re: [PATCH 1/3] drm/msm/gpu: Rename runtime suspend/resume functions

Il 11/03/22 00:46, Rob Clark ha scritto:

Quoted 2 lines

From: Rob Clark <[email protected]>

Hey Rob,
looks like you've somehow lost the commit description on this one!

Cheers,
Angelo

Quoted 5 lines
---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 6 +++---
  1 file changed, 3 insertions(+), 3 deletions(-)
author
#3 lore raw

PATCH 2/3 drm/msm/gpu: Park scheduler threads for system suspend

In reply to Rob Clark

From: Rob Clark <[email protected]>

In the system suspend path, we don't want to be racing with the scheduler kthreads pushing additional queued up jobs to the hw queue (ringbuffer). So park them first. While we are at it, move the wait for active jobs to complete into the new system- suspend path.

---
 drivers/gpu/drm/msm/adreno/adreno_device.c | 68 ++++++++++++++++++++--
 1 file changed, 64 insertions(+), 4 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +64 -4
index 8859834b51b8..0440a98988fc 100644
@@ -619,22 +619,82 @@ static int active_submits(struct msm_gpu *gpu)
 static int adreno_runtime_suspend(struct device *dev)
 {
 	struct msm_gpu *gpu = dev_to_gpu(dev);
-	int remaining;
+
+	/*
+	 * We should be holding a runpm ref, which will prevent
+	 * runtime suspend.  In the system suspend path, we've
+	 * already waited for active jobs to complete.
+	 */
+	WARN_ON_ONCE(gpu->active_submits);
+
+	return gpu->funcs->pm_suspend(gpu);
+}
+
+static void suspend_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	/*
+	 * Shut down the scheduler before we force suspend, so that
+	 * suspend isn't racing with scheduler kthread feeding us
+	 * more work.
+	 *
+	 * Note, we just want to park the thread, and let any jobs
+	 * that are already on the hw queue complete normally, as
+	 * opposed to the drm_sched_stop() path used for handling
+	 * faulting/timed-out jobs.  We can't really cancel any jobs
+	 * already on the hw queue without racing with the GPU.
+	 */
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_park(sched->thread);
+	}
+}
+
+static void resume_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_unpark(sched->thread);
+	}
+}
+
+static int adreno_system_suspend(struct device *dev)
+{
+	struct msm_gpu *gpu = dev_to_gpu(dev);
+	int remaining, ret;
+
+	suspend_scheduler(gpu);
 
 	remaining = wait_event_timeout(gpu->retire_event,
 				       active_submits(gpu) == 0,
 				       msecs_to_jiffies(1000));
 	if (remaining == 0) {
 		dev_err(dev, "Timeout waiting for GPU to suspend\n");
-		return -EBUSY;
+		ret = -EBUSY;
+		goto out;
 	}
 
-	return gpu->funcs->pm_suspend(gpu);
+	ret = pm_runtime_force_suspend(dev);
+out:
+	if (ret)
+		resume_scheduler(gpu);
+
+	return ret;
 }
+
+static int adreno_system_resume(struct device *dev)
+{
+	resume_scheduler(dev_to_gpu(dev));
+	return pm_runtime_force_resume(dev);
+}
+
 #endif
 
 static const struct dev_pm_ops adreno_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(adreno_system_suspend, adreno_system_resume)
 	SET_RUNTIME_PM_OPS(adreno_runtime_suspend, adreno_runtime_resume, NULL)
 };
 
Signature · 1 line
2.35.1
#4 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 41 lines · Rob Clark

From: Rob Clark <[email protected]>

In the system suspend path, we don't want to be racing with the scheduler kthreads pushing additional queued up jobs to the hw queue (ringbuffer). So park them first. While we are at it, move the wait for active jobs to complete into the new system- suspend path.

---
 drivers/gpu/drm/msm/adreno/adreno_device.c | 68 ++++++++++++++++++++--
 1 file changed, 64 insertions(+), 4 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +19 -1
index 8859834b51b8..0440a98988fc 100644
@@ -619,22 +619,82 @@ static int active_submits(struct msm_gpu *gpu)
 static int adreno_runtime_suspend(struct device *dev)
 {
 	struct msm_gpu *gpu = dev_to_gpu(dev);
-	int remaining;
+
+	/*
+	 * We should be holding a runpm ref, which will prevent
+	 * runtime suspend.  In the system suspend path, we've
+	 * already waited for active jobs to complete.
+	 */
+	WARN_ON_ONCE(gpu->active_submits);
+
+	return gpu->funcs->pm_suspend(gpu);
+}
+
+static void suspend_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	/*
+	 * Shut down the scheduler before we force suspend, so that
+	 * suspend isn't racing with scheduler kthread feeding us
+	 * more work.

drivers/gpu/drm/msm/adreno/adreno_device.c static int active_submits(struct msm_gpu *gpu) · line 650

Quoted 10 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +10 -0
+	 *
+	 * Note, we just want to park the thread, and let any jobs
+	 * that are already on the hw queue complete normally, as
+	 * opposed to the drm_sched_stop() path used for handling
+	 * faulting/timed-out jobs.  We can't really cancel any jobs
+	 * already on the hw queue without racing with the GPU.
+	 */
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_park(sched->thread);

Shouldn't we have some proper interfaces for this? Also I'm kinda wondering how other drivers do this, feels like we should have a standard way.

Finally not flushing out all in-flight requests sounds a bit like a bad idea for system suspend/resume since that's also the hibernation path, and that would mean your shrinker/page reclaim stops working. At least in full generality. Which ain't good for hibernation.

Adding Christian and Andrey.
-Daniel

Quoted 56 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +35 -3
+	}
+}
+
+static void resume_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_unpark(sched->thread);
+	}
+}
+
+static int adreno_system_suspend(struct device *dev)
+{
+	struct msm_gpu *gpu = dev_to_gpu(dev);
+	int remaining, ret;
+
+	suspend_scheduler(gpu);
 
 	remaining = wait_event_timeout(gpu->retire_event,
 				       active_submits(gpu) == 0,
 				       msecs_to_jiffies(1000));
 	if (remaining == 0) {
 		dev_err(dev, "Timeout waiting for GPU to suspend\n");
-		return -EBUSY;
+		ret = -EBUSY;
+		goto out;
 	}
 
-	return gpu->funcs->pm_suspend(gpu);
+	ret = pm_runtime_force_suspend(dev);
+out:
+	if (ret)
+		resume_scheduler(gpu);
+
+	return ret;
 }
+
+static int adreno_system_resume(struct device *dev)
+{
+	resume_scheduler(dev_to_gpu(dev));
+	return pm_runtime_force_resume(dev);
+}
+
 #endif
 
 static const struct dev_pm_ops adreno_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(adreno_system_suspend, adreno_system_resume)
 	SET_RUNTIME_PM_OPS(adreno_runtime_suspend, adreno_runtime_resume, NULL)
 };
 
Signature · 1 line
2.35.1
Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
#5 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Am 17.03.22 um 10:59 schrieb Daniel Vetter:

Quoted 45 lines · Rob Clark
Quoted 44 lines

From: Rob Clark <[email protected]>

In the system suspend path, we don't want to be racing with the scheduler kthreads pushing additional queued up jobs to the hw queue (ringbuffer). So park them first. While we are at it, move the wait for active jobs to complete into the new system- suspend path.

---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 68 ++++++++++++++++++++--
  1 file changed, 64 insertions(+), 4 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +22 -1
index 8859834b51b8..0440a98988fc 100644
@@ -619,22 +619,82 @@ static int active_submits(struct msm_gpu *gpu)
  static int adreno_runtime_suspend(struct device *dev)
  {
  	struct msm_gpu *gpu = dev_to_gpu(dev);
-	int remaining;
+
+	/*
+	 * We should be holding a runpm ref, which will prevent
+	 * runtime suspend.  In the system suspend path, we've
+	 * already waited for active jobs to complete.
+	 */
+	WARN_ON_ONCE(gpu->active_submits);
+
+	return gpu->funcs->pm_suspend(gpu);
+}
+
+static void suspend_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	/*
+	 * Shut down the scheduler before we force suspend, so that
+	 * suspend isn't racing with scheduler kthread feeding us
+	 * more work.
+	 *
+	 * Note, we just want to park the thread, and let any jobs
+	 * that are already on the hw queue complete normally, as
Quoted 8 lines

drivers/gpu/drm/msm/adreno/adreno_device.c static int active_submits(struct msm_gpu *gpu) · line 650

Quoted 7 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +7 -0
+	 * opposed to the drm_sched_stop() path used for handling
+	 * faulting/timed-out jobs.  We can't really cancel any jobs
+	 * already on the hw queue without racing with the GPU.
+	 */
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_park(sched->thread);

Shouldn't we have some proper interfaces for this?

If I'm not completely mistaken we already should have one, yes.

Quoted 7 lines

Also I'm kinda wondering how other drivers do this, feels like we should have a standard way.

Finally not flushing out all in-flight requests sounds a bit like a bad idea for system suspend/resume since that's also the hibernation path, and that would mean your shrinker/page reclaim stops working. At least in full generality. Which ain't good for hibernation.

Completely agree, that looks like an incorrect workaround to me.

During suspend all userspace applications should be frozen and all f their hardware activity flushed out and waited for completion.

I do remember that our internal guys came up with pretty much the same idea and it sounded broken to me back then as well.

Regards,
Christian.

Quoted 60 lines

Adding Christian and Andrey.
-Daniel

Quoted 56 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +35 -3
+	}
+}
+
+static void resume_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_unpark(sched->thread);
+	}
+}
+
+static int adreno_system_suspend(struct device *dev)
+{
+	struct msm_gpu *gpu = dev_to_gpu(dev);
+	int remaining, ret;
+
+	suspend_scheduler(gpu);
 
  	remaining = wait_event_timeout(gpu->retire_event,
  				       active_submits(gpu) == 0,
  				       msecs_to_jiffies(1000));
  	if (remaining == 0) {
  		dev_err(dev, "Timeout waiting for GPU to suspend\n");
-		return -EBUSY;
+		ret = -EBUSY;
+		goto out;
  	}
 
-	return gpu->funcs->pm_suspend(gpu);
+	ret = pm_runtime_force_suspend(dev);
+out:
+	if (ret)
+		resume_scheduler(gpu);
+
+	return ret;
  }
+
+static int adreno_system_resume(struct device *dev)
+{
+	resume_scheduler(dev_to_gpu(dev));
+	return pm_runtime_force_resume(dev);
+}
+
  #endif
 
  static const struct dev_pm_ops adreno_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(adreno_system_suspend, adreno_system_resume)
  	SET_RUNTIME_PM_OPS(adreno_runtime_suspend, adreno_runtime_resume, NULL)
  };
 
Signature · 1 line
2.35.1
#6 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 65 lines · Christian König

Am 17.03.22 um 10:59 schrieb Daniel Vetter:

Quoted 45 lines · Rob Clark
Quoted 44 lines

From: Rob Clark <[email protected]>

In the system suspend path, we don't want to be racing with the scheduler kthreads pushing additional queued up jobs to the hw queue (ringbuffer). So park them first. While we are at it, move the wait for active jobs to complete into the new system- suspend path.

---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 68 ++++++++++++++++++++--
  1 file changed, 64 insertions(+), 4 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +22 -1
index 8859834b51b8..0440a98988fc 100644
@@ -619,22 +619,82 @@ static int active_submits(struct msm_gpu *gpu)
  static int adreno_runtime_suspend(struct device *dev)
  {
  	struct msm_gpu *gpu = dev_to_gpu(dev);
-	int remaining;
+
+	/*
+	 * We should be holding a runpm ref, which will prevent
+	 * runtime suspend.  In the system suspend path, we've
+	 * already waited for active jobs to complete.
+	 */
+	WARN_ON_ONCE(gpu->active_submits);
+
+	return gpu->funcs->pm_suspend(gpu);
+}
+
+static void suspend_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	/*
+	 * Shut down the scheduler before we force suspend, so that
+	 * suspend isn't racing with scheduler kthread feeding us
+	 * more work.
+	 *
+	 * Note, we just want to park the thread, and let any jobs
+	 * that are already on the hw queue complete normally, as
Quoted 8 lines

drivers/gpu/drm/msm/adreno/adreno_device.c static int active_submits(struct msm_gpu *gpu) · line 650

Quoted 7 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +7 -0
+	 * opposed to the drm_sched_stop() path used for handling
+	 * faulting/timed-out jobs.  We can't really cancel any jobs
+	 * already on the hw queue without racing with the GPU.
+	 */
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_park(sched->thread);

Shouldn't we have some proper interfaces for this?

If I'm not completely mistaken we already should have one, yes.

Quoted 7 lines

Also I'm kinda wondering how other drivers do this, feels like we should have a standard way.

Finally not flushing out all in-flight requests sounds a bit like a bad idea for system suspend/resume since that's also the hibernation path, and that would mean your shrinker/page reclaim stops working. At least in full generality. Which ain't good for hibernation.

Quoted 5 lines

Completely agree, that looks like an incorrect workaround to me.

During suspend all userspace applications should be frozen and all f their hardware activity flushed out and waited for completion.

Isn't that what Rob is doing?

He kills the scheduler preventing any new job from being submitted then waits for an outstanding jobs to complete naturally complete (see the wait_event_timeout below). If the jobs don't naturally complete the suspend seems to be aborted? That flow makes sense to me and seems like a novel way to avoid races.

Matt

Quoted 67 lines

I do remember that our internal guys came up with pretty much the same idea and it sounded broken to me back then as well.

Regards,
Christian.

Quoted 60 lines

Adding Christian and Andrey.
-Daniel

Quoted 56 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +35 -3
+	}
+}
+
+static void resume_scheduler(struct msm_gpu *gpu)
+{
+	int i;
+
+	for (i = 0; i < gpu->nr_rings; i++) {
+		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+		kthread_unpark(sched->thread);
+	}
+}
+
+static int adreno_system_suspend(struct device *dev)
+{
+	struct msm_gpu *gpu = dev_to_gpu(dev);
+	int remaining, ret;
+
+	suspend_scheduler(gpu);
 
  	remaining = wait_event_timeout(gpu->retire_event,
  				       active_submits(gpu) == 0,
  				       msecs_to_jiffies(1000));
  	if (remaining == 0) {
  		dev_err(dev, "Timeout waiting for GPU to suspend\n");
-		return -EBUSY;
+		ret = -EBUSY;
+		goto out;
  	}
 
-	return gpu->funcs->pm_suspend(gpu);
+	ret = pm_runtime_force_suspend(dev);
+out:
+	if (ret)
+		resume_scheduler(gpu);
+
+	return ret;
  }
+
+static int adreno_system_resume(struct device *dev)
+{
+	resume_scheduler(dev_to_gpu(dev));
+	return pm_runtime_force_resume(dev);
+}
+
  #endif
 
  static const struct dev_pm_ops adreno_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(adreno_system_suspend, adreno_system_resume)
  	SET_RUNTIME_PM_OPS(adreno_runtime_suspend, adreno_runtime_resume, NULL)
  };
 
Signature · 1 line
2.35.1
author
#7 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

In reply to Christian König
Quoted 56 lines · Christian König

Am 17.03.22 um 10:59 schrieb Daniel Vetter:

Quoted 45 lines · Rob Clark
Quoted 44 lines

From: Rob Clark <[email protected]>

In the system suspend path, we don't want to be racing with the scheduler kthreads pushing additional queued up jobs to the hw queue (ringbuffer). So park them first. While we are at it, move the wait for active jobs to complete into the new system- suspend path.

---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 68 ++++++++++++++++++++--
  1 file changed, 64 insertions(+), 4 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +22 -1
index 8859834b51b8..0440a98988fc 100644
@@ -619,22 +619,82 @@ static int active_submits(struct msm_gpu *gpu)
  static int adreno_runtime_suspend(struct device *dev)
  {
     struct msm_gpu *gpu = dev_to_gpu(dev);
-    int remaining;
+
+    /*
+     * We should be holding a runpm ref, which will prevent
+     * runtime suspend.  In the system suspend path, we've
+     * already waited for active jobs to complete.
+     */
+    WARN_ON_ONCE(gpu->active_submits);
+
+    return gpu->funcs->pm_suspend(gpu);
+}
+
+static void suspend_scheduler(struct msm_gpu *gpu)
+{
+    int i;
+
+    /*
+     * Shut down the scheduler before we force suspend, so that
+     * suspend isn't racing with scheduler kthread feeding us
+     * more work.
+     *
+     * Note, we just want to park the thread, and let any jobs
+     * that are already on the hw queue complete normally, as
Quoted 8 lines

drivers/gpu/drm/msm/adreno/adreno_device.c static int active_submits(struct msm_gpu *gpu) · line 650

Quoted 7 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +7 -0
+     * opposed to the drm_sched_stop() path used for handling
+     * faulting/timed-out jobs.  We can't really cancel any jobs
+     * already on the hw queue without racing with the GPU.
+     */
+    for (i = 0; i < gpu->nr_rings; i++) {
+            struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+            kthread_park(sched->thread);

Shouldn't we have some proper interfaces for this?

Quoted 1 line

If I'm not completely mistaken we already should have one, yes.

drm_sched_stop() was my first thought, but it carries extra baggage. Really I *just* want to park the kthread.

Note that amdgpu does (for afaict different reasons) park the kthread directly as well.

Quoted 2 lines
Quoted 2 lines

Also I'm kinda wondering how other drivers do this, feels like we should have a standard way.

As far as other drivers, it seems like they largely ignore it. I suspect other drivers also have problems in this area.

Fwiw, I have a piglit test to try to exercise this path if you want to try it on other drivers.. might need some futzing around to make sure enough work is queued up that there is some on hw ring and some queued up in the scheduler when you try to suspend.

https://gitlab.freedesktop.org/mesa/piglit/-/merge_requests/643

Quoted 6 lines
Quoted 5 lines

Finally not flushing out all in-flight requests sounds a bit like a bad idea for system suspend/resume since that's also the hibernation path, and that would mean your shrinker/page reclaim stops working. At least in full generality. Which ain't good for hibernation.

Quoted 7 lines

Completely agree, that looks like an incorrect workaround to me.

During suspend all userspace applications should be frozen and all f their hardware activity flushed out and waited for completion.

I do remember that our internal guys came up with pretty much the same idea and it sounded broken to me back then as well.

userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

BR,
-R

Quoted 64 lines

Regards,
Christian.

Quoted 60 lines

Adding Christian and Andrey.
-Daniel

Quoted 56 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +35 -4
+    }
+}
+
+static void resume_scheduler(struct msm_gpu *gpu)
+{
+    int i;
+
+    for (i = 0; i < gpu->nr_rings; i++) {
+            struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
+            kthread_unpark(sched->thread);
+    }
+}
+
+static int adreno_system_suspend(struct device *dev)
+{
+    struct msm_gpu *gpu = dev_to_gpu(dev);
+    int remaining, ret;
+
+    suspend_scheduler(gpu);
 
     remaining = wait_event_timeout(gpu->retire_event,
                                    active_submits(gpu) == 0,
                                    msecs_to_jiffies(1000));
     if (remaining == 0) {
             dev_err(dev, "Timeout waiting for GPU to suspend\n");
-            return -EBUSY;
+            ret = -EBUSY;
+            goto out;
     }
 
-    return gpu->funcs->pm_suspend(gpu);
+    ret = pm_runtime_force_suspend(dev);
+out:
+    if (ret)
+            resume_scheduler(gpu);
+
+    return ret;
  }
+
+static int adreno_system_resume(struct device *dev)
+{
+    resume_scheduler(dev_to_gpu(dev));
+    return pm_runtime_force_resume(dev);
+}
+
  #endif
 
  static const struct dev_pm_ops adreno_pm_ops = {
-    SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+    SET_SYSTEM_SLEEP_PM_OPS(adreno_system_suspend, adreno_system_resume)
     SET_RUNTIME_PM_OPS(adreno_runtime_suspend, adreno_runtime_resume, NULL)
  };
 
--

2.35.1

#8 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

Regards,
Christian.

Quoted 4 lines

BR,
-R

author
#9 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 7 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Quoted 6 lines

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

Quoted 3 lines

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

I think amdgpu's use of kthread_park is not related to suspend, but didn't look too closely.

And perhaps the solution for this problem is more complex in the case of amdgpu, I'm not super familiar with the constraints there. But I think it is a fine solution for integrated GPUs.

BR,
-R

Quoted 8 lines

Regards,
Christian.

Quoted 4 lines

BR,
-R

#10 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 7 lines
Quoted 5 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Quoted 8 lines
Quoted 6 lines

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

Regards,
Christian.

Quoted 20 lines
Quoted 4 lines

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

I think amdgpu's use of kthread_park is not related to suspend, but didn't look too closely.

And perhaps the solution for this problem is more complex in the case of amdgpu, I'm not super familiar with the constraints there. But I think it is a fine solution for integrated GPUs.

BR,
-R

Quoted 6 lines

Regards,
Christian.

Quoted 3 lines

BR,
-R

#11 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 19 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 7 lines
Quoted 5 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Quoted 8 lines
Quoted 6 lines

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

Quoted 8 lines

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

Yeah I think properly flushing the scheduler and stopping it and cutting all drivers over to that sounds like the right approach. Generally suspend shouldn't be such a critical path that this will hurt us, all the other io queues get flushed too afaik.

Resume is the thing that needs to go real fast.

So a patch set to move all drivers that open code the kthread_park to the right scheduler function sounds like the right idea here to me. -Daniel

Quoted 25 lines

Regards,
Christian.

Quoted 20 lines
Quoted 4 lines

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

I think amdgpu's use of kthread_park is not related to suspend, but didn't look too closely.

And perhaps the solution for this problem is more complex in the case of amdgpu, I'm not super familiar with the constraints there. But I think it is a fine solution for integrated GPUs.

BR,
-R

Quoted 6 lines

Regards,
Christian.

Quoted 3 lines

BR,
-R

Signature · 3 lines
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
author
#12 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

In reply to Christian König
Quoted 15 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 7 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 5 lines

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

BR,
-R

Quoted 24 lines

Regards,
Christian.

Quoted 20 lines
Quoted 4 lines

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

I think amdgpu's use of kthread_park is not related to suspend, but didn't look too closely.

And perhaps the solution for this problem is more complex in the case of amdgpu, I'm not super familiar with the constraints there. But I think it is a fine solution for integrated GPUs.

BR,
-R

Quoted 6 lines

Regards,
Christian.

Quoted 3 lines

BR,
-R

#13 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 28 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 2 lines

And if you stop the scheduler they will just wait forever.

Quoted 8 lines
Quoted 3 lines

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

BR,
-R

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush, what prevents someone to push right away another job into the same entity's queue  right after that ? Shouldn't we first disable further pushing of jobs into entity before we wait for  sched->job_scheduled ?

Andrey

Quoted 23 lines
Quoted 23 lines

Regards,
Christian.

Quoted 19 lines
Quoted 3 lines

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

I think amdgpu's use of kthread_park is not related to suspend, but didn't look too closely.

And perhaps the solution for this problem is more complex in the case of amdgpu, I'm not super familiar with the constraints there. But I think it is a fine solution for integrated GPUs.

BR,
-R

Quoted 6 lines

Regards,
Christian.

Quoted 3 lines

BR,
-R

author
#14 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 41 lines · Andrey Grodzovsky
Quoted 28 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 2 lines

And if you stop the scheduler they will just wait forever.

Quoted 8 lines
Quoted 3 lines

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

BR,
-R

Quoted 7 lines

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself. We don't do that in drm/msm, but maybe you need to to move things btwn vram and system memory? But even in that case, if the # of jobs you push is bounded I guess that is ok?

BR,
-R

#15 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 48 lines · Rob Clark
Quoted 39 lines · Andrey Grodzovsky
Quoted 28 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 2 lines

And if you stop the scheduler they will just wait forever.

Quoted 8 lines
Quoted 3 lines

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

BR,
-R

Quoted 7 lines

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 3 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself.

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e. we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example for this i was looking at  flush_workqueue vs. drain_workqueue

Andrey

Quoted 3 lines

BR,
-R

author
#16 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 77 lines · Andrey Grodzovsky
Quoted 48 lines · Rob Clark
Quoted 39 lines · Andrey Grodzovsky
Quoted 28 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 2 lines

And if you stop the scheduler they will just wait forever.

Quoted 8 lines
Quoted 3 lines

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

BR,
-R

Quoted 7 lines

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 3 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself.

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in
suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

Quoted 5 lines

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e.
we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example
for this i was looking at flush_workqueue vs. drain_workqueue

Would it be possible for amdgpu to, in the system suspend task,

1) first queue up all the jobs needed to migrate bos out of vram, and whatever other housekeeping jobs are needed
2) then drain gpu scheduler's queues
3) and then finally wait for jobs executing on GPU to complete

BR,
-R

Quoted 6 lines

Andrey

Quoted 3 lines

BR,
-R

#17 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 79 lines · Rob Clark
Quoted 72 lines · Andrey Grodzovsky
Quoted 46 lines · Rob Clark
Quoted 35 lines · Andrey Grodzovsky
Quoted 34 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 5 lines

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

Quoted 9 lines
Quoted 2 lines

BR,
-R

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 4 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself. amdgpu_device_suspend

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in
suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

Quoted 5 lines

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e.
we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example
for this i was looking at flush_workqueue vs. drain_workqueue

Quoted 6 lines

Would it be possible for amdgpu to, in the system suspend task,

1) first queue up all the jobs needed to migrate bos out of vram, and whatever other housekeeping jobs are needed
2) then drain gpu scheduler's queues
3) and then finally wait for jobs executing on GPU to complete

We already do most of it in amdgpu_device_suspend, amdgpu_device_ip_suspend_phase1
followed by amdgpu_device_evict_resources followed by amdgpu_fence_driver_hw_fini is
exactly steps 1 + 3. What we are missing is step 2). For this step I suggest adding a function
called drm_sched_entity_drain which basically sets entity->stopped = true and then calls drm_sched_entity_flush. This will both reject any new insertions into entity's job queue and will flush all pending job submissions to HW from that entity. One point is we need to make make drm_sched_entity_push_job return value so the caller knows about job enqueue rejection.

What about runtime suspend ? I guess same issue with scheduler racing against HW susppend is relevant there ?

Also, could you point to a particular buggy scenario where the race between SW shceduler and suspend is causing a problem ?

Andrey

Quoted 9 lines

BR,
-R

Quoted 5 lines

Andrey

Quoted 2 lines

BR,
-R

author
#18 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 90 lines · Andrey Grodzovsky
Quoted 79 lines · Rob Clark
Quoted 72 lines · Andrey Grodzovsky
Quoted 46 lines · Rob Clark
Quoted 35 lines · Andrey Grodzovsky
Quoted 34 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 5 lines

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

Quoted 9 lines
Quoted 2 lines

BR,
-R

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 4 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself. amdgpu_device_suspend

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in
suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

Quoted 5 lines

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e.
we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example
for this i was looking at flush_workqueue vs. drain_workqueue

Quoted 6 lines

Would it be possible for amdgpu to, in the system suspend task,

1) first queue up all the jobs needed to migrate bos out of vram, and whatever other housekeeping jobs are needed
2) then drain gpu scheduler's queues
3) and then finally wait for jobs executing on GPU to complete

Quoted 13 lines

We already do most of it in amdgpu_device_suspend, amdgpu_device_ip_suspend_phase1
followed by amdgpu_device_evict_resources followed by amdgpu_fence_driver_hw_fini is
exactly steps 1 + 3. What we are missing is step 2). For this step I suggest adding a function
called drm_sched_entity_drain which basically sets entity->stopped = true and then calls drm_sched_entity_flush.
This will both reject any new insertions into entity's job queue and will flush all pending job submissions to HW from that entity. One point is we need to make make drm_sched_entity_push_job return value so the caller knows about job enqueue
rejection.

Hmm, seems like job enqueue that is rejected because we are in the process of suspending should be more of a WARN_ON() sort of thing? Not sure if there is something sensible to do for the caller at that point?

Quoted 3 lines

What about runtime suspend ? I guess same issue with scheduler racing against HW susppend is relevant there ?

Runtime suspend should be ok, as long as the driver holds a runpm reference whenever the hw needs to be awake. The problem with system suspend (at least if you are using pm_runtime_force_suspend() or doing something equivalent) is that it bypasses the runpm reference. (Which, IMO, seems like a bad design..)

Quoted 2 lines

Also, could you point to a particular buggy scenario where the race between SW shceduler and suspend is causing a problem ?

I wrote a piglit test[1] to try to trigger this scenario.. it isn't really that easy to hit

BR,
-R

[1] https://gitlab.freedesktop.org/mesa/piglit/-/merge_requests/643

Quoted 12 lines

Andrey

Quoted 9 lines

BR,
-R

Quoted 5 lines

Andrey

Quoted 2 lines

BR,
-R

#19 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 100 lines · Rob Clark
Quoted 85 lines · Andrey Grodzovsky
Quoted 76 lines · Rob Clark
Quoted 69 lines · Andrey Grodzovsky
Quoted 46 lines · Rob Clark
Quoted 35 lines · Andrey Grodzovsky
Quoted 34 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 5 lines

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

Quoted 9 lines
Quoted 2 lines

BR,
-R

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 4 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself. amdgpu_device_suspend

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in
suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

Quoted 5 lines

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e.
we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example
for this i was looking at flush_workqueue vs. drain_workqueue

Quoted 6 lines

Would it be possible for amdgpu to, in the system suspend task,

1) first queue up all the jobs needed to migrate bos out of vram, and whatever other housekeeping jobs are needed
2) then drain gpu scheduler's queues
3) and then finally wait for jobs executing on GPU to complete

Quoted 13 lines

We already do most of it in amdgpu_device_suspend, amdgpu_device_ip_suspend_phase1
followed by amdgpu_device_evict_resources followed by amdgpu_fence_driver_hw_fini is
exactly steps 1 + 3. What we are missing is step 2). For this step I suggest adding a function
called drm_sched_entity_drain which basically sets entity->stopped = true and then calls drm_sched_entity_flush.
This will both reject any new insertions into entity's job queue and will flush all pending job submissions to HW from that entity. One point is we need to make make drm_sched_entity_push_job return value so the caller knows about job enqueue
rejection.

Quoted 4 lines

Hmm, seems like job enqueue that is rejected because we are in the process of suspending should be more of a WARN_ON() sort of thing? Not sure if there is something sensible to do for the caller at that point?

What about the job's fence the caller is waiting on ? If we rejected job submission the caller must know about it to not get stuck waiting on that fence.

Quoted 8 lines
Quoted 3 lines

What about runtime suspend ? I guess same issue with scheduler racing against HW susppend is relevant there ?

Runtime suspend should be ok, as long as the driver holds a runpm reference whenever the hw needs to be awake. The problem with system suspend (at least if you are using pm_runtime_force_suspend() or doing something equivalent) is that it bypasses the runpm reference. (Which, IMO, seems like a bad design..)

I am not totally clear  yet - can you expand a bit why one case is ok but the other problematic ?

Andrey

Quoted 22 lines
Quoted 3 lines

Also, could you point to a particular buggy scenario where the race between SW shceduler and suspend is causing a problem ?

I wrote a piglit test[1] to try to trigger this scenario.. it isn't really that easy to hit

BR,
-R

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgitlab.freedesktop.org%2Fmesa%2Fpiglit%2F-%2Fmerge_requests%2F643&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C502ac8db4fb94b3b0e9d08da08fb270e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637832172051790527%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=u2Fqq%2BZpmjFHQFK77xwxEA5092O3Nc%2FdCMllfejgnvU%3D&amp;reserved=0

Quoted 11 lines

Andrey

Quoted 8 lines

BR,
-R

Quoted 5 lines

Andrey

Quoted 2 lines

BR,
-R

author
#20 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 109 lines · Andrey Grodzovsky
Quoted 100 lines · Rob Clark
Quoted 85 lines · Andrey Grodzovsky
Quoted 76 lines · Rob Clark
Quoted 69 lines · Andrey Grodzovsky
Quoted 46 lines · Rob Clark
Quoted 35 lines · Andrey Grodzovsky
Quoted 34 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 5 lines

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

Quoted 9 lines
Quoted 2 lines

BR,
-R

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 4 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself. amdgpu_device_suspend

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in
suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

Quoted 5 lines

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e.
we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example
for this i was looking at flush_workqueue vs. drain_workqueue

Quoted 6 lines

Would it be possible for amdgpu to, in the system suspend task,

1) first queue up all the jobs needed to migrate bos out of vram, and whatever other housekeeping jobs are needed
2) then drain gpu scheduler's queues
3) and then finally wait for jobs executing on GPU to complete

Quoted 13 lines

We already do most of it in amdgpu_device_suspend, amdgpu_device_ip_suspend_phase1
followed by amdgpu_device_evict_resources followed by amdgpu_fence_driver_hw_fini is
exactly steps 1 + 3. What we are missing is step 2). For this step I suggest adding a function
called drm_sched_entity_drain which basically sets entity->stopped = true and then calls drm_sched_entity_flush.
This will both reject any new insertions into entity's job queue and will flush all pending job submissions to HW from that entity. One point is we need to make make drm_sched_entity_push_job return value so the caller knows about job enqueue
rejection.

Quoted 4 lines

Hmm, seems like job enqueue that is rejected because we are in the process of suspending should be more of a WARN_ON() sort of thing? Not sure if there is something sensible to do for the caller at that point?

Quoted 4 lines

What about the job's fence the caller is waiting on ? If we rejected job submission the caller must know about it to not get stuck waiting on that fence.

Hmm, perhaps I'm not being imaginative enough, but this sort of scenario seems like it should only arise from a bug in the driver's suspend path, Ie. not doing all the job submission before shutting down the scheduler. I don't think anything good is going to result either way, which is why I was thinking you'd want a WARN_ON() to help debug/fix that case.

Quoted 11 lines
Quoted 9 lines
Quoted 4 lines

What about runtime suspend ? I guess same issue with scheduler racing against HW susppend is relevant there ?

Runtime suspend should be ok, as long as the driver holds a runpm reference whenever the hw needs to be awake. The problem with system suspend (at least if you are using pm_runtime_force_suspend() or doing something equivalent) is that it bypasses the runpm reference. (Which, IMO, seems like a bad design..)

Quoted 4 lines

I am not totally clear yet - can you expand a bit why one case is ok but the other
problematic ?

Sure, normally pm_runtime_get/put increment a reference count, as long as there have been more get's than puts, the device won't runtime suspend. So, for ex, msm's run_job fxn does a pm_runtime_get_sync(). And retire_submit() which runs after job completes on GPU does a pm_runtime_put_autosuspend().

System suspend, OTOH, bypasses this reference counting. Which is why extra care is needed.

BR,
-R

Quoted 25 lines

Andrey

Quoted 22 lines
Quoted 3 lines

Also, could you point to a particular buggy scenario where the race between SW shceduler and suspend is causing a problem ?

I wrote a piglit test[1] to try to trigger this scenario.. it isn't really that easy to hit

BR,
-R

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgitlab.freedesktop.org%2Fmesa%2Fpiglit%2F-%2Fmerge_requests%2F643&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C502ac8db4fb94b3b0e9d08da08fb270e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637832172051790527%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=u2Fqq%2BZpmjFHQFK77xwxEA5092O3Nc%2FdCMllfejgnvU%3D&amp;reserved=0

Quoted 11 lines

Andrey

Quoted 8 lines

BR,
-R

Quoted 5 lines

Andrey

Quoted 2 lines

BR,
-R

#21 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

Quoted 111 lines · Rob Clark
Quoted 100 lines · Andrey Grodzovsky
Quoted 98 lines · Rob Clark
Quoted 96 lines · Andrey Grodzovsky
Quoted 76 lines · Rob Clark
Quoted 69 lines · Andrey Grodzovsky
Quoted 46 lines · Rob Clark
Quoted 35 lines · Andrey Grodzovsky
Quoted 34 lines · Rob Clark
Quoted 14 lines · Christian König

Am 17.03.22 um 17:18 schrieb Rob Clark:

Quoted 13 lines
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is
trying to address, so we aren't racing between shutting down the hw
and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

Quoted 6 lines
Quoted 2 lines

Hmm, perhaps that is true if you need to migrate things out of vram? It is at least not a problem when vram is not involved.

No, it's much wider than that.

See what can happen is that the memory management shrinkers want to wait for a dma_fence during suspend.

we don't wait on fences in shrinker, only purging or evicting things that are already ready. Actually, waiting on fences in shrinker path sounds like a pretty bad idea.

Quoted 5 lines

And if you stop the scheduler they will just wait forever.

What you need to do instead is to drain the scheduler, e.g. call drm_sched_entity_flush() with a proper timeout for each entity you have created.

yeah, it would work to drain the scheduler.. I guess that might be the more portable approach as far as generic solution for suspend.

Quoted 9 lines
Quoted 2 lines

BR,
-R

I am not sure how this drains the scheduler ? Suppose we done the waiting in drm_sched_entity_flush,
what prevents someone to push right away another job into the same entity's queue right after that ?
Shouldn't we first disable further pushing of jobs into entity before we wait for sched->job_scheduled ?

Quoted 4 lines

In the system suspend path, userspace processes will have already been frozen, so there should be no way to push more jobs to the scheduler, unless they are pushed from the kernel itself. amdgpu_device_suspend

It was my suspicion but I wasn't sure about it.

Quoted 3 lines

We don't do that in
drm/msm, but maybe you need to to move things btwn vram and system memory?

Exactly, that was my main concern - if we use this method we have to use it in a point in
suspend sequence when all the in kernel job submissions activity already suspended

Quoted 2 lines

But even in that case, if the # of jobs you push is bounded I guess that is ok?

Submissions to scheduler entities are using unbounded queue, the bounded part is when
you extract next job from entity to submit to HW ring and it rejects if submission limit reached (drm_sched_ready)

Quoted 5 lines

In general - It looks to me at least that what we what we want her is more of a drain operation then flush (i.e.
we first want to disable any further job submission to entity's queue and then flush all in flight ones). As example
for this i was looking at flush_workqueue vs. drain_workqueue

Quoted 6 lines

Would it be possible for amdgpu to, in the system suspend task,

1) first queue up all the jobs needed to migrate bos out of vram, and whatever other housekeeping jobs are needed
2) then drain gpu scheduler's queues
3) and then finally wait for jobs executing on GPU to complete

We already do most of it in amdgpu_device_suspend, amdgpu_device_ip_suspend_phase1
followed by amdgpu_device_evict_resources followed by amdgpu_fence_driver_hw_fini is
exactly steps 1 + 3. What we are missing is step 2). For this step I suggest adding a function
called drm_sched_entity_drain which basically sets entity->stopped = true and then calls drm_sched_entity_flush.
This will both reject any new insertions into entity's job queue and will flush all pending job submissions to HW from that entity. One point is we need to make make drm_sched_entity_push_job return value so the caller knows about job enqueue
rejection.

Quoted 9 lines
Quoted 4 lines

Hmm, seems like job enqueue that is rejected because we are in the process of suspending should be more of a WARN_ON() sort of thing? Not sure if there is something sensible to do for the caller at that point?

What about the job's fence the caller is waiting on ? If we rejected job submission the caller must know about it to not get stuck waiting on that fence.

Quoted 6 lines

Hmm, perhaps I'm not being imaginative enough, but this sort of scenario seems like it should only arise from a bug in the driver's suspend path, Ie. not doing all the job submission before shutting down the scheduler. I don't think anything good is going to result either way, which is why I was thinking you'd want a WARN_ON() to help debug/fix that case.

Yes, I just wanted the code to not allow such bugs to go through unnoticed. I guess WARN_ON should give laud enough warning anyway

Andrey

Quoted 49 lines
Quoted 8 lines
Quoted 7 lines
Quoted 2 lines

What about runtime suspend ? I guess same issue with scheduler racing against HW susppend is relevant there ?

Runtime suspend should be ok, as long as the driver holds a runpm reference whenever the hw needs to be awake. The problem with system suspend (at least if you are using pm_runtime_force_suspend() or doing something equivalent) is that it bypasses the runpm reference. (Which, IMO, seems like a bad design..)

Quoted 4 lines

I am not totally clear yet - can you expand a bit why one case is ok but the other
problematic ?

Sure, normally pm_runtime_get/put increment a reference count, as long as there have been more get's than puts, the device won't runtime suspend. So, for ex, msm's run_job fxn does a pm_runtime_get_sync(). And retire_submit() which runs after job completes on GPU does a pm_runtime_put_autosuspend().

System suspend, OTOH, bypasses this reference counting. Which is why extra care is needed.

BR,
-R

Quoted 24 lines

Andrey

Quoted 21 lines
Quoted 2 lines

Also, could you point to a particular buggy scenario where the race between SW shceduler and suspend is causing a problem ?

I wrote a piglit test[1] to try to trigger this scenario.. it isn't really that easy to hit

BR,
-R

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgitlab.freedesktop.org%2Fmesa%2Fpiglit%2F-%2Fmerge_requests%2F643&amp;data=04%7C01%7Candrey.grodzovsky%40amd.com%7C35f0d7d9282044651c9708da0903d4f4%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637832209324217553%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&amp;sdata=dwjPEVAYgCI%2BtEyzBirfAQkJjZax2NdiLQfNeFfImtU%3D&amp;reserved=0

Quoted 11 lines

Andrey

Quoted 8 lines

BR,
-R

Quoted 5 lines

Andrey

Quoted 2 lines

BR,
-R

#22 lore raw

Re: [PATCH 2/3] drm/msm/gpu: Park scheduler threads for system suspend

In reply to Christian König
Quoted 11 lines · Christian König

Am 17.03.22 um 16:10 schrieb Rob Clark:

Quoted 4 lines

[SNIP]
userspace frozen != kthread frozen .. that is what this patch is trying to address, so we aren't racing between shutting down the hw and the scheduler shoveling more jobs at us.

Well exactly that's the problem. The scheduler is supposed to shoveling more jobs at us until it is empty.

Thinking more about it we will then keep some dma_fence instance unsignaled and that is and extremely bad idea since it can lead to deadlocks during suspend.

So this patch here is an absolute clear NAK from my side. If amdgpu is doing something similar that is a severe bug and needs to be addressed somehow.

From looking at latest amd-stagin-drm-next we only use directly kthread_park during in debugfs IB hooks.
For S3  suspend (amdgpu_pmops_suspend) we will only  flush all the HW fences (amdgpu_fence_wait_empty) so we don't freeze the scheduler thread and don't flush scheduler entities.

Andrey

Quoted 9 lines

Regards,
Christian.

Quoted 4 lines

BR,
-R

author
#23 lore raw

PATCH 3/3 drm/msm/gpu: Remove mutex from wait_event condition

In reply to Rob Clark

From: Rob Clark <[email protected]>

The mutex wasn't really protecting anything before. Before the previous patch we could still be racing with the scheduler's kthread, as that is not necessarily frozen yet. Now that we've parked the sched threads, the only race is with jobs retiring, and that is harmless, ie.

---
 drivers/gpu/drm/msm/adreno/adreno_device.c | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +1 -10
index 0440a98988fc..661dfa7681fb 100644
@@ -607,15 +607,6 @@ static int adreno_runtime_resume(struct device *dev)
 	return gpu->funcs->pm_resume(gpu);
 }
 
-static int active_submits(struct msm_gpu *gpu)
-{
-	int active_submits;
-	mutex_lock(&gpu->active_lock);
-	active_submits = gpu->active_submits;
-	mutex_unlock(&gpu->active_lock);
-	return active_submits;
-}
-
 static int adreno_runtime_suspend(struct device *dev)
 {
 	struct msm_gpu *gpu = dev_to_gpu(dev);
@@ -669,7 +660,7 @@ static int adreno_system_suspend(struct device *dev)
 	suspend_scheduler(gpu);
 
 	remaining = wait_event_timeout(gpu->retire_event,
-				       active_submits(gpu) == 0,
+				       gpu->active_submits == 0,
 				       msecs_to_jiffies(1000));
 	if (remaining == 0) {
 		dev_err(dev, "Timeout waiting for GPU to suspend\n");
Signature · 1 line
2.35.1
#24 lore raw

Re: [Freedreno] [PATCH 3/3] drm/msm/gpu: Remove mutex from wait_event condition

Quoted 17 lines · Rob Clark

From: Rob Clark <[email protected]>

The mutex wasn't really protecting anything before. Before the previous patch we could still be racing with the scheduler's kthread, as that is not necessarily frozen yet. Now that we've parked the sched threads, the only race is with jobs retiring, and that is harmless, ie.

---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 11 +----------
  1 file changed, 1 insertion(+), 10 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +0 -0
index 0440a98988fc..661dfa7681fb 100644

drivers/gpu/drm/msm/adreno/adreno_device.c static int adreno_runtime_resume(struct device *dev) · line 615

Quoted 9 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +0 -6
@@ -607,15 +607,6 @@ static int adreno_runtime_resume(struct device *dev)
  	return gpu->funcs->pm_resume(gpu);
  }
 
-static int active_submits(struct msm_gpu *gpu)
-{
-	int active_submits;
-	mutex_lock(&gpu->active_lock);
-	active_submits = gpu->active_submits;
-	mutex_unlock(&gpu->active_lock);

I assumed that this lock here was to ensure proper barriers while reading active_submits. Is that not required?

-Akhil.

Quoted 15 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +1 -4
-	return active_submits;
-}
-
  static int adreno_runtime_suspend(struct device *dev)
  {
  	struct msm_gpu *gpu = dev_to_gpu(dev);
@@ -669,7 +660,7 @@ static int adreno_system_suspend(struct device *dev)
  	suspend_scheduler(gpu);
 
  	remaining = wait_event_timeout(gpu->retire_event,
-				       active_submits(gpu) == 0,
+				       gpu->active_submits == 0,
  				       msecs_to_jiffies(1000));
  	if (remaining == 0) {
  		dev_err(dev, "Timeout waiting for GPU to suspend\n");
author
#25 lore raw

Re: [Freedreno] [PATCH 3/3] drm/msm/gpu: Remove mutex from wait_event condition

Quoted 22 lines · Akhil P Oommen
Quoted 20 lines · Rob Clark

From: Rob Clark <[email protected]>

The mutex wasn't really protecting anything before. Before the previous patch we could still be racing with the scheduler's kthread, as that is not necessarily frozen yet. Now that we've parked the sched threads, the only race is with jobs retiring, and that is harmless, ie.

---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 11 +----------
  1 file changed, 1 insertion(+), 10 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +0 -0
index 0440a98988fc..661dfa7681fb 100644
@@ -607,15 +607,6 @@ static int adreno_runtime_resume(struct device *dev)
      return gpu->funcs->pm_resume(gpu);
  }
 
Quoted 8 lines

drivers/gpu/drm/msm/adreno/adreno_device.c static int adreno_runtime_resume(struct device *dev) · line 615

Quoted 6 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +0 -6
-static int active_submits(struct msm_gpu *gpu)
-{
-     int active_submits;
-     mutex_lock(&gpu->active_lock);
-     active_submits = gpu->active_submits;
-     mutex_unlock(&gpu->active_lock);

I assumed that this lock here was to ensure proper barriers while reading active_submits. Is that not required?

There is a spinlock in prepare_to_wait_event() ahead of checking the condition, which AFAIU is a sufficient barrier

BR,
-R

Quoted 18 lines

-Akhil.

Quoted 15 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +1 -4
-     return active_submits;
-}
-
  static int adreno_runtime_suspend(struct device *dev)
  {
      struct msm_gpu *gpu = dev_to_gpu(dev);
@@ -669,7 +660,7 @@ static int adreno_system_suspend(struct device *dev)
      suspend_scheduler(gpu);
 
      remaining = wait_event_timeout(gpu->retire_event,
-                                    active_submits(gpu) == 0,
+                                    gpu->active_submits == 0,
                                     msecs_to_jiffies(1000));
      if (remaining == 0) {
              dev_err(dev, "Timeout waiting for GPU to suspend\n");
#26 lore raw

Re: [Freedreno] [PATCH 3/3] drm/msm/gpu: Remove mutex from wait_event condition

Quoted 32 lines · On Thu, 17 Mar 2022 14:07:45 -0700 Rob Clark wrote:
Quoted 22 lines · Akhil P Oommen
Quoted 20 lines · Rob Clark

From: Rob Clark <[email protected]>

The mutex wasn't really protecting anything before. Before the previous patch we could still be racing with the scheduler's kthread, as that is not necessarily frozen yet. Now that we've parked the sched threads, the only race is with jobs retiring, and that is harmless, ie.

---
  drivers/gpu/drm/msm/adreno/adreno_device.c | 11 +----------
  1 file changed, 1 insertion(+), 10 deletions(-)
drivers/gpu/drm/msm/adreno/adreno_device.c +0 -0
index 0440a98988fc..661dfa7681fb 100644
@@ -607,15 +607,6 @@ static int adreno_runtime_resume(struct device *dev)
      return gpu->funcs->pm_resume(gpu);
  }
 
Quoted 8 lines

drivers/gpu/drm/msm/adreno/adreno_device.c static int adreno_runtime_resume(struct device *dev) · line 615

Quoted 6 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +0 -6
-static int active_submits(struct msm_gpu *gpu)
-{
-     int active_submits;
-     mutex_lock(&gpu->active_lock);
-     active_submits = gpu->active_submits;
-     mutex_unlock(&gpu->active_lock);

I assumed that this lock here was to ensure proper barriers while reading active_submits. Is that not required?

Quoted 2 lines

There is a spinlock in prepare_to_wait_event() ahead of checking the condition, which AFAIU is a sufficient barrier

set_current_state() is instead - feel free to grep it in <linux/wait.h>

Hillf

Quoted 22 lines

BR,
-R

Quoted 18 lines

-Akhil.

Quoted 15 lines
drivers/gpu/drm/msm/adreno/adreno_device.c +1 -4
-     return active_submits;
-}
-
  static int adreno_runtime_suspend(struct device *dev)
  {
      struct msm_gpu *gpu = dev_to_gpu(dev);
@@ -669,7 +660,7 @@ static int adreno_system_suspend(struct device *dev)
      suspend_scheduler(gpu);
 
      remaining = wait_event_timeout(gpu->retire_event,
-                                    active_submits(gpu) == 0,
+                                    gpu->active_submits == 0,
                                     msecs_to_jiffies(1000));
      if (remaining == 0) {
              dev_err(dev, "Timeout waiting for GPU to suspend\n");