Skip to contentSkip to conversation

Add workaround for fixing runpm

PATCH New Karol Herbst 1 participant 9 messages Sep 23, 2019 1 message not in the archive

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover Add workaround for fixing runpm

Changes since last sent:
* add a patch to set the device into DRM_SWITCH_POWER_CHANGING state (can be dropped actually, I thought I was needing it, came up with a different approach and forgot to delete it, doesn't hurt though)
* expose information about runtime suspending to nvkm so that we can run the pcie workaround only on runtime suspend

Karol Herbst (8):
pci: disable ASPM before changing the link speed pci/gk104: enable dl_mgr safe mode
pci/gk104: wait for ltssm idle before changing the link
pci: enable pcie link changes for pascal
drm: set power_state to DRM_SWITCH_POWER_CHANGING before changing
drm: be explicit about runtime suspend in fini
pci: set the pcie link speed to 8.0 when suspending
drm: abort runtime suspend if we hit an error

 bin/nv_init.c                           |  2 +-
 drm/nouveau/include/nvif/client.h       |  2 +-
 drm/nouveau/include/nvif/driver.h       |  2 +-
 drm/nouveau/include/nvkm/core/device.h  |  5 ++-
 drm/nouveau/include/nvkm/core/engine.h  |  2 +-
 drm/nouveau/include/nvkm/core/object.h  | 13 ++++++--
 drm/nouveau/include/nvkm/core/oproxy.h  |  2 +-
 drm/nouveau/include/nvkm/core/subdev.h  |  5 +--
 drm/nouveau/include/nvkm/subdev/pci.h   |  3 +-
 drm/nouveau/nouveau_drm.c               | 10 +++++-
 drm/nouveau/nouveau_nvif.c              |  6 ++--
 drm/nouveau/nvif/client.c               |  4 +--
 drm/nouveau/nvkm/core/client.c          |  5 ++-
 drm/nouveau/nvkm/core/engine.c          |  4 +--
 drm/nouveau/nvkm/core/ioctl.c           |  4 +--
 drm/nouveau/nvkm/core/object.c          | 22 ++++++++++---
 drm/nouveau/nvkm/core/oproxy.c          |  2 +-
 drm/nouveau/nvkm/core/subdev.c          |  4 +--
 drm/nouveau/nvkm/engine/device/base.c   | 11 ++++---
 drm/nouveau/nvkm/engine/device/pci.c    |  2 +-
 drm/nouveau/nvkm/engine/device/priv.h   |  2 +-
 drm/nouveau/nvkm/engine/device/tegra.c  |  3 +-
 drm/nouveau/nvkm/engine/device/user.c   |  2 +-
 drm/nouveau/nvkm/engine/disp/base.c     |  2 +-
 drm/nouveau/nvkm/engine/disp/channv50.c |  2 +-
 drm/nouveau/nvkm/engine/falcon.c        |  2 +-
 drm/nouveau/nvkm/engine/fifo/base.c     |  2 +-
 drm/nouveau/nvkm/engine/fifo/chan.c     |  5 +--
 drm/nouveau/nvkm/engine/fifo/gf100.c    |  2 +-
 drm/nouveau/nvkm/engine/fifo/gk104.c    |  2 +-
 drm/nouveau/nvkm/engine/gr/base.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv04.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv10.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv20.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv20.h       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv40.c       |  2 +-
 drm/nouveau/nvkm/engine/mpeg/nv44.c     |  2 +-
 drm/nouveau/nvkm/engine/pm/base.c       |  2 +-
 drm/nouveau/nvkm/engine/sec2/base.c     |  2 +-
 drm/nouveau/nvkm/engine/xtensa.c        |  2 +-
 drm/nouveau/nvkm/subdev/bar/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/clk/base.c      |  4 +--
 drm/nouveau/nvkm/subdev/devinit/base.c  |  2 +-
 drm/nouveau/nvkm/subdev/fault/base.c    |  2 +-
 drm/nouveau/nvkm/subdev/fault/user.c    |  2 +-
 drm/nouveau/nvkm/subdev/gpio/base.c     |  2 +-
 drm/nouveau/nvkm/subdev/i2c/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/instmem/base.c  |  2 +-
 drm/nouveau/nvkm/subdev/mc/base.c       |  2 +-
 drm/nouveau/nvkm/subdev/pci/base.c      |  4 ++-
 drm/nouveau/nvkm/subdev/pci/g84.c       |  9 ++++++
 drm/nouveau/nvkm/subdev/pci/g92.c       |  1 +
 drm/nouveau/nvkm/subdev/pci/g94.c       |  1 +
 drm/nouveau/nvkm/subdev/pci/gf100.c     |  1 +
 drm/nouveau/nvkm/subdev/pci/gf106.c     |  1 +
 drm/nouveau/nvkm/subdev/pci/gk104.c     | 29 +++++++++++++----
 drm/nouveau/nvkm/subdev/pci/gp100.c     | 11 +++++++
 drm/nouveau/nvkm/subdev/pci/pcie.c      | 43 ++++++++++++++++++++++---
 drm/nouveau/nvkm/subdev/pci/priv.h      | 16 +++++++++
 drm/nouveau/nvkm/subdev/pmu/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/secboot/base.c  |  2 +-
 drm/nouveau/nvkm/subdev/therm/base.c    |  2 +-
 drm/nouveau/nvkm/subdev/timer/base.c    |  2 +-
 lib/drm.c                               |  2 +-
 lib/main.c                              |  6 ++--
 lib/null.c                              |  6 ++--
 66 files changed, 224 insertions(+), 88 deletions(-)
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau

A message that is not in the archive (sent off-list or not yet received): 20190923203951.1652-1-kherbst-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org

#2 lore raw

PATCH 1/8 pci: disable ASPM before changing the link speed

taken from nvgpu

v2: rename force_aspm_off to aspm_off
rework interface to return old value and take a mask as the input

---
 drm/nouveau/nvkm/subdev/pci/g84.c   |  9 +++++++++
 drm/nouveau/nvkm/subdev/pci/g92.c   |  1 +
 drm/nouveau/nvkm/subdev/pci/g94.c   |  1 +
 drm/nouveau/nvkm/subdev/pci/gf100.c |  1 +
 drm/nouveau/nvkm/subdev/pci/gf106.c |  1 +
 drm/nouveau/nvkm/subdev/pci/gk104.c |  1 +
 drm/nouveau/nvkm/subdev/pci/pcie.c  | 15 +++++++++++++++
 drm/nouveau/nvkm/subdev/pci/priv.h  | 10 ++++++++++
 8 files changed, 39 insertions(+)
drm/nouveau/nvkm/subdev/pci/g84.c +9 -0
index 62438d892..5e17e0589 100644
@@ -122,6 +122,14 @@ g84_pci_init(struct nvkm_pci *pci)
 		nvkm_pci_mask(pci, 0x041c, 0x00000060, 0x00000000);
 }
 
+enum nvkm_pci_aspm
+g84_pcie_aspm_off(struct nvkm_pci *pci, enum nvkm_pci_aspm states)
+{
+	u32 value = states << 7;
+
+	return (nvkm_pci_mask(pci, 0x150, 0x180, value) & 0x180) >> 7;
+}
+
 int
 g84_pcie_init(struct nvkm_pci *pci)
 {
@@ -147,6 +155,7 @@ g84_pci_func = {
 	.pcie.set_version = g84_pcie_set_version,
 	.pcie.version = g84_pcie_version,
 	.pcie.version_supported = g84_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/g92.c +1 -0
index 48874359d..85629ffee 100644
@@ -48,6 +48,7 @@ g92_pci_func = {
 	.pcie.set_version = g84_pcie_set_version,
 	.pcie.version = g84_pcie_version,
 	.pcie.version_supported = g92_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/g94.c +1 -0
index 09adb37a5..588bea4f6 100644
@@ -40,6 +40,7 @@ g94_pci_func = {
 	.pcie.set_version = g84_pcie_set_version,
 	.pcie.version = g84_pcie_version,
 	.pcie.version_supported = g92_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/gf100.c +1 -0
index 00a5e7d3e..7948942b6 100644
@@ -93,6 +93,7 @@ gf100_pci_func = {
 	.pcie.set_version = gf100_pcie_set_version,
 	.pcie.version = gf100_pcie_version,
 	.pcie.version_supported = g92_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/gf106.c +1 -0
index 11bf419af..c62c66a2e 100644
@@ -40,6 +40,7 @@ gf106_pci_func = {
 	.pcie.set_version = gf100_pcie_set_version,
 	.pcie.version = gf100_pcie_version,
 	.pcie.version_supported = g92_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/gk104.c +1 -0
index e68030507..a4014d4a7 100644
@@ -219,6 +219,7 @@ gk104_pci_func = {
 	.pcie.set_version = gf100_pcie_set_version,
 	.pcie.version = gf100_pcie_version,
 	.pcie.version_supported = gk104_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/pcie.c +15 -0
index d71e5db50..ed015b252 100644
@@ -111,12 +111,21 @@ nvkm_pcie_init(struct nvkm_pci *pci)
 	return 0;
 }
 
+enum nvkm_pci_aspm
+nvkm_pcie_aspm_off(struct nvkm_pci *pci, enum nvkm_pci_aspm states)
+{
+	if (!pci->func->pcie.aspm_off)
+		return NVKM_PCI_ASPM_NONE;
+	return pci->func->pcie.aspm_off(pci, states);
+}
+
 int
 nvkm_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 {
 	struct nvkm_subdev *subdev = &pci->subdev;
 	enum nvkm_pcie_speed cur_speed, max_speed;
 	struct pci_bus *pbus;
+	enum nvkm_pci_aspm old_aspm_state;
 	int ret;
 
 	if (!pci || !pci_is_pcie(pci->pdev))
@@ -157,9 +166,15 @@ nvkm_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 	nvkm_debug(subdev, "set link to %s x%i\n",
 		   nvkm_pcie_speeds[speed], width);
 
+	/* disable ASPM */
+	old_aspm_state = nvkm_pcie_aspm_off(pci, NVKM_PCI_ASPM_L0S_L1);
+
 	ret = pci->func->pcie.set_link(pci, speed, width);
 	if (ret < 0)
 		nvkm_error(subdev, "setting link failed: %i\n", ret);
 
+	/* restore old ASPM state */
+	nvkm_pcie_aspm_off(pci, old_aspm_state);
+
 	return ret;
 }
drm/nouveau/nvkm/subdev/pci/priv.h +10 -0
index 7009aad86..8518ff056 100644
@@ -7,6 +7,13 @@
 int nvkm_pci_new_(const struct nvkm_pci_func *, struct nvkm_device *,
 		  int index, struct nvkm_pci **);
 
+enum nvkm_pci_aspm {
+	NVKM_PCI_ASPM_NONE = 0,
+	NVKM_PCI_ASPM_L0S = 1,
+	NVKM_PCI_ASPM_L1 = 2,
+	NVKM_PCI_ASPM_L0S_L1 = 3,
+};
+
 struct nvkm_pci_func {
 	void (*init)(struct nvkm_pci *);
 	u32 (*rd32)(struct nvkm_pci *, u16 addr);
@@ -24,6 +31,8 @@ struct nvkm_pci_func {
 		void (*set_version)(struct nvkm_pci *, u8);
 		int (*version)(struct nvkm_pci *);
 		int (*version_supported)(struct nvkm_pci *);
+		enum nvkm_pci_aspm (*aspm_off)(struct nvkm_pci *,
+					       enum nvkm_pci_aspm);
 	} pcie;
 };
 
@@ -42,6 +51,7 @@ int g84_pcie_version(struct nvkm_pci *);
 void g84_pcie_set_link_speed(struct nvkm_pci *, enum nvkm_pcie_speed);
 enum nvkm_pcie_speed g84_pcie_cur_speed(struct nvkm_pci *);
 enum nvkm_pcie_speed g84_pcie_max_speed(struct nvkm_pci *);
+enum nvkm_pci_aspm g84_pcie_aspm_off(struct nvkm_pci *, enum nvkm_pci_aspm);
 int g84_pcie_init(struct nvkm_pci *);
 int g84_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8);
 
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#3 lore raw

PATCH 2/8 pci/gk104: enable dl_mgr safe mode

In reply to a message not in the archive

no idea why and what it does, taken from nvgpu

---
 drm/nouveau/nvkm/subdev/pci/gk104.c | 3 +++
 1 file changed, 3 insertions(+)
drm/nouveau/nvkm/subdev/pci/gk104.c +3 -0
index a4014d4a7..1c4f9a1cd 100644
@@ -184,6 +184,7 @@ gk104_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 	struct nvkm_subdev *subdev = &pci->subdev;
 	enum nvkm_pcie_speed lnk_ctl_speed = gk104_pcie_lnkctl_speed(pci);
 	enum nvkm_pcie_speed lnk_cap_speed = gk104_pcie_cap_speed(pci);
+	u32 old_dl_mgr;
 
 	if (speed > lnk_cap_speed) {
 		speed = lnk_cap_speed;
@@ -197,7 +198,9 @@ gk104_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 			  " lnkctl speed\n");
 	}
 
+	old_dl_mgr = nvkm_mask(subdev->device, 0x8b8c0, 0x4, 0x4);
 	gk104_pcie_set_link_speed(pci, speed);
+	nvkm_wr32(subdev->device, 0x8b8c0, old_dl_mgr);
 	return 0;
 }
 
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#4 lore raw

PATCH 3/8 pci/gk104: wait for ltssm idle before changing the link

In reply to a message not in the archive

taken from nvgpu

v2: return -EBUSY when the timeout is reached

---
 drm/nouveau/nvkm/subdev/pci/gk104.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
drm/nouveau/nvkm/subdev/pci/gk104.c +14 -3
index 1c4f9a1cd..dda113a45 100644
@@ -23,6 +23,8 @@
  */
 #include "priv.h"
 
+#include <subdev/timer.h>
+
 static int
 gk104_pcie_version_supported(struct nvkm_pci *pci)
 {
@@ -123,7 +125,7 @@ gk104_pcie_max_speed(struct nvkm_pci *pci)
 	return NVKM_PCIE_SPEED_2_5;
 }
 
-static void
+static int
 gk104_pcie_set_link_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
 {
 	struct nvkm_device *device = pci->subdev.device;
@@ -142,8 +144,16 @@ gk104_pcie_set_link_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
 		break;
 	}
 
+	/* wait for ltssm idle */
+	if (nvkm_msec(device, 200,
+		if ((nvkm_rd32(device, 0x8c040) & 0x1f) == 0)
+			break;
+	) < 0) {
+		return -EBUSY;
+	}
 	nvkm_mask(device, 0x8c040, 0xc0000, mask_value);
 	nvkm_mask(device, 0x8c040, 0x1, 0x1);
+	return 0;
 }
 
 static int
@@ -185,6 +195,7 @@ gk104_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 	enum nvkm_pcie_speed lnk_ctl_speed = gk104_pcie_lnkctl_speed(pci);
 	enum nvkm_pcie_speed lnk_cap_speed = gk104_pcie_cap_speed(pci);
 	u32 old_dl_mgr;
+	int ret;
 
 	if (speed > lnk_cap_speed) {
 		speed = lnk_cap_speed;
@@ -199,9 +210,9 @@ gk104_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 	}
 
 	old_dl_mgr = nvkm_mask(subdev->device, 0x8b8c0, 0x4, 0x4);
-	gk104_pcie_set_link_speed(pci, speed);
+	ret = gk104_pcie_set_link_speed(pci, speed);
 	nvkm_wr32(subdev->device, 0x8b8c0, old_dl_mgr);
-	return 0;
+	return ret;
 }
 
 
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#5 lore raw

PATCH 4/8 pci: enable pcie link changes for pascal

In reply to a message not in the archive

v2: add force disable ASPM func pointer

---
 drm/nouveau/nvkm/subdev/pci/gk104.c |  8 ++++----
 drm/nouveau/nvkm/subdev/pci/gp100.c | 11 +++++++++++
 drm/nouveau/nvkm/subdev/pci/priv.h  |  5 +++++
 3 files changed, 20 insertions(+), 4 deletions(-)
drm/nouveau/nvkm/subdev/pci/gk104.c +4 -4
index dda113a45..d6ba1918e 100644
@@ -25,7 +25,7 @@
 
 #include <subdev/timer.h>
 
-static int
+int
 gk104_pcie_version_supported(struct nvkm_pci *pci)
 {
 	return (nvkm_rd32(pci->subdev.device, 0x8c1c0) & 0x4) == 0x4 ? 2 : 1;
@@ -110,7 +110,7 @@ gk104_pcie_lnkctl_speed(struct nvkm_pci *pci)
 	return -1;
 }
 
-static enum nvkm_pcie_speed
+enum nvkm_pcie_speed
 gk104_pcie_max_speed(struct nvkm_pci *pci)
 {
 	u32 max_speed = nvkm_rd32(pci->subdev.device, 0x8c1c0) & 0x300000;
@@ -156,7 +156,7 @@ gk104_pcie_set_link_speed(struct nvkm_pci *pci, enum nvkm_pcie_speed speed)
 	return 0;
 }
 
-static int
+int
 gk104_pcie_init(struct nvkm_pci * pci)
 {
 	enum nvkm_pcie_speed lnkctl_speed, max_speed, cap_speed;
@@ -188,7 +188,7 @@ gk104_pcie_init(struct nvkm_pci * pci)
 	return 0;
 }
 
-static int
+int
 gk104_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 {
 	struct nvkm_subdev *subdev = &pci->subdev;
drm/nouveau/nvkm/subdev/pci/gp100.c +11 -0
index 82c5234a0..c102109d9 100644
@@ -35,6 +35,17 @@ gp100_pci_func = {
 	.wr08 = nv40_pci_wr08,
 	.wr32 = nv40_pci_wr32,
 	.msi_rearm = gp100_pci_msi_rearm,
+
+	.pcie.init = gk104_pcie_init,
+	.pcie.set_link = gk104_pcie_set_link,
+
+	.pcie.max_speed = gk104_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = gf100_pcie_set_version,
+	.pcie.version = gf100_pcie_version,
+	.pcie.version_supported = gk104_pcie_version_supported,
+	.pcie.aspm_off = g84_pcie_aspm_off,
 };
 
 int
drm/nouveau/nvkm/subdev/pci/priv.h +5 -0
index 8518ff056..17be1d563 100644
@@ -64,6 +64,11 @@ int gf100_pcie_cap_speed(struct nvkm_pci *);
 int gf100_pcie_init(struct nvkm_pci *);
 int gf100_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8);
 
+int gk104_pcie_init(struct nvkm_pci *);
+int gk104_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8 width);
+enum nvkm_pcie_speed gk104_pcie_max_speed(struct nvkm_pci *);
+int gk104_pcie_version_supported(struct nvkm_pci *);
+
 int nvkm_pcie_oneinit(struct nvkm_pci *);
 int nvkm_pcie_init(struct nvkm_pci *);
 #endif
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#6 lore raw

PATCH 5/8 drm: set power_state to DRM_SWITCH_POWER_CHANGING before changing

In reply to a message not in the archive
---
 drm/nouveau/nouveau_drm.c | 2 ++
 1 file changed, 2 insertions(+)
drm/nouveau/nouveau_drm.c +2 -0
index 3d32afe8a..466271c0c 100644
@@ -917,6 +917,7 @@ nouveau_pmops_runtime_suspend(struct device *dev)
 		return -EBUSY;
 	}
 
+	drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 	nouveau_switcheroo_optimus_dsm();
 	ret = nouveau_do_suspend(drm_dev, true);
 	pci_save_state(pdev);
@@ -941,6 +942,7 @@ nouveau_pmops_runtime_resume(struct device *dev)
 		return -EBUSY;
 	}
 
+	drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 	pci_set_power_state(pdev, PCI_D0);
 	pci_restore_state(pdev);
 	ret = pci_enable_device(pdev);
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#7 lore raw

PATCH 6/8 drm: be explicit about runtime suspend in fini

In reply to a message not in the archive

Allows fini functions to check if we do a real system suspend or just runtime suspending the GPU.

---
 bin/nv_init.c                           |  2 +-
 drm/nouveau/include/nvif/client.h       |  2 +-
 drm/nouveau/include/nvif/driver.h       |  2 +-
 drm/nouveau/include/nvkm/core/device.h  |  3 ++-
 drm/nouveau/include/nvkm/core/engine.h  |  2 +-
 drm/nouveau/include/nvkm/core/object.h  | 13 +++++++++++--
 drm/nouveau/include/nvkm/core/oproxy.h  |  2 +-
 drm/nouveau/include/nvkm/core/subdev.h  |  5 +++--
 drm/nouveau/nouveau_drm.c               |  2 +-
 drm/nouveau/nouveau_nvif.c              |  6 ++++--
 drm/nouveau/nvif/client.c               |  4 ++--
 drm/nouveau/nvkm/core/client.c          |  5 ++---
 drm/nouveau/nvkm/core/engine.c          |  4 ++--
 drm/nouveau/nvkm/core/ioctl.c           |  4 ++--
 drm/nouveau/nvkm/core/object.c          | 22 ++++++++++++++++++----
 drm/nouveau/nvkm/core/oproxy.c          |  2 +-
 drm/nouveau/nvkm/core/subdev.c          |  4 ++--
 drm/nouveau/nvkm/engine/device/base.c   | 11 ++++++-----
 drm/nouveau/nvkm/engine/device/pci.c    |  2 +-
 drm/nouveau/nvkm/engine/device/priv.h   |  2 +-
 drm/nouveau/nvkm/engine/device/tegra.c  |  3 ++-
 drm/nouveau/nvkm/engine/device/user.c   |  2 +-
 drm/nouveau/nvkm/engine/disp/base.c     |  2 +-
 drm/nouveau/nvkm/engine/disp/channv50.c |  2 +-
 drm/nouveau/nvkm/engine/falcon.c        |  2 +-
 drm/nouveau/nvkm/engine/fifo/base.c     |  2 +-
 drm/nouveau/nvkm/engine/fifo/chan.c     |  5 +++--
 drm/nouveau/nvkm/engine/fifo/gf100.c    |  2 +-
 drm/nouveau/nvkm/engine/fifo/gk104.c    |  2 +-
 drm/nouveau/nvkm/engine/gr/base.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv04.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv10.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv20.c       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv20.h       |  2 +-
 drm/nouveau/nvkm/engine/gr/nv40.c       |  2 +-
 drm/nouveau/nvkm/engine/mpeg/nv44.c     |  2 +-
 drm/nouveau/nvkm/engine/pm/base.c       |  2 +-
 drm/nouveau/nvkm/engine/sec2/base.c     |  2 +-
 drm/nouveau/nvkm/engine/xtensa.c        |  2 +-
 drm/nouveau/nvkm/subdev/bar/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/clk/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/devinit/base.c  |  2 +-
 drm/nouveau/nvkm/subdev/fault/base.c    |  2 +-
 drm/nouveau/nvkm/subdev/fault/user.c    |  2 +-
 drm/nouveau/nvkm/subdev/gpio/base.c     |  2 +-
 drm/nouveau/nvkm/subdev/i2c/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/instmem/base.c  |  2 +-
 drm/nouveau/nvkm/subdev/mc/base.c       |  2 +-
 drm/nouveau/nvkm/subdev/pci/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/pmu/base.c      |  2 +-
 drm/nouveau/nvkm/subdev/secboot/base.c  |  2 +-
 drm/nouveau/nvkm/subdev/therm/base.c    |  2 +-
 drm/nouveau/nvkm/subdev/timer/base.c    |  2 +-
 lib/drm.c                               |  2 +-
 lib/main.c                              |  6 ++++--
 lib/null.c                              |  6 ++++--
 56 files changed, 108 insertions(+), 75 deletions(-)
bin/nv_init.c +1 -1
index 90c14d5ea..f67ca9844 100644
@@ -38,7 +38,7 @@ main(int argc, char **argv)
 		return ret;
 
 	if (suspend) {
-		nvif_client_suspend(&client);
+		nvif_client_suspend(&client, false);
 		nvif_client_resume(&client);
 	}
 
drm/nouveau/include/nvif/client.h +1 -1
index e63c6c965..6948ec148 100644
@@ -16,7 +16,7 @@ int  nvif_client_init(struct nvif_client *parent, const char *name, u64 device,
 		      struct nvif_client *);
 void nvif_client_fini(struct nvif_client *);
 int  nvif_client_ioctl(struct nvif_client *, void *, u32);
-int  nvif_client_suspend(struct nvif_client *);
+int  nvif_client_suspend(struct nvif_client *, bool runtime);
 int  nvif_client_resume(struct nvif_client *);
 
 /*XXX*/
drm/nouveau/include/nvif/driver.h +1 -1
index 8e85b936e..a77450add 100644
@@ -9,7 +9,7 @@ struct nvif_driver {
 	int (*init)(const char *name, u64 device, const char *cfg,
 		    const char *dbg, void **priv);
 	void (*fini)(void *priv);
-	int (*suspend)(void *priv);
+	int (*suspend)(void *priv, bool runtime);
 	int (*resume)(void *priv);
 	int (*ioctl)(void *priv, bool super, void *data, u32 size, void **hack);
 	void __iomem *(*map)(void *priv, u64 handle, u32 size);
drm/nouveau/include/nvkm/core/device.h +2 -1
index 6d55cd047..ef8e652a0 100644
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: MIT */
 #ifndef __NVKM_DEVICE_H__
 #define __NVKM_DEVICE_H__
+#include <core/object.h>
 #include <core/oclass.h>
 #include <core/event.h>
 
@@ -188,7 +189,7 @@ struct nvkm_device_func {
 	void *(*dtor)(struct nvkm_device *);
 	int (*preinit)(struct nvkm_device *);
 	int (*init)(struct nvkm_device *);
-	void (*fini)(struct nvkm_device *, bool suspend);
+	void (*fini)(struct nvkm_device *, enum nvkm_suspend_type);
 	resource_size_t (*resource_addr)(struct nvkm_device *, unsigned bar);
 	resource_size_t (*resource_size)(struct nvkm_device *, unsigned bar);
 	bool cpu_coherent;
drm/nouveau/include/nvkm/core/engine.h +1 -1
index c6b401a6e..a020c5589 100644
@@ -20,7 +20,7 @@ struct nvkm_engine_func {
 	int (*oneinit)(struct nvkm_engine *);
 	int (*info)(struct nvkm_engine *, u64 mthd, u64 *data);
 	int (*init)(struct nvkm_engine *);
-	int (*fini)(struct nvkm_engine *, bool suspend);
+	int (*fini)(struct nvkm_engine *, enum nvkm_suspend_type);
 	void (*intr)(struct nvkm_engine *);
 	void (*tile)(struct nvkm_engine *, int region, struct nvkm_fb_tile *);
 	bool (*chsw_load)(struct nvkm_engine *);
drm/nouveau/include/nvkm/core/object.h +11 -2
index 7efcd5d2f..2eecf3c60 100644
@@ -25,10 +25,19 @@ enum nvkm_object_map {
 	NVKM_OBJECT_MAP_VA
 };
 
+enum nvkm_suspend_type {
+	NVKM_SUSPEND_NONE = 0,
+	NVKM_SUSPEND_SYSTEM = 1,
+	NVKM_SUSPEND_RUNTIME = 2,
+};
+
+const char*
+nvkm_suspend_type_str(enum nvkm_suspend_type);
+
 struct nvkm_object_func {
 	void *(*dtor)(struct nvkm_object *);
 	int (*init)(struct nvkm_object *);
-	int (*fini)(struct nvkm_object *, bool suspend);
+	int (*fini)(struct nvkm_object *, enum nvkm_suspend_type);
 	int (*mthd)(struct nvkm_object *, u32 mthd, void *data, u32 size);
 	int (*ntfy)(struct nvkm_object *, u32 mthd, struct nvkm_event **);
 	int (*map)(struct nvkm_object *, void *argv, u32 argc,
@@ -55,7 +64,7 @@ int nvkm_object_new(const struct nvkm_oclass *, void *data, u32 size,
 void nvkm_object_del(struct nvkm_object **);
 void *nvkm_object_dtor(struct nvkm_object *);
 int nvkm_object_init(struct nvkm_object *);
-int nvkm_object_fini(struct nvkm_object *, bool suspend);
+int nvkm_object_fini(struct nvkm_object *, enum nvkm_suspend_type);
 int nvkm_object_mthd(struct nvkm_object *, u32 mthd, void *data, u32 size);
 int nvkm_object_ntfy(struct nvkm_object *, u32 mthd, struct nvkm_event **);
 int nvkm_object_map(struct nvkm_object *, void *argv, u32 argc,
drm/nouveau/include/nvkm/core/oproxy.h +1 -1
index 0e70a9afb..9eb316ad3 100644
@@ -13,7 +13,7 @@ struct nvkm_oproxy {
 struct nvkm_oproxy_func {
 	void (*dtor[2])(struct nvkm_oproxy *);
 	int  (*init[2])(struct nvkm_oproxy *);
-	int  (*fini[2])(struct nvkm_oproxy *, bool suspend);
+	int  (*fini[2])(struct nvkm_oproxy *, enum nvkm_suspend_type);
 };
 
 void nvkm_oproxy_ctor(const struct nvkm_oproxy_func *,
drm/nouveau/include/nvkm/core/subdev.h +3 -2
index 1218f28c1..6bd9ee200 100644
@@ -2,6 +2,7 @@
 #ifndef __NVKM_SUBDEV_H__
 #define __NVKM_SUBDEV_H__
 #include <core/device.h>
+#include <core/object.h>
 
 struct nvkm_subdev {
 	const struct nvkm_subdev_func *func;
@@ -19,7 +20,7 @@ struct nvkm_subdev_func {
 	int (*oneinit)(struct nvkm_subdev *);
 	int (*info)(struct nvkm_subdev *, u64 mthd, u64 *data);
 	int (*init)(struct nvkm_subdev *);
-	int (*fini)(struct nvkm_subdev *, bool suspend);
+	int (*fini)(struct nvkm_subdev *, enum nvkm_suspend_type);
 	void (*intr)(struct nvkm_subdev *);
 };
 
@@ -29,7 +30,7 @@ void nvkm_subdev_ctor(const struct nvkm_subdev_func *, struct nvkm_device *,
 void nvkm_subdev_del(struct nvkm_subdev **);
 int  nvkm_subdev_preinit(struct nvkm_subdev *);
 int  nvkm_subdev_init(struct nvkm_subdev *);
-int  nvkm_subdev_fini(struct nvkm_subdev *, bool suspend);
+int  nvkm_subdev_fini(struct nvkm_subdev *, enum nvkm_suspend_type);
 int  nvkm_subdev_info(struct nvkm_subdev *, u64, u64 *);
 void nvkm_subdev_intr(struct nvkm_subdev *);
 
drm/nouveau/nouveau_drm.c +1 -1
index 466271c0c..0162cea8e 100644
@@ -783,7 +783,7 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
 	}
 
 	NV_DEBUG(drm, "suspending object tree...\n");
-	ret = nvif_client_suspend(&drm->master.base);
+	ret = nvif_client_suspend(&drm->master.base, runtime);
 	if (ret)
 		goto fail_client;
 
drm/nouveau/nouveau_nvif.c +4 -2
index b3f29b1ce..885abb3a5 100644
@@ -65,10 +65,12 @@ nvkm_client_resume(void *priv)
 }
 
 static int
-nvkm_client_suspend(void *priv)
+nvkm_client_suspend(void *priv, bool runtime)
 {
 	struct nvkm_client *client = priv;
-	return nvkm_object_fini(&client->object, true);
+	enum nvkm_suspend_type suspend =
+		runtime ? NVKM_SUSPEND_RUNTIME : NVKM_SUSPEND_SYSTEM;
+	return nvkm_object_fini(&client->object, suspend);
 }
 
 static int
drm/nouveau/nvif/client.c +2 -2
index 12db54965..30fc0b284 100644
@@ -36,9 +36,9 @@ nvif_client_ioctl(struct nvif_client *client, void *data, u32 size)
 }
 
 int
-nvif_client_suspend(struct nvif_client *client)
+nvif_client_suspend(struct nvif_client *client, bool runtime)
 {
-	return client->driver->suspend(client->object.priv);
+	return client->driver->suspend(client->object.priv, runtime);
 }
 
 int
drm/nouveau/nvkm/core/client.c +2 -3
index ac6712029..42a6850c6 100644
@@ -253,12 +253,11 @@ nvkm_client_child_get(struct nvkm_object *object, int index,
 }
 
 static int
-nvkm_client_fini(struct nvkm_object *object, bool suspend)
+nvkm_client_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_client *client = nvkm_client(object);
-	const char *name[2] = { "fini", "suspend" };
 	int i;
-	nvif_debug(object, "%s notify\n", name[suspend]);
+	nvif_debug(object, "%s notify\n", nvkm_suspend_type_str(suspend));
 	for (i = 0; i < ARRAY_SIZE(client->notify); i++)
 		nvkm_client_notify_put(client, i);
 	return 0;
drm/nouveau/nvkm/core/engine.c +2 -2
index 1a47c40e1..9a5739ca2 100644
@@ -42,7 +42,7 @@ nvkm_engine_unref(struct nvkm_engine **pengine)
 	if (engine) {
 		mutex_lock(&engine->subdev.mutex);
 		if (--engine->usecount == 0)
-			nvkm_subdev_fini(&engine->subdev, false);
+			nvkm_subdev_fini(&engine->subdev, NVKM_SUSPEND_NONE);
 		mutex_unlock(&engine->subdev.mutex);
 		*pengine = NULL;
 	}
@@ -98,7 +98,7 @@ nvkm_engine_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data)
 }
 
 static int
-nvkm_engine_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_engine_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_engine *engine = nvkm_engine(subdev);
 	if (engine->func->fini)
drm/nouveau/nvkm/core/ioctl.c +2 -2
index d777df5a6..b2277e19d 100644
@@ -135,7 +135,7 @@ nvkm_ioctl_new(struct nvkm_client *client,
 			}
 			ret = -EEXIST;
 		}
-		nvkm_object_fini(object, false);
+		nvkm_object_fini(object, NVKM_SUSPEND_NONE);
 	}
 
 	nvkm_object_del(&object);
@@ -154,7 +154,7 @@ nvkm_ioctl_del(struct nvkm_client *client,
 	nvif_ioctl(object, "delete size %d\n", size);
 	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		nvif_ioctl(object, "delete\n");
-		nvkm_object_fini(object, false);
+		nvkm_object_fini(object, NVKM_SUSPEND_NONE);
 		nvkm_object_del(&object);
 	}
 
drm/nouveau/nvkm/core/object.c +18 -4
index 301a5e5b5..8ab0e4437 100644
@@ -175,10 +175,24 @@ nvkm_object_bind(struct nvkm_object *object, struct nvkm_gpuobj *gpuobj,
 	return -ENODEV;
 }
 
+const char *
+nvkm_suspend_type_str(enum nvkm_suspend_type type)
+{
+	switch (type) {
+	case NVKM_SUSPEND_NONE:
+		return "fini";
+	case NVKM_SUSPEND_SYSTEM:
+		return "suspend";
+	case NVKM_SUSPEND_RUNTIME:
+		return "runtime suspend";
+	}
+	return "";
+}
+
 int
-nvkm_object_fini(struct nvkm_object *object, bool suspend)
+nvkm_object_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
-	const char *action = suspend ? "suspend" : "fini";
+	const char *action = nvkm_suspend_type_str(suspend);
 	struct nvkm_object *child;
 	s64 time;
 	int ret;
@@ -246,11 +260,11 @@ nvkm_object_init(struct nvkm_object *object)
 
 fail_child:
 	list_for_each_entry_continue_reverse(child, &object->tree, head)
-		nvkm_object_fini(child, false);
+		nvkm_object_fini(child, NVKM_SUSPEND_NONE);
 fail:
 	nvif_error(object, "init failed with %d\n", ret);
 	if (object->func->fini)
-		object->func->fini(object, false);
+		object->func->fini(object, NVKM_SUSPEND_NONE);
 	return ret;
 }
 
drm/nouveau/nvkm/core/oproxy.c +1 -1
index 16299837a..91bdd9ea3 100644
@@ -106,7 +106,7 @@ nvkm_oproxy_sclass(struct nvkm_object *object, int index,
 }
 
 static int
-nvkm_oproxy_fini(struct nvkm_object *object, bool suspend)
+nvkm_oproxy_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_oproxy *oproxy = nvkm_oproxy(object);
 	int ret;
drm/nouveau/nvkm/core/subdev.c +2 -2
index 245990de1..95402b6c9 100644
@@ -107,10 +107,10 @@ nvkm_subdev_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data)
 }
 
 int
-nvkm_subdev_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_subdev_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_device *device = subdev->device;
-	const char *action = suspend ? "suspend" : "fini";
+	const char *action = nvkm_suspend_type_str(suspend);
 	s64 time;
 
 	nvkm_trace(subdev, "%s running...\n", action);
drm/nouveau/nvkm/engine/device/base.c +6 -5
index c3c7159f3..4c5cdba82 100644
@@ -2724,13 +2724,14 @@ nvkm_device_engine(struct nvkm_device *device, int index)
 }
 
 int
-nvkm_device_fini(struct nvkm_device *device, bool suspend)
+nvkm_device_fini(struct nvkm_device *device, enum nvkm_suspend_type suspend)
 {
-	const char *action = suspend ? "suspend" : "fini";
 	struct nvkm_subdev *subdev;
 	int ret, i;
 	s64 time;
 
+	const char *action = nvkm_suspend_type_str(suspend);
+
 	nvdev_trace(device, "%s running...\n", action);
 	time = ktime_to_us(ktime_get());
 
@@ -2814,7 +2815,7 @@ nvkm_device_init(struct nvkm_device *device)
 	if (ret)
 		return ret;
 
-	nvkm_device_fini(device, false);
+	nvkm_device_fini(device, NVKM_SUSPEND_NONE);
 
 	nvdev_trace(device, "init running...\n");
 	time = ktime_to_us(ktime_get());
@@ -2843,11 +2844,11 @@ nvkm_device_init(struct nvkm_device *device)
 fail_subdev:
 	do {
 		if ((subdev = nvkm_device_subdev(device, i)))
-			nvkm_subdev_fini(subdev, false);
+			nvkm_subdev_fini(subdev, NVKM_SUSPEND_NONE);
 	} while (--i >= 0);
 
 fail:
-	nvkm_device_fini(device, false);
+	nvkm_device_fini(device, NVKM_SUSPEND_NONE);
 
 	nvdev_error(device, "init failed with %d\n", ret);
 	return ret;
drm/nouveau/nvkm/engine/device/pci.c +1 -1
index f302d2b57..83e558b22 100644
@@ -1575,7 +1575,7 @@ nvkm_device_pci_resource_size(struct nvkm_device *device, unsigned bar)
 }
 
 static void
-nvkm_device_pci_fini(struct nvkm_device *device, bool suspend)
+nvkm_device_pci_fini(struct nvkm_device *device, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_device_pci *pdev = nvkm_device_pci(device);
 	if (suspend) {
drm/nouveau/nvkm/engine/device/priv.h +1 -1
index d8be2f77a..f08d27708 100644
@@ -56,5 +56,5 @@ int  nvkm_device_ctor(const struct nvkm_device_func *,
 		      bool detect, bool mmio, u64 subdev_mask,
 		      struct nvkm_device *);
 int  nvkm_device_init(struct nvkm_device *);
-int  nvkm_device_fini(struct nvkm_device *, bool suspend);
+int  nvkm_device_fini(struct nvkm_device *, enum nvkm_suspend_type);
 #endif
drm/nouveau/nvkm/engine/device/tegra.c +2 -1
index 0e372a190..cefed7718 100644
@@ -223,7 +223,8 @@ nvkm_device_tegra_intr(int irq, void *arg)
 }
 
 static void
-nvkm_device_tegra_fini(struct nvkm_device *device, bool suspend)
+nvkm_device_tegra_fini(struct nvkm_device *device,
+		       enum nvkm_suspend_type suspend)
 {
 	struct nvkm_device_tegra *tdev = nvkm_device_tegra(device);
 	if (tdev->irq) {
drm/nouveau/nvkm/engine/device/user.c +1 -1
index 03c6d9aef..46f6c64ad 100644
@@ -291,7 +291,7 @@ nvkm_udevice_map(struct nvkm_object *object, void *argv, u32 argc,
 }
 
 static int
-nvkm_udevice_fini(struct nvkm_object *object, bool suspend)
+nvkm_udevice_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_udevice *udev = nvkm_udevice(object);
 	struct nvkm_device *device = udev->device;
drm/nouveau/nvkm/engine/disp/base.c +1 -1
index cbd33e87b..b6a11eca8 100644
@@ -214,7 +214,7 @@ nvkm_disp_intr(struct nvkm_engine *engine)
 }
 
 static int
-nvkm_disp_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_disp_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_disp *disp = nvkm_disp(engine);
 	struct nvkm_conn *conn;
drm/nouveau/nvkm/engine/disp/channv50.c +1 -1
index bcf32d92e..043771de2 100644
@@ -293,7 +293,7 @@ nv50_disp_chan_child_get(struct nvkm_object *object, int index,
 }
 
 static int
-nv50_disp_chan_fini(struct nvkm_object *object, bool suspend)
+nv50_disp_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
 	chan->func->fini(chan);
drm/nouveau/nvkm/engine/falcon.c +1 -1
index 8675613e1..f15a7a938 100644
@@ -93,7 +93,7 @@ nvkm_falcon_intr(struct nvkm_engine *engine)
 }
 
 static int
-nvkm_falcon_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_falcon_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_falcon *falcon = nvkm_falcon(engine);
 	struct nvkm_device *device = falcon->engine.subdev.device;
drm/nouveau/nvkm/engine/fifo/base.c +1 -1
index c773caf21..b714bf01c 100644
@@ -279,7 +279,7 @@ nvkm_fifo_intr(struct nvkm_engine *engine)
 }
 
 static int
-nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_fifo_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
 	if (fifo->func->fini)
drm/nouveau/nvkm/engine/fifo/chan.c +3 -2
index d83485385..9bd1ae91a 100644
@@ -36,7 +36,8 @@ struct nvkm_fifo_chan_object {
 };
 
 static int
-nvkm_fifo_chan_child_fini(struct nvkm_oproxy *base, bool suspend)
+nvkm_fifo_chan_child_fini(struct nvkm_oproxy *base,
+			  enum nvkm_suspend_type suspend)
 {
 	struct nvkm_fifo_chan_object *object =
 		container_of(base, typeof(*object), oproxy);
@@ -294,7 +295,7 @@ nvkm_fifo_chan_wr32(struct nvkm_object *object, u64 addr, u32 data)
 }
 
 static int
-nvkm_fifo_chan_fini(struct nvkm_object *object, bool suspend)
+nvkm_fifo_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object);
 	chan->func->fini(chan);
drm/nouveau/nvkm/engine/fifo/gf100.c +1 -1
index 5a39e51d4..2ff9436a1 100644
@@ -162,7 +162,7 @@ gf100_fifo_recover_work(struct work_struct *w)
 
 	for (todo = mask; engn = __ffs64(todo), todo; todo &= ~BIT_ULL(engn)) {
 		if ((engine = nvkm_device_engine(device, engn))) {
-			nvkm_subdev_fini(&engine->subdev, false);
+			nvkm_subdev_fini(&engine->subdev, NVKM_SUSPEND_NONE);
 			WARN_ON(nvkm_subdev_init(&engine->subdev));
 		}
 	}
drm/nouveau/nvkm/engine/fifo/gk104.c +1 -1
index 5d4b695ca..4460e6240 100644
@@ -292,7 +292,7 @@ gk104_fifo_recover_work(struct work_struct *w)
 
 	for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT(engn)) {
 		if ((engine = fifo->engine[engn].engine)) {
-			nvkm_subdev_fini(&engine->subdev, false);
+			nvkm_subdev_fini(&engine->subdev, NVKM_SUSPEND_NONE);
 			WARN_ON(nvkm_subdev_init(&engine->subdev));
 		}
 	}
drm/nouveau/nvkm/engine/gr/base.c +1 -1
index d41fb9452..25d6bbe70 100644
@@ -143,7 +143,7 @@ nvkm_gr_init(struct nvkm_engine *engine)
 }
 
 static int
-nvkm_gr_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_gr_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_gr *gr = nvkm_gr(engine);
 	if (gr->func->fini)
drm/nouveau/nvkm/engine/gr/nv04.c +1 -1
index 9c2e985dc..715bdd063 100644
@@ -1158,7 +1158,7 @@ nv04_gr_chan_dtor(struct nvkm_object *object)
 }
 
 static int
-nv04_gr_chan_fini(struct nvkm_object *object, bool suspend)
+nv04_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nv04_gr_chan *chan = nv04_gr_chan(object);
 	struct nv04_gr *gr = chan->gr;
drm/nouveau/nvkm/engine/gr/nv10.c +1 -1
index 4ebbfbdd8..f0bb7ed2a 100644
@@ -951,7 +951,7 @@ nv10_gr_context_switch(struct nv10_gr *gr)
 }
 
 static int
-nv10_gr_chan_fini(struct nvkm_object *object, bool suspend)
+nv10_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nv10_gr_chan *chan = nv10_gr_chan(object);
 	struct nv10_gr *gr = chan->gr;
drm/nouveau/nvkm/engine/gr/nv20.c +1 -1
index d837630a3..a0730b598 100644
@@ -27,7 +27,7 @@ nv20_gr_chan_init(struct nvkm_object *object)
 }
 
 int
-nv20_gr_chan_fini(struct nvkm_object *object, bool suspend)
+nv20_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nv20_gr_chan *chan = nv20_gr_chan(object);
 	struct nv20_gr *gr = chan->gr;
drm/nouveau/nvkm/engine/gr/nv20.h +1 -1
index e57407a8a..cf8e5e2eb 100644
@@ -31,5 +31,5 @@ struct nv20_gr_chan {
 
 void *nv20_gr_chan_dtor(struct nvkm_object *);
 int nv20_gr_chan_init(struct nvkm_object *);
-int nv20_gr_chan_fini(struct nvkm_object *, bool);
+int nv20_gr_chan_fini(struct nvkm_object *, enum nvkm_suspend_type);
 #endif
drm/nouveau/nvkm/engine/gr/nv40.c +1 -1
index 5f1ad8344..e98c7f387 100644
@@ -89,7 +89,7 @@ nv40_gr_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent,
 }
 
 static int
-nv40_gr_chan_fini(struct nvkm_object *object, bool suspend)
+nv40_gr_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nv40_gr_chan *chan = nv40_gr_chan(object);
 	struct nv40_gr *gr = chan->gr;
drm/nouveau/nvkm/engine/mpeg/nv44.c +1 -1
index c3cf02ed4..6c540bc92 100644
@@ -65,7 +65,7 @@ nv44_mpeg_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent,
 }
 
 static int
-nv44_mpeg_chan_fini(struct nvkm_object *object, bool suspend)
+nv44_mpeg_chan_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 
 	struct nv44_mpeg_chan *chan = nv44_mpeg_chan(object);
drm/nouveau/nvkm/engine/pm/base.c +1 -1
index b2785bee4..e87e0c311 100644
@@ -820,7 +820,7 @@ nvkm_perfdom_new(struct nvkm_pm *pm, const char *name, u32 mask,
 }
 
 static int
-nvkm_pm_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_pm_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_pm *pm = nvkm_pm(engine);
 	if (pm->func->fini)
drm/nouveau/nvkm/engine/sec2/base.c +1 -1
index 1b49e5b67..22d66f356 100644
@@ -87,7 +87,7 @@ nvkm_sec2_oneinit(struct nvkm_engine *engine)
 }
 
 static int
-nvkm_sec2_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_sec2_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_sec2 *sec2 = nvkm_sec2(engine);
 	flush_work(&sec2->work);
drm/nouveau/nvkm/engine/xtensa.c +1 -1
index 70549381e..9e7bcac13 100644
@@ -76,7 +76,7 @@ nvkm_xtensa_intr(struct nvkm_engine *engine)
 }
 
 static int
-nvkm_xtensa_fini(struct nvkm_engine *engine, bool suspend)
+nvkm_xtensa_fini(struct nvkm_engine *engine, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_xtensa *xtensa = nvkm_xtensa(engine);
 	struct nvkm_device *device = xtensa->engine.subdev.device;
drm/nouveau/nvkm/subdev/bar/base.c +1 -1
index 209a6a408..d2ddb8a50 100644
@@ -90,7 +90,7 @@ nvkm_bar_bar2_init(struct nvkm_device *device)
 }
 
 static int
-nvkm_bar_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_bar_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_bar *bar = nvkm_bar(subdev);
 	if (bar->func->bar1.fini)
drm/nouveau/nvkm/subdev/clk/base.c +1 -1
index ba6a868d4..2b15bc74b 100644
@@ -576,7 +576,7 @@ nvkm_clk_read(struct nvkm_clk *clk, enum nv_clk_src src)
 }
 
 static int
-nvkm_clk_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_clk_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_clk *clk = nvkm_clk(subdev);
 	nvkm_notify_put(&clk->pwrsrc_ntfy);
drm/nouveau/nvkm/subdev/devinit/base.c +1 -1
index 4756019dd..5a58f4dec 100644
@@ -66,7 +66,7 @@ nvkm_devinit_post(struct nvkm_devinit *init, u64 *disable)
 }
 
 static int
-nvkm_devinit_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_devinit_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_devinit *init = nvkm_devinit(subdev);
 	/* force full reinit on resume */
drm/nouveau/nvkm/subdev/fault/base.c +1 -1
index ca251560d..6b2f9d101 100644
@@ -67,7 +67,7 @@ nvkm_fault_intr(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_fault_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_fault_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_fault *fault = nvkm_fault(subdev);
 	if (fault->func->fini)
drm/nouveau/nvkm/subdev/fault/user.c +1 -1
index ac835c958..3400add40 100644
@@ -52,7 +52,7 @@ nvkm_ufault_ntfy(struct nvkm_object *object, u32 type,
 }
 
 static int
-nvkm_ufault_fini(struct nvkm_object *object, bool suspend)
+nvkm_ufault_fini(struct nvkm_object *object, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object);
 	buffer->fault->func->buffer.fini(buffer);
drm/nouveau/nvkm/subdev/gpio/base.c +1 -1
index 914276410..c71426919 100644
@@ -162,7 +162,7 @@ nvkm_gpio_intr(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_gpio_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_gpio_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_gpio *gpio = nvkm_gpio(subdev);
 	u32 mask = (1ULL << gpio->func->lines) - 1;
drm/nouveau/nvkm/subdev/i2c/base.c +1 -1
index 719345074..0fdab8515 100644
@@ -156,7 +156,7 @@ nvkm_i2c_intr(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_i2c_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_i2c_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_i2c *i2c = nvkm_i2c(subdev);
 	struct nvkm_i2c_pad *pad;
drm/nouveau/nvkm/subdev/instmem/base.c +1 -1
index 364ea4492..3f0587d6b 100644
@@ -157,7 +157,7 @@ nvkm_instmem_boot(struct nvkm_instmem *imem)
 }
 
 static int
-nvkm_instmem_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_instmem_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_instmem *imem = nvkm_instmem(subdev);
 	struct nvkm_instobj *iobj;
drm/nouveau/nvkm/subdev/mc/base.c +1 -1
index 0e57ab2a7..7fef7331f 100644
@@ -175,7 +175,7 @@ nvkm_mc_enabled(struct nvkm_device *device, enum nvkm_devidx devidx)
 
 
 static int
-nvkm_mc_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_mc_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	nvkm_mc_intr_unarm(subdev->device);
 	return 0;
drm/nouveau/nvkm/subdev/pci/base.c +1 -1
index ee2431a78..9db880898 100644
@@ -84,7 +84,7 @@ nvkm_pci_intr(int irq, void *arg)
 }
 
 static int
-nvkm_pci_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_pci_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_pci *pci = nvkm_pci(subdev);
 
drm/nouveau/nvkm/subdev/pmu/base.c +1 -1
index ea2e11771..6df9c5586 100644
@@ -77,7 +77,7 @@ nvkm_pmu_intr(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_pmu_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_pmu *pmu = nvkm_pmu(subdev);
 
drm/nouveau/nvkm/subdev/secboot/base.c +1 -1
index ee29c6c11..95a65e626 100644
@@ -161,7 +161,7 @@ nvkm_secboot_oneinit(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_secboot_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_secboot_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_secboot *sb = nvkm_secboot(subdev);
 	int ret = 0;
drm/nouveau/nvkm/subdev/therm/base.c +1 -1
index 4a4d1e224..66884a697 100644
@@ -341,7 +341,7 @@ nvkm_therm_intr(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_therm_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_therm_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 {
 	struct nvkm_therm *therm = nvkm_therm(subdev);
 
drm/nouveau/nvkm/subdev/timer/base.c +1 -1
index dd9220336..3251c9de0 100644
@@ -149,7 +149,7 @@ nvkm_timer_intr(struct nvkm_subdev *subdev)
 }
 
 static int
-nvkm_timer_fini(struct nvkm_subdev *subdev, bool suspend)
+nvkm_timer_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type type)
 {
 	struct nvkm_timer *tmr = nvkm_timer(subdev);
 	tmr->func->alarm_fini(tmr);
lib/drm.c +1 -1
index 39bc36955..6af0babe3 100644
@@ -74,7 +74,7 @@ drm_client_resume(void *priv)
 }
 
 static int
-drm_client_suspend(void *priv)
+drm_client_suspend(void *priv, bool runtime)
 {
 	return -ENOSYS;
 }
lib/main.c +4 -2
index 2e127c250..9a1cbeb87 100644
@@ -250,10 +250,12 @@ os_client_resume(void *priv)
 }
 
 static int
-os_client_suspend(void *priv)
+os_client_suspend(void *priv, bool runtime)
 {
 	struct nvkm_client *client = priv;
-	return nvkm_object_fini(&client->object, true);
+	enum nvkm_suspend_type suspend =
+		runtime ? NVKM_SUSPEND_RUNTIME : NVKM_SUSPEND_SYSTEM;
+	return nvkm_object_fini(&client->object, suspend);
 }
 
 static void
lib/null.c +4 -2
index 912d139fa..2c08b1b61 100644
@@ -88,10 +88,12 @@ null_client_resume(void *priv)
 }
 
 static int
-null_client_suspend(void *priv)
+null_client_suspend(void *priv, bool runtime)
 {
 	struct nvkm_client *client = priv;
-	return nvkm_object_fini(&client->object, true);
+	enum nvkm_suspend_type suspend =
+		runtime ? NVKM_SUSPEND_RUNTIME : NVKM_SUSPEND_SYSTEM;
+	return nvkm_object_fini(&client->object, suspend);
 }
 
 static void
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#8 lore raw

PATCH 7/8 pci: set the pcie link speed to 8.0 when suspending

In reply to a message not in the archive

Apperantly things go south if we suspend the device with a PCIe link speed set to 2.5. Fixes runtime suspend on my gp107.

This all looks like some bug inside the pci subsystem and I would prefer a fix there instead of nouveau, but maybe there is no real nice way of doing that outside of drivers?

v2: squashed together patch 4 and 5
v3: only restore pcie speed on machines with runpm
add NvRunpmWorkaround config option to disable the workaround
v4: only run the code on suspend
always put the card into 8.0 mode, not what nouveau detected on load
v5: only enable workaround on runtime suspend

---
 drm/nouveau/include/nvkm/core/device.h |  2 ++
 drm/nouveau/include/nvkm/subdev/pci.h  |  3 ++-
 drm/nouveau/nouveau_drm.c              |  1 +
 drm/nouveau/nvkm/subdev/clk/base.c     |  2 +-
 drm/nouveau/nvkm/subdev/pci/base.c     |  2 ++
 drm/nouveau/nvkm/subdev/pci/pcie.c     | 28 ++++++++++++++++++++++----
 drm/nouveau/nvkm/subdev/pci/priv.h     |  1 +
 7 files changed, 33 insertions(+), 6 deletions(-)
drm/nouveau/include/nvkm/core/device.h +2 -0
index ef8e652a0..173dfcb31 100644
@@ -126,6 +126,8 @@ struct nvkm_device {
 	u8  chiprev;
 	u32 crystal;
 
+	bool has_runpm;
+
 	struct {
 		struct notifier_block nb;
 	} acpi;
drm/nouveau/include/nvkm/subdev/pci.h +2 -1
index 4803a4fad..10d361e56 100644
@@ -52,5 +52,6 @@ int gk104_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int gp100_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 
 /* pcie functions */
-int nvkm_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8 width);
+int nvkm_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8 width,
+		       bool save);
 #endif
drm/nouveau/nouveau_drm.c +1 -0
index 0162cea8e..dce8e7fe6 100644
@@ -676,6 +676,7 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
 
 	if (nouveau_atomic)
 		driver_pci.driver_features |= DRIVER_ATOMIC;
+	device->has_runpm = nouveau_pmops_runtime();
 
 	drm_dev = drm_dev_alloc(&driver_pci, &pdev->dev);
 	if (IS_ERR(drm_dev)) {
drm/nouveau/nvkm/subdev/clk/base.c +1 -1
index 2b15bc74b..2d1b10f6c 100644
@@ -277,7 +277,7 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 	nvkm_debug(subdev, "setting performance state %d\n", pstatei);
 	clk->pstate = pstatei;
 
-	nvkm_pcie_set_link(pci, pstate->pcie_speed, pstate->pcie_width);
+	nvkm_pcie_set_link(pci, pstate->pcie_speed, pstate->pcie_width, true);
 
 	if (fb && fb->ram && fb->ram->func->calc) {
 		struct nvkm_ram *ram = fb->ram;
drm/nouveau/nvkm/subdev/pci/base.c +2 -0
index 9db880898..986d61ce9 100644
@@ -90,6 +90,8 @@ nvkm_pci_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_type suspend)
 
 	if (pci->agp.bridge)
 		nvkm_agp_fini(pci);
+	else if (pci_is_pcie(pci->pdev))
+		nvkm_pcie_fini(pci, suspend);
 
 	return 0;
 }
drm/nouveau/nvkm/subdev/pci/pcie.c +24 -4
index ed015b252..0363f54ac 100644
@@ -23,6 +23,8 @@
  */
 #include "priv.h"
 
+#include <core/option.h>
+
 static char *nvkm_pcie_speeds[] = {
 	"2.5GT/s",
 	"5.0GT/s",
@@ -106,11 +108,26 @@ nvkm_pcie_init(struct nvkm_pci *pci)
 		pci->func->pcie.init(pci);
 
 	if (pci->pcie.speed != -1)
-		nvkm_pcie_set_link(pci, pci->pcie.speed, pci->pcie.width);
+		nvkm_pcie_set_link(pci, pci->pcie.speed,
+				   pci->pcie.width, false);
 
 	return 0;
 }
 
+int
+nvkm_pcie_fini(struct nvkm_pci *pci, enum nvkm_suspend_type suspend)
+{
+	struct nvkm_device *device = pci->subdev.device;
+
+	if (!device->has_runpm || suspend != NVKM_SUSPEND_RUNTIME)
+		return 0;
+
+	if (!nvkm_boolopt(device->cfgopt, "NvRunpmWorkaround", true))
+		return 0;
+
+	return nvkm_pcie_set_link(pci, NVKM_PCIE_SPEED_8_0, 16, false);
+}
+
 enum nvkm_pci_aspm
 nvkm_pcie_aspm_off(struct nvkm_pci *pci, enum nvkm_pci_aspm states)
 {
@@ -120,7 +137,8 @@ nvkm_pcie_aspm_off(struct nvkm_pci *pci, enum nvkm_pci_aspm states)
 }
 
 int
-nvkm_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
+nvkm_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width,
+		   bool save)
 {
 	struct nvkm_subdev *subdev = &pci->subdev;
 	enum nvkm_pcie_speed cur_speed, max_speed;
@@ -155,8 +173,10 @@ nvkm_pcie_set_link(struct nvkm_pci *pci, enum nvkm_pcie_speed speed, u8 width)
 		speed = max_speed;
 	}
 
-	pci->pcie.speed = speed;
-	pci->pcie.width = width;
+	if (save) {
+		pci->pcie.speed = speed;
+		pci->pcie.width = width;
+	}
 
 	if (speed == cur_speed) {
 		nvkm_debug(subdev, "requested matches current speed\n");
drm/nouveau/nvkm/subdev/pci/priv.h +1 -0
index 17be1d563..120ba2138 100644
@@ -71,4 +71,5 @@ int gk104_pcie_version_supported(struct nvkm_pci *);
 
 int nvkm_pcie_oneinit(struct nvkm_pci *);
 int nvkm_pcie_init(struct nvkm_pci *);
+int nvkm_pcie_fini(struct nvkm_pci *, enum nvkm_suspend_type);
 #endif
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau
#9 lore raw

PATCH 8/8 drm: abort runtime suspend if we hit an error

In reply to a message not in the archive
---
 drm/nouveau/nouveau_drm.c | 5 +++++
 1 file changed, 5 insertions(+)
drm/nouveau/nouveau_drm.c +5 -0
index dce8e7fe6..ad1311b21 100644
@@ -911,6 +911,7 @@ nouveau_pmops_runtime_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct nouveau_drm *drm = nouveau_drm(drm_dev);
 	int ret;
 
 	if (!nouveau_pmops_runtime()) {
@@ -921,6 +922,10 @@ nouveau_pmops_runtime_suspend(struct device *dev)
 	drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 	nouveau_switcheroo_optimus_dsm();
 	ret = nouveau_do_suspend(drm_dev, true);
+	if (ret) {
+		NV_ERROR(drm, "suspend failed with: %d\n", ret);
+		return ret;
+	}
 	pci_save_state(pdev);
 	pci_disable_device(pdev);
 	pci_ignore_hotplug(pdev);
Signature · 6 lines
2.21.0

_______________________________________________
Nouveau mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/nouveau