Skip to contentSkip to conversation

drm/xe/pm: d3cold policy

RFC In review Anshuman Gupta 4 participants 8 messages May 12, 2023 – Jun 8, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

RFC 1/1 drm/xe/pm: d3cold policy

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the permissisble threshold for vram save/restore. If vram used is more then the threshold value then disabllow the d3cold to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
 drivers/gpu/drm/xe/xe_device_types.h |  3 ++
 drivers/gpu/drm/xe/xe_module.c       |  4 +++
 drivers/gpu/drm/xe/xe_module.h       |  1 +
 drivers/gpu/drm/xe/xe_pci.c          | 46 ++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++
 drivers/gpu/drm/xe/xe_pm.h           |  1 +
 6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +3 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
 		bool hold_rpm;
 	} mem_access;
 
+	/** d3cold_capable: Indicates if root port is d3cold capable */
+	bool d3cold_capable;
+
 	/** @d3cold_allowed: Indicates if d3cold is a valid device state */
 	bool d3cold_allowed;
 
drivers/gpu/drm/xe/xe_module.c +4 -0
index 6860586ce7f8..1dd515ed8a65 100644
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
 module_param_named(vram_bar_size, xe_force_vram_bar_size, uint, 0600);
 MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");
 
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold, xe_vram_save_restore_threshold, uint, 0600);
+MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram used threshold below which vram save/restore premissible (in MB)");
+
 int xe_guc_log_level = 5;
 module_param_named(guc_log_level, xe_guc_log_level, int, 0600);
 MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)");
drivers/gpu/drm/xe/xe_module.h +1 -0
index 7169907c3365..620d1b80cab6 100644
@@ -9,5 +9,6 @@
 extern bool enable_guc;
 extern bool enable_display;
 extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
 extern int xe_guc_log_level;
 extern char *xe_param_force_probe;
drivers/gpu/drm/xe/xe_pci.c +46 -0

76 lines not on this page · raw message

drivers/gpu/drm/xe/xe_pm.c +20 -0
index b7b57f10ba25..523d5f89ec89 100644
@@ -16,6 +16,7 @@
 #include "xe_ggtt.h"
 #include "xe_gt.h"
 #include "xe_irq.h"
+#include "xe_module.h"
 #include "xe_pcode.h"
 
 /**
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
 	return 0;
 }
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe)
+{
+	struct ttm_resource_manager *man;
+	u64 vram_used;
+	size_t total_vram_used_mb = 0;
+	int i;
+
+	/* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+	for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+		man = ttm_manager_type(&xe->ttm, i);
+		if (man) {
+			vram_used = ttm_resource_manager_usage(man);
+			total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+		}
+	}
+
+	return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+
 void xe_pm_runtime_init(struct xe_device *xe)
 {
 	struct device *dev = xe->drm.dev;
drivers/gpu/drm/xe/xe_pm.h +1 -0
index 6a885585f653..33515d5c2cc2 100644
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
 void xe_pm_runtime_fini(struct xe_device *xe);
 int xe_pm_runtime_suspend(struct xe_device *xe);
 int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
 int xe_pm_runtime_get(struct xe_device *xe);
 int xe_pm_runtime_put(struct xe_device *xe);
 bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
Signature · 1 line
2.38.0
#1 lore raw

[Intel-xe] ✗ CI.Patch_applied: failure for drm/xe/pm: d3cold policy

== Series Details ==

== Series Details ==

Series: drm/xe/pm: d3cold policy
URL : https://patchwork.freedesktop.org/series/117680/
State : failure

== Summary ==

=== Applying kernel patches on branch 'drm-xe-next' with base: === Base commit: a9affcc66 fixup! drm/i915/display: Remaining changes to make xe compile === git am output follows ===
error: patch failed: drivers/gpu/drm/xe/xe_pci.c:592
error: drivers/gpu/drm/xe/xe_pci.c: patch does not apply
hint: Use 'git am --show-current-patch' to see the failed patch
Applying: drm/xe/pm: d3cold policy
Patch failed at 0001 drm/xe/pm: d3cold policy
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".

#2 lore raw
In reply to Anshuman Gupta
Quoted 2 lines · Anshuman Gupta

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the

Please use imperative language as per the Linux patch-submission guidelines.

"Add pci d3cold capable check support."

Typos:

Quoted 1 line

permissisble threshold for vram save/restore. If vram used

permissible

Quoted 1 line

is more then

more than

Quoted 1 line

the threshold value then disabllow

disallow

Quoted 47 lines

the d3cold
to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
 drivers/gpu/drm/xe/xe_device_types.h |  3 ++ drivers/gpu/drm/xe/xe_module.c       |  4 +++ drivers/gpu/drm/xe/xe_module.h       |  1 + drivers/gpu/drm/xe/xe_pci.c          | 46
++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h           |  1 +

 6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +0 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
                 bool hold_rpm;
         } mem_access;


+       /** d3cold_capable: Indicates if root port is d3cold capable */
+       bool d3cold_capable;
+
        /** @d3cold_allowed: Indicates if d3cold is a valid device state */
        bool d3cold_allowed;

drivers/gpu/drm/xe/xe_module.c +0 -0
index 6860586ce7f8..1dd515ed8a65 100644
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
  module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,
 0600);
  MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");

drivers/gpu/drm/xe/xe_module.c +2 -0
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold,
Quoted 3 lines

xe_vram_save_restore_threshold, uint, 0600); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram used threshold below which vram save/restore premissible (in MB)");

With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?

Thanks,
Thomas

Quoted 163 lines

+
 int xe_guc_log_level = 5; module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)");

drivers/gpu/drm/xe/xe_module.h +1 -0
index 7169907c3365..620d1b80cab6 100644
@@ -9,5 +9,6 @@
  extern bool enable_guc;
  extern bool enable_display;
  extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
  extern int xe_guc_log_level;
  extern char *xe_param_force_probe;
drivers/gpu/drm/xe/xe_pci.c +0 -0
index ee61a03d1afd..2c86a8ab727a 100644
@@ -592,6 +592,40 @@ static void
 xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev)
         }
  }

drivers/gpu/drm/xe/xe_pci.c +10 -0
+static bool xe_pci_d3cold_capable(struct pci_dev *pdev)
+{
+       struct pci_dev *root_pdev;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return false;
+
+       /* D3Cold requires PME capability and _PR3 power resource */
+       if (!pci_pme_capable(root_pdev, PCI_D3cold) ||

!pci_pr3_present(root_pdev))
+               return false;

drivers/gpu/drm/xe/xe_pci.c +5 -0
+
+       return true;
+}
+
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, bool

enable)

drivers/gpu/drm/xe/xe_pci.c +18 -0
+{
+       struct xe_device *xe = pdev_to_xe_device(pdev);
+       struct pci_dev *root_pdev;
+
+       if (!xe->d3cold_capable)
+               return;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return;
+
+       if (enable)
+               pci_d3cold_enable(pdev);
+       else
+               pci_d3cold_disable(pdev);
+
+}
+

 static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
        const struct xe_device_desc *desc = (const void *)ent-

Quoted 1 line

driver_data;

drivers/gpu/drm/xe/xe_pci.c +0 -0
@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev,
 const struct pci_device_id *ent)
                 return err;
         }


+       xe->d3cold_capable = xe_pci_d3cold_capable(pdev); +
        xe_pm_runtime_init(xe);

        return 0;

drivers/gpu/drm/xe/xe_pci.c +1 -0
@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device
 *dev)
                 pci_set_power_state(pdev, PCI_D3cold);
         } else {
                 pci_set_power_state(pdev, PCI_D3hot);
+               xe_pci_d3cold_enable_disable(pdev, false);
         }


        return 0;

drivers/gpu/drm/xe/xe_pci.c +0 -0
@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device
 *dev)
                         return err;


                pci_set_master(pdev);
+       } else {
+               xe_pci_d3cold_enable_disable(pdev, true); }

        return xe_pm_runtime_resume(xe);

drivers/gpu/drm/xe/xe_pci.c +0 -0
@@ -774,6 +813,13 @@ static int xe_pci_runtime_idle(struct device
 *dev)
         struct pci_dev *pdev = to_pci_dev(dev);
         struct xe_device *xe = pdev_to_xe_device(pdev);


+       if (!xe->d3cold_capable) {
+               xe->d3cold_allowed = false;
+               return 0;
+       }

drivers/gpu/drm/xe/xe_pci.c +3 -0
+
+       xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+

        /*
         * TODO: d3cold should be allowed (true) if
         * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))

drivers/gpu/drm/xe/xe_pm.c +1 -0
index b7b57f10ba25..523d5f89ec89 100644
@@ -16,6 +16,7 @@
  #include "xe_ggtt.h"
  #include "xe_gt.h"
  #include "xe_irq.h"
+#include "xe_module.h"
  #include "xe_pcode.h"


 /**

drivers/gpu/drm/xe/xe_pm.c +0 -0
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
         return 0;
  }

drivers/gpu/drm/xe/xe_pm.c +13 -0
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe)
+{
+       struct ttm_resource_manager *man;
+       u64 vram_used;
+       size_t total_vram_used_mb = 0;
+       int i;
+
+       /* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+       for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+               man = ttm_manager_type(&xe->ttm, i);
+               if (man) {
+                       vram_used = ttm_resource_manager_usage(man);
+                       total_vram_used_mb +=

DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+               }
+       }

drivers/gpu/drm/xe/xe_pm.c +4 -0
+
+       return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+

 void xe_pm_runtime_init(struct xe_device *xe)
 {
        struct device *dev = xe->drm.dev;

drivers/gpu/drm/xe/xe_pm.h +1 -0
index 6a885585f653..33515d5c2cc2 100644
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
  void xe_pm_runtime_fini(struct xe_device *xe);
  int xe_pm_runtime_suspend(struct xe_device *xe);
  int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
  int xe_pm_runtime_get(struct xe_device *xe);
  int xe_pm_runtime_put(struct xe_device *xe);
  bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
#3 lore raw
Quoted 61 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the

Please use imperative language as per the Linux patch-submission guidelines.

"Add pci d3cold capable check support."

Typos:

Quoted 1 line

permissisble threshold for vram save/restore. If vram used

permissible

Quoted 1 line

is more then

more than

Quoted 1 line

the threshold value then disabllow

disallow

Quoted 45 lines

the d3cold
to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
 drivers/gpu/drm/xe/xe_device_types.h |  3 ++ drivers/gpu/drm/xe/xe_module.c       |  4 +++ drivers/gpu/drm/xe/xe_module.h       |  1 + drivers/gpu/drm/xe/xe_pci.c          | 46
++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h           |  1 +

 6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +0 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
                 bool hold_rpm;
         } mem_access;


+       /** d3cold_capable: Indicates if root port is d3cold capable */
+       bool d3cold_capable;
+
        /** @d3cold_allowed: Indicates if d3cold is a valid device state */
        bool d3cold_allowed;

drivers/gpu/drm/xe/xe_module.c +0 -0
index 6860586ce7f8..1dd515ed8a65 100644
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
  module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,
 0600);
  MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");

Quoted 8 lines
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c +2 -0
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold,

xe_vram_save_restore_threshold, uint, 0600); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram used threshold below which vram save/restore premissible (in MB)");

With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?

probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.

Quoted 170 lines

Thanks,
Thomas

Quoted 163 lines

+
 int xe_guc_log_level = 5; module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)");

drivers/gpu/drm/xe/xe_module.h +1 -0
index 7169907c3365..620d1b80cab6 100644
@@ -9,5 +9,6 @@
  extern bool enable_guc;
  extern bool enable_display;
  extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
  extern int xe_guc_log_level;
  extern char *xe_param_force_probe;
drivers/gpu/drm/xe/xe_pci.c +0 -0
index ee61a03d1afd..2c86a8ab727a 100644
@@ -592,6 +592,40 @@ static void
 xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev)
         }
  }

drivers/gpu/drm/xe/xe_pci.c +10 -0
+static bool xe_pci_d3cold_capable(struct pci_dev *pdev)
+{
+       struct pci_dev *root_pdev;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return false;
+
+       /* D3Cold requires PME capability and _PR3 power resource */
+       if (!pci_pme_capable(root_pdev, PCI_D3cold) ||

!pci_pr3_present(root_pdev))
+               return false;

drivers/gpu/drm/xe/xe_pci.c +5 -0
+
+       return true;
+}
+
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, bool

enable)

drivers/gpu/drm/xe/xe_pci.c +18 -0
+{
+       struct xe_device *xe = pdev_to_xe_device(pdev);
+       struct pci_dev *root_pdev;
+
+       if (!xe->d3cold_capable)
+               return;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return;
+
+       if (enable)
+               pci_d3cold_enable(pdev);
+       else
+               pci_d3cold_disable(pdev);
+
+}
+

 static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
        const struct xe_device_desc *desc = (const void *)ent-

Quoted 1 line

driver_data;

drivers/gpu/drm/xe/xe_pci.c +0 -0
@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev,
 const struct pci_device_id *ent)
                 return err;
         }


+       xe->d3cold_capable = xe_pci_d3cold_capable(pdev); +
        xe_pm_runtime_init(xe);

        return 0;

drivers/gpu/drm/xe/xe_pci.c +1 -0
@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device
 *dev)
                 pci_set_power_state(pdev, PCI_D3cold);
         } else {
                 pci_set_power_state(pdev, PCI_D3hot);
+               xe_pci_d3cold_enable_disable(pdev, false);
         }


        return 0;

drivers/gpu/drm/xe/xe_pci.c +0 -0
@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device
 *dev)
                         return err;


                pci_set_master(pdev);
+       } else {
+               xe_pci_d3cold_enable_disable(pdev, true); }

        return xe_pm_runtime_resume(xe);

drivers/gpu/drm/xe/xe_pci.c +0 -0
@@ -774,6 +813,13 @@ static int xe_pci_runtime_idle(struct device
 *dev)
         struct pci_dev *pdev = to_pci_dev(dev);
         struct xe_device *xe = pdev_to_xe_device(pdev);


+       if (!xe->d3cold_capable) {
+               xe->d3cold_allowed = false;
+               return 0;
+       }

drivers/gpu/drm/xe/xe_pci.c +3 -0
+
+       xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+

        /*
         * TODO: d3cold should be allowed (true) if
         * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))

drivers/gpu/drm/xe/xe_pm.c +1 -0
index b7b57f10ba25..523d5f89ec89 100644
@@ -16,6 +16,7 @@
  #include "xe_ggtt.h"
  #include "xe_gt.h"
  #include "xe_irq.h"
+#include "xe_module.h"
  #include "xe_pcode.h"


 /**

drivers/gpu/drm/xe/xe_pm.c +0 -0
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
         return 0;
  }

drivers/gpu/drm/xe/xe_pm.c +13 -0
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe)
+{
+       struct ttm_resource_manager *man;
+       u64 vram_used;
+       size_t total_vram_used_mb = 0;
+       int i;
+
+       /* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+       for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+               man = ttm_manager_type(&xe->ttm, i);
+               if (man) {
+                       vram_used = ttm_resource_manager_usage(man);
+                       total_vram_used_mb +=

DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+               }
+       }

drivers/gpu/drm/xe/xe_pm.c +4 -0
+
+       return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+

 void xe_pm_runtime_init(struct xe_device *xe)
 {
        struct device *dev = xe->drm.dev;

drivers/gpu/drm/xe/xe_pm.h +1 -0
index 6a885585f653..33515d5c2cc2 100644
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
  void xe_pm_runtime_fini(struct xe_device *xe);
  int xe_pm_runtime_suspend(struct xe_device *xe);
  int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
  int xe_pm_runtime_get(struct xe_device *xe);
  int xe_pm_runtime_put(struct xe_device *xe);
  bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
#4 lore raw
Quoted 77 lines

-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policy

Quoted 61 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the

Please use imperative language as per the Linux patch-submission guidelines.

"Add pci d3cold capable check support."

Typos:

Quoted 1 line

permissisble threshold for vram save/restore. If vram used

permissible

Quoted 1 line

is more then

more than

Quoted 1 line

the threshold value then disabllow

disallow

Quoted 45 lines

the d3cold
to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
 drivers/gpu/drm/xe/xe_device_types.h |  3 ++ drivers/gpu/drm/xe/xe_module.c       |  4 +++ drivers/gpu/drm/xe/xe_module.h       |  1 + drivers/gpu/drm/xe/xe_pci.c          | 46
++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h           |  1 +

 6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +3 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
                 bool hold_rpm;
         } mem_access;
 
+       /** d3cold_capable: Indicates if root port is d3cold capable
 */
+       bool d3cold_capable;
+
         /** @d3cold_allowed: Indicates if d3cold is a valid device

state */
        bool d3cold_allowed;

(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_module.c index 6860586ce7f8..1dd515ed8a65 100644

drivers/gpu/drm/xe/xe_module.c +0 -0
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
  module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,
 0600);
  MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");
 
Quoted 4 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 33

Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c +3 -0
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold,
 xe_vram_save_restore_threshold, uint, 0600);
+MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram

used

Quoted 7 lines
Quoted 4 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 34

Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0 -0
 threshold below which vram save/restore premissible (in MB)");

With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?

probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.

I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?
d3cold is state a for a pci device, so "vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under /sys/bus/pci/devices/$bdf
directory ?

Thanks ,
Anshuman Gupta.

Quoted 169 lines
Quoted 13 lines

Thanks,
Thomas

Quoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1 -0
+

 int xe_guc_log_level = 5; module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)"); diff --git a/drivers/gpu/drm/xe/xe_module.h

b/drivers/gpu/drm/xe/xe_module.h

Quoted 155 lines
Quoted 154 lines

index 7169907c3365..620d1b80cab6 100644

drivers/gpu/drm/xe/xe_module.h +1 -0
@@ -9,5 +9,6 @@
  extern bool enable_guc;
  extern bool enable_display;
  extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
  extern int xe_guc_log_level;
  extern char *xe_param_force_probe;
(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_pci.c index ee61a03d1afd..2c86a8ab727a 100644

drivers/gpu/drm/xe/xe_pci.c +33 -0
@@ -592,6 +592,40 @@ static void
 xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev)
         }
  }
 
+static bool xe_pci_d3cold_capable(struct pci_dev *pdev) {
+       struct pci_dev *root_pdev;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return false;
+
+       /* D3Cold requires PME capability and _PR3 power resource */
+       if (!pci_pme_capable(root_pdev, PCI_D3cold) ||
 !pci_pr3_present(root_pdev))
+               return false;
+
+       return true;
+}
+
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, bool
 enable)
+{
+       struct xe_device *xe = pdev_to_xe_device(pdev);
+       struct pci_dev *root_pdev;
+
+       if (!xe->d3cold_capable)
+               return;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return;
+
+       if (enable)
+               pci_d3cold_enable(pdev);
+       else
+               pci_d3cold_disable(pdev);
+
+}
+

 static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
        const struct xe_device_desc *desc = (const void *)ent-

Quoted 1 line

driver_data;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev,
 const struct pci_device_id *ent)
                 return err;
         }
 
+       xe->d3cold_capable = xe_pci_d3cold_capable(pdev);
+
         xe_pm_runtime_init(xe);
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +1 -0
@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device
 *dev)
                 pci_set_power_state(pdev, PCI_D3cold);
         } else {
                 pci_set_power_state(pdev, PCI_D3hot);
+               xe_pci_d3cold_enable_disable(pdev, false);
         }
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device
 *dev)
                         return err;
 
                 pci_set_master(pdev);
+       } else {
+               xe_pci_d3cold_enable_disable(pdev, true);
         }
 

        return xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@ static int xe_pci_runtime_idle(struct device
*dev)
        struct pci_dev *pdev = to_pci_dev(dev);
        struct xe_device *xe = pdev_to_xe_device(pdev);

+       if (!xe->d3cold_capable) {
+               xe->d3cold_allowed = false;
+               return 0;
+       }

drivers/gpu/drm/xe/xe_pci.c +3 -0
+
+       xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+

        /*
         * TODO: d3cold should be allowed (true) if
         * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))

drivers/gpu/drm/xe/xe_pm.c +19 -0
index b7b57f10ba25..523d5f89ec89 100644
@@ -16,6 +16,7 @@
  #include "xe_ggtt.h"
  #include "xe_gt.h"
  #include "xe_irq.h"
+#include "xe_module.h"
  #include "xe_pcode.h"
 
  /**
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
         return 0;
  }
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe) {
+       struct ttm_resource_manager *man;
+       u64 vram_used;
+       size_t total_vram_used_mb = 0;
+       int i;
+
+       /* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+       for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+               man = ttm_manager_type(&xe->ttm, i);
+               if (man) {
+                       vram_used = ttm_resource_manager_usage(man);
+                       total_vram_used_mb +=
 DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+               }
+       }
+
+       return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+
  void xe_pm_runtime_init(struct xe_device *xe)
  {

        struct device *dev = xe->drm.dev; diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 6a885585f653..33515d5c2cc2 100644

drivers/gpu/drm/xe/xe_pm.h +1 -0
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
  void xe_pm_runtime_fini(struct xe_device *xe);
  int xe_pm_runtime_suspend(struct xe_device *xe);
  int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
  int xe_pm_runtime_get(struct xe_device *xe);
  int xe_pm_runtime_put(struct xe_device *xe);
  bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
#5 lore raw

Hi!

Quoted 76 lines · Anshuman
Quoted 76 lines

-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policy

Quoted 60 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the

Please use imperative language as per the Linux patch-submission guidelines.

"Add pci d3cold capable check support."

Typos:

Quoted 1 line

permissisble threshold for vram save/restore. If vram used

permissible

Quoted 1 line

is more then

more than

Quoted 1 line

the threshold value then disabllow

disallow

Quoted 45 lines

the d3cold
to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
  drivers/gpu/drm/xe/xe_device_types.h |  3 ++
  drivers/gpu/drm/xe/xe_module.c       |  4 +++
  drivers/gpu/drm/xe/xe_module.h       |  1 +
  drivers/gpu/drm/xe/xe_pci.c          | 46
++++++++++++++++++++++++++++
  drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++
  drivers/gpu/drm/xe/xe_pm.h           |  1 +
  6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +3 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
                 bool hold_rpm;
         } mem_access;
 
+       /** d3cold_capable: Indicates if root port is d3cold capable
 */
+       bool d3cold_capable;
+
         /** @d3cold_allowed: Indicates if d3cold is a valid device

state */
        bool d3cold_allowed;

(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_module.c index 6860586ce7f8..1dd515ed8a65 100644

drivers/gpu/drm/xe/xe_module.c +0 -0
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
  module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,
 0600);
  MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");
 
Quoted 4 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 33

Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c +3 -0
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold,
 xe_vram_save_restore_threshold, uint, 0600);
+MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram

used

Quoted 1 line
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0 -0
 threshold below which vram save/restore premissible (in MB)");
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?

probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.

I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?

I was thinking about an example where you have, lets say 4 DG2s that are bound to the same xe driver instance. You want them to have different limits.

Quoted 4 lines

d3cold is state a for a pci device, so "vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under /sys/bus/pci/devices/$bdf
directory ?

I'm not fully familiar with the sysfs layout used on xe, perhaps Rodrigo can fill in, but perhaps it makes sense to have them in the same directory as the gt0..gtX subdirectories? Looks like it's the same path as you write above.

/Thomas

Quoted 169 lines

Thanks ,
Anshuman Gupta.

Quoted 166 lines
Quoted 11 lines

Thanks,
Thomas

Quoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1 -0
+

 int xe_guc_log_level = 5; module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level
(0=disable, 1..5=enable with verbosity min..max)"); diff --git a/drivers/gpu/drm/xe/xe_module.h

b/drivers/gpu/drm/xe/xe_module.h

Quoted 154 lines
Quoted 154 lines

index 7169907c3365..620d1b80cab6 100644

drivers/gpu/drm/xe/xe_module.h +1 -0
@@ -9,5 +9,6 @@
  extern bool enable_guc;
  extern bool enable_display;
  extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
  extern int xe_guc_log_level;
  extern char *xe_param_force_probe;
(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_pci.c index ee61a03d1afd..2c86a8ab727a 100644

drivers/gpu/drm/xe/xe_pci.c +33 -0
@@ -592,6 +592,40 @@ static void
 xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev)
         }
  }
 
+static bool xe_pci_d3cold_capable(struct pci_dev *pdev) {
+       struct pci_dev *root_pdev;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return false;
+
+       /* D3Cold requires PME capability and _PR3 power resource */
+       if (!pci_pme_capable(root_pdev, PCI_D3cold) ||
 !pci_pr3_present(root_pdev))
+               return false;
+
+       return true;
+}
+
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, bool
 enable)
+{
+       struct xe_device *xe = pdev_to_xe_device(pdev);
+       struct pci_dev *root_pdev;
+
+       if (!xe->d3cold_capable)
+               return;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return;
+
+       if (enable)
+               pci_d3cold_enable(pdev);
+       else
+               pci_d3cold_disable(pdev);
+
+}
+
  static int xe_pci_probe(struct pci_dev *pdev, const struct

pci_device_id *ent)
 {
        const struct xe_device_desc *desc = (const void *)ent-

Quoted 1 line

driver_data;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev,
 const struct pci_device_id *ent)
                 return err;
         }
 
+       xe->d3cold_capable = xe_pci_d3cold_capable(pdev);
+
         xe_pm_runtime_init(xe);
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +1 -0
@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device
 *dev)
                 pci_set_power_state(pdev, PCI_D3cold);
         } else {
                 pci_set_power_state(pdev, PCI_D3hot);
+               xe_pci_d3cold_enable_disable(pdev, false);
         }
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device
 *dev)
                         return err;
 
                 pci_set_master(pdev);
+       } else {
+               xe_pci_d3cold_enable_disable(pdev, true);
         }
 

        return xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@ static
int xe_pci_runtime_idle(struct device
*dev)
        struct pci_dev *pdev = to_pci_dev(dev);
        struct xe_device *xe = pdev_to_xe_device(pdev);

+       if (!xe->d3cold_capable) {
+               xe->d3cold_allowed = false;
+               return 0;
+       }

drivers/gpu/drm/xe/xe_pci.c +3 -0
+
+       xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+
         /*
          * TODO: d3cold should be allowed (true) if
          * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))
drivers/gpu/drm/xe/xe_pm.c +19 -0
index b7b57f10ba25..523d5f89ec89 100644
@@ -16,6 +16,7 @@
  #include "xe_ggtt.h"
  #include "xe_gt.h"
  #include "xe_irq.h"
+#include "xe_module.h"
  #include "xe_pcode.h"
 
  /**
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
         return 0;
  }
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe) {
+       struct ttm_resource_manager *man;
+       u64 vram_used;
+       size_t total_vram_used_mb = 0;
+       int i;
+
+       /* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+       for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+               man = ttm_manager_type(&xe->ttm, i);
+               if (man) {
+                       vram_used = ttm_resource_manager_usage(man);
+                       total_vram_used_mb +=
 DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+               }
+       }
+
+       return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+
  void xe_pm_runtime_init(struct xe_device *xe)
  {
         struct device *dev = xe->drm.dev; diff --git

a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 6a885585f653..33515d5c2cc2 100644

drivers/gpu/drm/xe/xe_pm.h +1 -0
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
  void xe_pm_runtime_fini(struct xe_device *xe);
  int xe_pm_runtime_suspend(struct xe_device *xe);
  int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
  int xe_pm_runtime_get(struct xe_device *xe);
  int xe_pm_runtime_put(struct xe_device *xe);
  bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
#6 lore raw
Quoted 106 lines

-----Original Message-----
From: Thomas Hellström <[email protected]>
Sent: Monday, May 15, 2023 3:08 PM
To: Gupta, Anshuman <[email protected]>; Rodrigo Vivi <[email protected]>
Cc: [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policy

Hi!

Quoted 61 lines · Anshuman
Quoted 61 lines

-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policy

Quoted 51 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the

Please use imperative language as per the Linux patch-submission guidelines.

"Add pci d3cold capable check support."

Typos:

Quoted 1 line

permissisble threshold for vram save/restore. If vram used

permissible

Quoted 1 line

is more then

more than

Quoted 1 line

the threshold value then disabllow

disallow

Quoted 36 lines

the d3cold
to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
  drivers/gpu/drm/xe/xe_device_types.h |  3 ++
  drivers/gpu/drm/xe/xe_module.c       |  4 +++
  drivers/gpu/drm/xe/xe_module.h       |  1 +
  drivers/gpu/drm/xe/xe_pci.c          | 46
++++++++++++++++++++++++++++
  drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++
  drivers/gpu/drm/xe/xe_pm.h           |  1 +
  6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +4 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
                 bool hold_rpm;
         } mem_access;
 
+       /** d3cold_capable: Indicates if root port is d3cold
+capable
 */
+       bool d3cold_capable;
+
         /** @d3cold_allowed: Indicates if d3cold is a valid device

state */
        bool d3cold_allowed;

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_module.c index

6860586ce7f8..1dd515ed8a65

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 26

Quoted 5 lines

100644

drivers/gpu/drm/xe/xe_module.c +0 -0
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
  module_param_named(vram_bar_size, xe_force_vram_bar_size,

uint,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 28

Quoted 2 lines
drivers/gpu/drm/xe/xe_module.c +0 -0
 0600);
  MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in

MiB)");

Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 32

Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c +3 -0
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold,
 xe_vram_save_restore_threshold, uint, 0600);
+MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram

used

Quoted 1 line
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0 -0
 threshold below which vram save/restore premissible (in MB)");
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?

probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.

I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?

I was thinking about an example where you have, lets say 4 DG2s that are bound to the same xe driver instance. You want them to have different limits.

Quoted 8 lines
Quoted 4 lines

d3cold is state a for a pci device, so "vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under /sys/bus/pci/devices/$bdf directory ?

I'm not fully familiar with the sysfs layout used on xe, perhaps Rodrigo can fill in, but perhaps it makes sense to have them in the same directory as the gt0..gtX subdirectories? Looks like it's the same path as you write above.

Thanks thomas for review comment, I will fix this in next revision. Br,
Anshuman Gupta.

Quoted 177 lines

/Thomas

Quoted 173 lines

Thanks ,
Anshuman Gupta.

Quoted 170 lines
Quoted 11 lines

Thanks,
Thomas

Quoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1 -0
+
  int xe_guc_log_level = 5;

 module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level
(0=disable, 1..5=enable with verbosity min..max)"); diff --git a/drivers/gpu/drm/xe/xe_module.h

b/drivers/gpu/drm/xe/xe_module.h

Quoted 158 lines
Quoted 158 lines

index 7169907c3365..620d1b80cab6 100644

drivers/gpu/drm/xe/xe_module.h +1 -0
@@ -9,5 +9,6 @@
  extern bool enable_guc;
  extern bool enable_display;
  extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
  extern int xe_guc_log_level;
  extern char *xe_param_force_probe; diff --git

a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index ee61a03d1afd..2c86a8ab727a
100644

drivers/gpu/drm/xe/xe_pci.c +35 -0
@@ -592,6 +592,40 @@ static void
 xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev)
         }
  }
 
+static bool xe_pci_d3cold_capable(struct pci_dev *pdev) {
+       struct pci_dev *root_pdev;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return false;
+
+       /* D3Cold requires PME capability and _PR3 power resource
+*/
+       if (!pci_pme_capable(root_pdev, PCI_D3cold) ||
 !pci_pr3_present(root_pdev))
+               return false;
+
+       return true;
+}
+
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev,
+bool
 enable)
+{
+       struct xe_device *xe = pdev_to_xe_device(pdev);
+       struct pci_dev *root_pdev;
+
+       if (!xe->d3cold_capable)
+               return;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return;
+
+       if (enable)
+               pci_d3cold_enable(pdev);
+       else
+               pci_d3cold_disable(pdev);
+
+}
+
  static int xe_pci_probe(struct pci_dev *pdev, const struct

pci_device_id *ent)
 {
        const struct xe_device_desc *desc = (const void *)ent-

Quoted 1 line

driver_data;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev,
 const struct pci_device_id *ent)
                 return err;
         }
 
+       xe->d3cold_capable = xe_pci_d3cold_capable(pdev);
+
         xe_pm_runtime_init(xe);
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +1 -0
@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device
 *dev)
                 pci_set_power_state(pdev, PCI_D3cold);
         } else {
                 pci_set_power_state(pdev, PCI_D3hot);
+               xe_pci_d3cold_enable_disable(pdev, false);
         }
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device
 *dev)
                         return err;
 
                 pci_set_master(pdev);
+       } else {
+               xe_pci_d3cold_enable_disable(pdev, true);
         }
 

        return xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@
static int xe_pci_runtime_idle(struct device
*dev)
        struct pci_dev *pdev = to_pci_dev(dev); struct xe_device *xe = pdev_to_xe_device(pdev);

+       if (!xe->d3cold_capable) {
+               xe->d3cold_allowed = false;
+               return 0;
+       }

drivers/gpu/drm/xe/xe_pci.c +3 -0
+
+       xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+
         /*
          * TODO: d3cold should be allowed (true) if
          * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))
(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_pm.c index b7b57f10ba25..523d5f89ec89 100644

drivers/gpu/drm/xe/xe_pm.c +20 -0
@@ -16,6 +16,7 @@
  #include "xe_ggtt.h"
  #include "xe_gt.h"
  #include "xe_irq.h"
+#include "xe_module.h"
  #include "xe_pcode.h"
 
  /**
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
         return 0;
  }
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe) {
+       struct ttm_resource_manager *man;
+       u64 vram_used;
+       size_t total_vram_used_mb = 0;
+       int i;
+
+       /* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+       for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+               man = ttm_manager_type(&xe->ttm, i);
+               if (man) {
+                       vram_used =
+ttm_resource_manager_usage(man);
+                       total_vram_used_mb +=
 DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+               }
+       }
+
+       return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+
  void xe_pm_runtime_init(struct xe_device *xe)
  {

        struct device *dev = xe->drm.dev; diff --git
a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 6a885585f653..33515d5c2cc2 100644

drivers/gpu/drm/xe/xe_pm.h +1 -0
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
  void xe_pm_runtime_fini(struct xe_device *xe);
  int xe_pm_runtime_suspend(struct xe_device *xe);
  int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
  int xe_pm_runtime_get(struct xe_device *xe);
  int xe_pm_runtime_put(struct xe_device *xe);
  bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
#7 lore raw
In reply to Thomas Hellström
Quoted 106 lines

-----Original Message-----
From: Thomas Hellström <[email protected]>
Sent: Monday, May 15, 2023 3:08 PM
To: Gupta, Anshuman <[email protected]>; Rodrigo Vivi <[email protected]>
Cc: [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policy

Hi!

Quoted 61 lines · Anshuman
Quoted 61 lines

-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policy

Quoted 51 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta

Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the

Please use imperative language as per the Linux patch-submission guidelines.

"Add pci d3cold capable check support."

Typos:

Quoted 1 line

permissisble threshold for vram save/restore. If vram used

permissible

Quoted 1 line

is more then

more than

Quoted 1 line

the threshold value then disabllow

disallow

Quoted 36 lines

the d3cold
to avoid the vram save/restore latency.

TODO: At resume vram save/restore only in case card really lost power.

---
  drivers/gpu/drm/xe/xe_device_types.h |  3 ++
  drivers/gpu/drm/xe/xe_module.c       |  4 +++
  drivers/gpu/drm/xe/xe_module.h       |  1 +
  drivers/gpu/drm/xe/xe_pci.c          | 46
++++++++++++++++++++++++++++
  drivers/gpu/drm/xe/xe_pm.c           | 20 ++++++++++++
  drivers/gpu/drm/xe/xe_pm.h           |  1 +
  6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +4 -0
index 1cb404e48aaa..1137f544575d 100644
@@ -263,6 +263,9 @@ struct xe_device {
                 bool hold_rpm;
         } mem_access;
 
+       /** d3cold_capable: Indicates if root port is d3cold
+capable
 */
+       bool d3cold_capable;
+
         /** @d3cold_allowed: Indicates if d3cold is a valid device

state */
        bool d3cold_allowed;

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_module.c index

6860586ce7f8..1dd515ed8a65

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 26

Quoted 5 lines

100644

drivers/gpu/drm/xe/xe_module.c +0 -0
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;
  module_param_named(vram_bar_size, xe_force_vram_bar_size,

uint,

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 28

Quoted 2 lines
drivers/gpu/drm/xe/xe_module.c +0 -0
 0600);
  MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in

MiB)");

Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 32

Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c +3 -0
+u32 xe_vram_save_restore_threshold = 300;
+module_param_named(vram_save_restore_threshold,
 xe_vram_save_restore_threshold, uint, 0600);
+MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram

used

Quoted 1 line
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0 -0
 threshold below which vram save/restore premissible (in MB)");
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?

probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.

I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?

I was thinking about an example where you have, lets say 4 DG2s that are bound to the same xe driver instance. You want them to have different limits.

Quoted 8 lines
Quoted 4 lines

d3cold is state a for a pci device, so "vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under /sys/bus/pci/devices/$bdf directory ?

I'm not fully familiar with the sysfs layout used on xe, perhaps Rodrigo can fill in, but perhaps it makes sense to have them in the same directory as the gt0..gtX subdirectories? Looks like it's the same path as you write above.

Hi Thomas,
I have added a new sysfs called vram_d3cold_threshold and floated the patch
https://patchwork.freedesktop.org/patch/541520/?series=118035&rev=2
But am skeptical on the user space consumer for this sysfs,
I can write a IGT to use this sysfs but will that suffice the user space consumer requirement ? Do you have any idea how other GPU handles it or they do always evict entire vram ?

Thanks,
Anshuman Gupta.
Br,
Anshuman Gupta.

Quoted 177 lines

/Thomas

Quoted 173 lines

Thanks ,
Anshuman Gupta.

Quoted 170 lines
Quoted 11 lines

Thanks,
Thomas

Quoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1 -0
+
  int xe_guc_log_level = 5;

 module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level
(0=disable, 1..5=enable with verbosity min..max)"); diff --git a/drivers/gpu/drm/xe/xe_module.h

b/drivers/gpu/drm/xe/xe_module.h

Quoted 158 lines
Quoted 158 lines

index 7169907c3365..620d1b80cab6 100644

drivers/gpu/drm/xe/xe_module.h +1 -0
@@ -9,5 +9,6 @@
  extern bool enable_guc;
  extern bool enable_display;
  extern u32 xe_force_vram_bar_size;
+extern u32 xe_vram_save_restore_threshold;
  extern int xe_guc_log_level;
  extern char *xe_param_force_probe; diff --git

a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index ee61a03d1afd..2c86a8ab727a
100644

drivers/gpu/drm/xe/xe_pci.c +35 -0
@@ -592,6 +592,40 @@ static void
 xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev)
         }
  }
 
+static bool xe_pci_d3cold_capable(struct pci_dev *pdev) {
+       struct pci_dev *root_pdev;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return false;
+
+       /* D3Cold requires PME capability and _PR3 power resource
+*/
+       if (!pci_pme_capable(root_pdev, PCI_D3cold) ||
 !pci_pr3_present(root_pdev))
+               return false;
+
+       return true;
+}
+
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev,
+bool
 enable)
+{
+       struct xe_device *xe = pdev_to_xe_device(pdev);
+       struct pci_dev *root_pdev;
+
+       if (!xe->d3cold_capable)
+               return;
+
+       root_pdev = pcie_find_root_port(pdev);
+       if (!root_pdev)
+               return;
+
+       if (enable)
+               pci_d3cold_enable(pdev);
+       else
+               pci_d3cold_disable(pdev);
+
+}
+
  static int xe_pci_probe(struct pci_dev *pdev, const struct

pci_device_id *ent)
 {
        const struct xe_device_desc *desc = (const void *)ent-

Quoted 1 line

driver_data;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev,
 const struct pci_device_id *ent)
                 return err;
         }
 
+       xe->d3cold_capable = xe_pci_d3cold_capable(pdev);
+
         xe_pm_runtime_init(xe);
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +1 -0
@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device
 *dev)
                 pci_set_power_state(pdev, PCI_D3cold);
         } else {
                 pci_set_power_state(pdev, PCI_D3hot);
+               xe_pci_d3cold_enable_disable(pdev, false);
         }
 

        return 0;

drivers/gpu/drm/xe/xe_pci.c +2 -0
@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device
 *dev)
                         return err;
 
                 pci_set_master(pdev);
+       } else {
+               xe_pci_d3cold_enable_disable(pdev, true);
         }
 

        return xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@
static int xe_pci_runtime_idle(struct device
*dev)
        struct pci_dev *pdev = to_pci_dev(dev); struct xe_device *xe = pdev_to_xe_device(pdev);

+       if (!xe->d3cold_capable) {
+               xe->d3cold_allowed = false;
+               return 0;
+       }

drivers/gpu/drm/xe/xe_pci.c +3 -0
+
+       xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+
         /*
          * TODO: d3cold should be allowed (true) if
          * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))
(quoted diff) +0 -0

b/drivers/gpu/drm/xe/xe_pm.c index b7b57f10ba25..523d5f89ec89 100644

drivers/gpu/drm/xe/xe_pm.c +20 -0
@@ -16,6 +16,7 @@
  #include "xe_ggtt.h"
  #include "xe_gt.h"
  #include "xe_irq.h"
+#include "xe_module.h"
  #include "xe_pcode.h"
 
  /**
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)
         return 0;
  }
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe) {
+       struct ttm_resource_manager *man;
+       u64 vram_used;
+       size_t total_vram_used_mb = 0;
+       int i;
+
+       /* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+       for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+               man = ttm_manager_type(&xe->ttm, i);
+               if (man) {
+                       vram_used =
+ttm_resource_manager_usage(man);
+                       total_vram_used_mb +=
 DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+               }
+       }
+
+       return total_vram_used_mb < xe_vram_save_restore_threshold;
+}
+
  void xe_pm_runtime_init(struct xe_device *xe)
  {

        struct device *dev = xe->drm.dev; diff --git
a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 6a885585f653..33515d5c2cc2 100644

drivers/gpu/drm/xe/xe_pm.h +1 -0
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);
  void xe_pm_runtime_fini(struct xe_device *xe);
  int xe_pm_runtime_suspend(struct xe_device *xe);
  int xe_pm_runtime_resume(struct xe_device *xe);
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
  int xe_pm_runtime_get(struct xe_device *xe);
  int xe_pm_runtime_put(struct xe_device *xe);
  bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);