drm/xe/pm: d3cold policy
Subsystems (2)
RFC 1/1 drm/xe/pm: d3cold policy
Adding pci d3cold capable check support.
Adding a vram_save_restore_threshold modparam, that is the permissisble threshold for vram save/restore. If vram used is more then the threshold value then disabllow the d3cold to avoid the vram save/restore latency.
TODO: At resume vram save/restore only in case card really lost power.
--- drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 + 6 files changed, 75 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +3 -0
@@ -263,6 +263,9 @@ struct xe_device {bool hold_rpm; } mem_access; +/** d3cold_capable: Indicates if root port is d3cold capable */ +bool d3cold_capable; +/** @d3cold_allowed: Indicates if d3cold is a valid device state */ bool d3cold_allowed;
drivers/gpu/drm/xe/xe_module.c +4 -0
@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size;module_param_named (vram_bar_size, xe_force_vram_bar_size, uint,0600 );MODULE_PARM_DESC (vram_bar_size,"Set the vram bar size(in MiB)" ); +u32 xe_vram_save_restore_threshold =300 ; +module_param_named (vram_save_restore_threshold, xe_vram_save_restore_threshold, uint,0600 ); +MODULE_PARM_DESC (vram_save_restore_threshold,"Set the vram used threshold below which vram save/restore premissible (in MB)" ); +int xe_guc_log_level =5 ;module_param_named (guc_log_level, xe_guc_log_level, int,0600 );MODULE_PARM_DESC (guc_log_level,"GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)" );
drivers/gpu/drm/xe/xe_module.h +1 -0
@@ -9,5 +9,6 @@extern bool enable_guc;extern bool enable_display;extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold;extern int xe_guc_log_level;extern char *xe_param_force_probe;
drivers/gpu/drm/xe/xe_pci.c +46 -0
76 lines not on this page · raw message
drivers/gpu/drm/xe/xe_pm.c +20 -0
@@ -16,6 +16,7 @@#include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h" /**
@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe)return 0 ; } +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) +{ +struct ttm_resource_manager *man; +u64 vram_used; +size_t total_vram_used_mb =0 ; +int i; + +/* TODO: Extend the logic to beyond XE_PL_VRAM1 */ +for (i =XE_PL_VRAM0 ; i <=XE_PL_VRAM1 ; ++i) { + man =ttm_manager_type (&xe->ttm, i); +if (man) { + vram_used =ttm_resource_manager_usage (man); + total_vram_used_mb +=DIV_ROUND_UP_ULL (vram_used,1024 *1024 ); + } + } + +return total_vram_used_mb < xe_vram_save_restore_threshold; +} +void xe_pm_runtime_init (struct xe_device *xe) {struct device *dev = xe->drm.dev;
drivers/gpu/drm/xe/xe_pm.h +1 -0
@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe);void xe_pm_runtime_fini (struct xe_device *xe);int xe_pm_runtime_suspend (struct xe_device *xe);int xe_pm_runtime_resume (struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe);int xe_pm_runtime_get (struct xe_device *xe);int xe_pm_runtime_put (struct xe_device *xe);bool xe_pm_runtime_resume_if_suspended (struct xe_device *xe);
Signature · 1 line
2.38.0
Patchwork
CI[Intel-xe] ✗ CI.Patch_applied: failure for drm/xe/pm: d3cold policy
== Series Details ==
== Series Details ==
Series: drm/xe/pm: d3cold policy
URL : https://patchwork.freedesktop.org/series/117680/
State : failure
== Summary ==
=== Applying kernel patches on branch 'drm-xe-next' with base: === Base commit: a9affcc66 fixup! drm/i915/display: Remaining changes to make xe compile === git am output follows ===
error: patch failed: drivers/gpu/drm/xe/xe_pci.c:592
error: drivers/gpu/drm/xe/xe_pci.c: patch does not apply
hint: Use 'git am --show-current-patch' to see the failed patch
Applying: drm/xe/pm: d3cold policy
Patch failed at 0001 drm/xe/pm: d3cold policy
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".
Thomas Hellström
maintainerQuoted 2 lines · Anshuman Gupta
Adding pci d3cold capable check support.
Adding avram_save_restore_thresholdmodparam, that is the
Please use imperative language as per the Linux patch-submission guidelines.
"Add pci d3cold capable check support."
Typos:
Quoted 1 line
permissisble threshold for vram save/restore. If vram used
permissible
Quoted 1 line
is more then
more than
Quoted 1 line
the threshold value then disabllow
disallow
Quoted 47 lines
the d3cold
to avoid the vram save/restore latency.TODO: At resume vram save/restore only in case card really lost power.
Cc Rodrigo Vivi <[email protected]>Signed-off-by Anshuman Gupta <[email protected]>---
drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46
++++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 +6 files changed, 75 insertions(+)drivers/gpu/drm/xe/xe_device_types.h +0
-0@@ -263,6 +263,9 @@ struct xe_device { bool hold_rpm; } mem_access;
+ /**d3cold_capable: Indicates if root port is d3cold capable */
+ boold3cold_capable;
+
/** @d3cold_allowed: Indicates if d3cold is a valid device state */
boold3cold_allowed;drivers/gpu/drm/xe/xe_module.c +0
-0@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size; module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,0600 ); MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");drivers/gpu/drm/xe/xe_module.c +2
-0+u32 xe_vram_save_restore_threshold =300 ; +module_param_named(vram_save_restore_threshold,
Quoted 3 lines
xe_vram_save_restore_threshold, uint, 0600); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram used threshold below which vram save/restore premissible (in MB)");
With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?
Thanks,
Thomas
Quoted 163 lines
+
intxe_guc_log_level= 5;module_param_named(guc_log_level, xe_guc_log_level, int, 0600);MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)");drivers/gpu/drm/xe/xe_module.h +1
-0@@ -9,5 +9,6 @@ extern bool enable_guc; extern bool enable_display; extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold; extern int xe_guc_log_level; extern char *xe_param_force_probe;drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -592,6 +592,40 @@ static void xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev) } }drivers/gpu/drm/xe/xe_pci.c +10
-0+static bool xe_pci_d3cold_capable (struct pci_dev *pdev) +{ + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability and _PR3 power resource */ + if (!pci_pme_capable(root_pdev, PCI_D3cold) ||!pci_pr3_present(root_pdev))
+ return false;drivers/gpu/drm/xe/xe_pci.c +5
-0+ + return true; +} + +static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, boolenable)
drivers/gpu/drm/xe/xe_pci.c +18
-0+{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold_capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + if (enable) + pci_d3cold_enable(pdev); + else + pci_d3cold_disable(pdev); + +} +static int
xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
const structxe_device_desc*desc = (const void *)ent-Quoted 1 line
driver_data;drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; }
+ xe->d3cold_capable =xe_pci_d3cold_capable(pdev); +
xe_pm_runtime_init(xe);
return 0;drivers/gpu/drm/xe/xe_pci.c +1
-0@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device *dev) pci_set_power_state(pdev, PCI_D3cold); } else { pci_set_power_state(pdev, PCI_D3hot); + xe_pci_d3cold_enable_disable(pdev, false); }
return 0;drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device *dev) return err;
pci_set_master(pdev);
+ } else {
+ xe_pci_d3cold_enable_disable(pdev, true); }
returnxe_pm_runtime_resume(xe);drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -774,6 +813,13 @@ static int xe_pci_runtime_idle(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); struct xe_device *xe = pdev_to_xe_device(pdev);
+ if (!xe->d3cold_capable) {
+ xe->d3cold_allowed = false;
+ return 0;
+ }drivers/gpu/drm/xe/xe_pci.c +3
-0+ + xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe); +/*
* TODO: d3cold should be allowed (true) if
* (IS_DGFX(xe)&& !xe_device_mem_access_ongoing(xe))drivers/gpu/drm/xe/xe_pm.c +1
-0@@ -16,6 +16,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h"
/**drivers/gpu/drm/xe/xe_pm.c +0
-0@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe) return 0; }drivers/gpu/drm/xe/xe_pm.c +13
-0+bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) +{ + struct ttm_resource_manager *man; + u64 vram_used; + size_t total_vram_used_mb = 0; + int i; + + /* TODO: Extend the logic to beyond XE_PL_VRAM1 */ + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = ttm_resource_manager_usage(man); + total_vram_used_mb +=
DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+ }
+ }drivers/gpu/drm/xe/xe_pm.c +4
-0+ + return total_vram_used_mb < xe_vram_save_restore_threshold; +} +void
xe_pm_runtime_init(struct xe_device *xe)
{
struct device *dev =xe->drm.dev;drivers/gpu/drm/xe/xe_pm.h +1
-0@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe); void xe_pm_runtime_fini(struct xe_device *xe); int xe_pm_runtime_suspend(struct xe_device *xe); int xe_pm_runtime_resume(struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe); int xe_pm_runtime_get(struct xe_device *xe); int xe_pm_runtime_put(struct xe_device *xe); bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
Quoted 61 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta
Adding pci d3cold capable check support.
Adding avram_save_restore_thresholdmodparam, that is thePlease use imperative language as per the Linux patch-submission guidelines.
"Add pci d3cold capable check support."
Typos:
Quoted 1 line
permissisble threshold for vram save/restore. If vram used
permissible
Quoted 1 line
is more then
more than
Quoted 1 line
the threshold value then disabllow
disallow
Quoted 45 lines
the d3cold
to avoid the vram save/restore latency.TODO: At resume vram save/restore only in case card really lost power.
Cc Rodrigo Vivi <[email protected]>Signed-off-by Anshuman Gupta <[email protected]>---
drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46
++++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 +6 files changed, 75 insertions(+)drivers/gpu/drm/xe/xe_device_types.h +0
-0@@ -263,6 +263,9 @@ struct xe_device { bool hold_rpm; } mem_access;
+ /**d3cold_capable: Indicates if root port is d3cold capable */
+ boold3cold_capable;
+
/** @d3cold_allowed: Indicates if d3cold is a valid device state */
boold3cold_allowed;drivers/gpu/drm/xe/xe_module.c +0
-0@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size; module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,0600 ); MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");
Quoted 8 lines
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c +2
-0+u32 xe_vram_save_restore_threshold =300 ; +module_param_named(vram_save_restore_threshold,
xe_vram_save_restore_threshold, uint, 0600); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vram used threshold below which vram save/restore premissible (in MB)");With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?
probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.
Quoted 170 lines
Thanks,
ThomasQuoted 163 lines
+
intxe_guc_log_level= 5;module_param_named(guc_log_level, xe_guc_log_level, int, 0600);MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)");drivers/gpu/drm/xe/xe_module.h +1
-0@@ -9,5 +9,6 @@ extern bool enable_guc; extern bool enable_display; extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold; extern int xe_guc_log_level; extern char *xe_param_force_probe;drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -592,6 +592,40 @@ static void xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev) } }drivers/gpu/drm/xe/xe_pci.c +10
-0+static bool xe_pci_d3cold_capable (struct pci_dev *pdev) +{ + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability and _PR3 power resource */ + if (!pci_pme_capable(root_pdev, PCI_D3cold) ||!pci_pr3_present(root_pdev))
+ return false;drivers/gpu/drm/xe/xe_pci.c +5
-0+ + return true; +} + +static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, boolenable)
drivers/gpu/drm/xe/xe_pci.c +18
-0+{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold_capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + if (enable) + pci_d3cold_enable(pdev); + else + pci_d3cold_disable(pdev); + +} +static int
xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
const structxe_device_desc*desc = (const void *)ent-Quoted 1 line
driver_data;drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; }
+ xe->d3cold_capable =xe_pci_d3cold_capable(pdev); +
xe_pm_runtime_init(xe);
return 0;drivers/gpu/drm/xe/xe_pci.c +1
-0@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device *dev) pci_set_power_state(pdev, PCI_D3cold); } else { pci_set_power_state(pdev, PCI_D3hot); + xe_pci_d3cold_enable_disable(pdev, false); }
return 0;drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device *dev) return err;
pci_set_master(pdev);
+ } else {
+ xe_pci_d3cold_enable_disable(pdev, true); }
returnxe_pm_runtime_resume(xe);drivers/gpu/drm/xe/xe_pci.c +0
-0@@ -774,6 +813,13 @@ static int xe_pci_runtime_idle(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); struct xe_device *xe = pdev_to_xe_device(pdev);
+ if (!xe->d3cold_capable) {
+ xe->d3cold_allowed = false;
+ return 0;
+ }drivers/gpu/drm/xe/xe_pci.c +3
-0+ + xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe); +/*
* TODO: d3cold should be allowed (true) if
* (IS_DGFX(xe)&& !xe_device_mem_access_ongoing(xe))drivers/gpu/drm/xe/xe_pm.c +1
-0@@ -16,6 +16,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h"
/**drivers/gpu/drm/xe/xe_pm.c +0
-0@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe) return 0; }drivers/gpu/drm/xe/xe_pm.c +13
-0+bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) +{ + struct ttm_resource_manager *man; + u64 vram_used; + size_t total_vram_used_mb = 0; + int i; + + /* TODO: Extend the logic to beyond XE_PL_VRAM1 */ + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = ttm_resource_manager_usage(man); + total_vram_used_mb +=
DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+ }
+ }drivers/gpu/drm/xe/xe_pm.c +4
-0+ + return total_vram_used_mb < xe_vram_save_restore_threshold; +} +void
xe_pm_runtime_init(struct xe_device *xe)
{
struct device *dev =xe->drm.dev;drivers/gpu/drm/xe/xe_pm.h +1
-0@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe); void xe_pm_runtime_fini(struct xe_device *xe); int xe_pm_runtime_suspend(struct xe_device *xe); int xe_pm_runtime_resume(struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe); int xe_pm_runtime_get(struct xe_device *xe); int xe_pm_runtime_put(struct xe_device *xe); bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
Quoted 77 lines
-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policyQuoted 61 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta
Adding pci d3cold capable check support.
Adding avram_save_restore_thresholdmodparam, that is thePlease use imperative language as per the Linux patch-submission guidelines.
"Add pci d3cold capable check support."
Typos:
Quoted 1 line
permissisble threshold for vram save/restore. If vram used
permissible
Quoted 1 line
is more then
more than
Quoted 1 line
the threshold value then disabllow
disallow
Quoted 45 lines
the d3cold
to avoid the vram save/restore latency.TODO: At resume vram save/restore only in case card really lost power.
Cc Rodrigo Vivi <[email protected]>Signed-off-by Anshuman Gupta <[email protected]>---
drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46
++++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 +6 files changed, 75 insertions(+)drivers/gpu/drm/xe/xe_device_types.h +3
-0@@ -263,6 +263,9 @@ struct xe_device { bool hold_rpm; } mem_access; + /** d3cold_capable: Indicates if root port is d3cold capable */ + bool d3cold_capable; + /** @d3cold_allowed: Indicates if d3cold is a valid devicestate */
boold3cold_allowed;(quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_module.cindex6860586ce7f8..1dd515ed8a65100644drivers/gpu/drm/xe/xe_module.c +0
-0@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size; module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,0600 ); MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 33
Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c +3
-0+u32 xe_vram_save_restore_threshold =300 ; +module_param_named (vram_save_restore_threshold, xe_vram_save_restore_threshold, uint,0600 ); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vramused
Quoted 7 lines
Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 34
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0
-0threshold below which vram save/restore premissible (in MB)");With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?
probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.
I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?
d3cold is state a for a pci device, so "vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under /sys/bus/pci/devices/$bdf
directory ?
Thanks ,
Anshuman Gupta.
Quoted 169 lines
Quoted 13 lines
Thanks,
ThomasQuoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1
-0+int
xe_guc_log_level= 5;module_param_named(guc_log_level, xe_guc_log_level, int, 0600);MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)"); diff --gita/drivers/gpu/drm/xe/xe_module.h
b/drivers/gpu/drm/xe/xe_module.hQuoted 155 lines
Quoted 154 lines
index
7169907c3365..620d1b80cab6100644drivers/gpu/drm/xe/xe_module.h +1
-0@@ -9,5 +9,6 @@ extern bool enable_guc; extern bool enable_display; extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold; extern int xe_guc_log_level; extern char *xe_param_force_probe;(quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_pci.cindexee61a03d1afd..2c86a8ab727a100644drivers/gpu/drm/xe/xe_pci.c +33
-0@@ -592,6 +592,40 @@ static void xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev) } } +static bool xe_pci_d3cold_capable (struct pci_dev *pdev) { + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability and _PR3 power resource */ + if (!pci_pme_capable(root_pdev, PCI_D3cold) || !pci_pr3_present (root_pdev)) + return false; + + return true; +} + +static void xe_pci_d3cold_enable_disable (struct pci_dev *pdev,bool enable) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold_capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + if (enable) + pci_d3cold_enable(pdev); + else + pci_d3cold_disable(pdev); + +} +static int
xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
const structxe_device_desc*desc = (const void *)ent-Quoted 1 line
driver_data;drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; } + xe->d3cold_capable = xe_pci_d3cold_capable(pdev); + xe_pm_runtime_init(xe);return 0;
drivers/gpu/drm/xe/xe_pci.c +1
-0@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device *dev) pci_set_power_state(pdev, PCI_D3cold); } else { pci_set_power_state(pdev, PCI_D3hot); + xe_pci_d3cold_enable_disable(pdev, false); }return 0;
drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device *dev) return err; pci_set_master(pdev); + } else { + xe_pci_d3cold_enable_disable(pdev, true); }return
xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@ static int xe_pci_runtime_idle(struct device
*dev)
structpci_dev*pdev =to_pci_dev(dev);
structxe_device*xe =pdev_to_xe_device(pdev);+ if (!xe->d3cold_capable) {
+ xe->d3cold_allowed = false;
+ return 0;
+ }drivers/gpu/drm/xe/xe_pci.c +3
-0+ + xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe); +/*
* TODO: d3cold should be allowed (true) if
* (IS_DGFX(xe)&& !xe_device_mem_access_ongoing(xe))drivers/gpu/drm/xe/xe_pm.c +19
-0@@ -16,6 +16,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h" /**@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe) return 0; } +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) { + struct ttm_resource_manager *man; + u64 vram_used; + size_t total_vram_used_mb = 0; + int i; + + /* TODO: Extend the logic to beyond XE_PL_VRAM1 */ + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = ttm_resource_manager_usage(man); + total_vram_used_mb +=DIV_ROUND_UP_ULL (vram_used,1024 *1024 ); + } + } + + return total_vram_used_mb < xe_vram_save_restore_threshold; +} + void xe_pm_runtime_init(struct xe_device *xe) {struct device *dev =
xe->drm.dev; diff --gita/drivers/gpu/drm/xe/xe_pm.hb/drivers/gpu/drm/xe/xe_pm.hindex6a885585f653..33515d5c2cc2100644drivers/gpu/drm/xe/xe_pm.h +1
-0@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe); void xe_pm_runtime_fini(struct xe_device *xe); int xe_pm_runtime_suspend(struct xe_device *xe); int xe_pm_runtime_resume(struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe); int xe_pm_runtime_get(struct xe_device *xe); int xe_pm_runtime_put(struct xe_device *xe); bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
Thomas Hellström
maintainerHi!
Quoted 76 lines · Anshuman
Quoted 76 lines
-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policyQuoted 60 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta
Adding pci d3cold capable check support.
Adding avram_save_restore_thresholdmodparam, that is thePlease use imperative language as per the Linux patch-submission guidelines.
"Add pci d3cold capable check support."
Typos:
Quoted 1 line
permissisble threshold for vram save/restore. If vram used
permissible
Quoted 1 line
is more then
more than
Quoted 1 line
the threshold value then disabllow
disallow
Quoted 45 lines
the d3cold
to avoid the vram save/restore latency.TODO: At resume vram save/restore only in case card really lost power.
Cc Rodrigo Vivi <[email protected]>Signed-off-by Anshuman Gupta <[email protected]>--- drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 + 6 files changed, 75 insertions(+)drivers/gpu/drm/xe/xe_device_types.h +3
-0@@ -263,6 +263,9 @@ struct xe_device { bool hold_rpm; } mem_access; + /** d3cold_capable: Indicates if root port is d3cold capable */ + bool d3cold_capable; + /** @d3cold_allowed: Indicates if d3cold is a valid devicestate */
boold3cold_allowed;(quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_module.cindex6860586ce7f8..1dd515ed8a65100644drivers/gpu/drm/xe/xe_module.c +0
-0@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size; module_param_named(vram_bar_size, xe_force_vram_bar_size, uint,0600 ); MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(in MiB)");Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 33
Quoted 4 lines
drivers/gpu/drm/xe/xe_module.c +3
-0+u32 xe_vram_save_restore_threshold =300 ; +module_param_named (vram_save_restore_threshold, xe_vram_save_restore_threshold, uint,0600 ); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vramused
Quoted 1 line
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0
-0threshold below which vram save/restore premissible (in MB)");
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?
probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?
I was thinking about an example where you have, lets say 4 DG2s that are bound to the same xe driver instance. You want them to have different limits.
Quoted 4 lines
d3cold is state a for a pci device, so "
vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under/sys/bus/pci/devices/$bdf
directory ?
I'm not fully familiar with the sysfs layout used on xe, perhaps Rodrigo can fill in, but perhaps it makes sense to have them in the same directory as the gt0..gtX subdirectories? Looks like it's the same path as you write above.
/Thomas
Quoted 169 lines
Thanks ,
Anshuman Gupta.Quoted 166 lines
Quoted 11 lines
Thanks,
ThomasQuoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1
-0+int
xe_guc_log_level= 5;module_param_named(guc_log_level, xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level
(0=disable, 1..5=enable with verbosity min..max)"); diff --gita/drivers/gpu/drm/xe/xe_module.h
b/drivers/gpu/drm/xe/xe_module.hQuoted 154 lines
Quoted 154 lines
index
7169907c3365..620d1b80cab6100644drivers/gpu/drm/xe/xe_module.h +1
-0@@ -9,5 +9,6 @@ extern bool enable_guc; extern bool enable_display; extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold; extern int xe_guc_log_level; extern char *xe_param_force_probe;(quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_pci.cindexee61a03d1afd..2c86a8ab727a100644drivers/gpu/drm/xe/xe_pci.c +33
-0@@ -592,6 +592,40 @@ static void xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev) } } +static bool xe_pci_d3cold_capable (struct pci_dev *pdev) { + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability and _PR3 power resource */ + if (!pci_pme_capable(root_pdev, PCI_D3cold) || !pci_pr3_present (root_pdev)) + return false; + + return true; +} + +static void xe_pci_d3cold_enable_disable (struct pci_dev *pdev,bool enable) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold_capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + if (enable) + pci_d3cold_enable(pdev); + else + pci_d3cold_disable(pdev); + +} + static int xe_pci_probe(struct pci_dev *pdev, const struct
pci_device_id*ent)
{
const structxe_device_desc*desc = (const void *)ent-Quoted 1 line
driver_data;drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; } + xe->d3cold_capable = xe_pci_d3cold_capable(pdev); + xe_pm_runtime_init(xe);return 0;
drivers/gpu/drm/xe/xe_pci.c +1
-0@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device *dev) pci_set_power_state(pdev, PCI_D3cold); } else { pci_set_power_state(pdev, PCI_D3hot); + xe_pci_d3cold_enable_disable(pdev, false); }return 0;
drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device *dev) return err; pci_set_master(pdev); + } else { + xe_pci_d3cold_enable_disable(pdev, true); }return
xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@ static
int xe_pci_runtime_idle(struct device
*dev)
structpci_dev*pdev =to_pci_dev(dev);
structxe_device*xe =pdev_to_xe_device(pdev);+ if (!xe->d3cold_capable) {
+ xe->d3cold_allowed = false;
+ return 0;
+ }drivers/gpu/drm/xe/xe_pci.c +3
-0+ + xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe); + /** TODO: d3cold should be allowed (true) if * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe)) drivers/gpu/drm/xe/xe_pm.c +19
-0@@ -16,6 +16,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h" /**@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe) return 0; } +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) { + struct ttm_resource_manager *man; + u64 vram_used; + size_t total_vram_used_mb = 0; + int i; + + /* TODO: Extend the logic to beyond XE_PL_VRAM1 */ + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = ttm_resource_manager_usage(man); + total_vram_used_mb +=DIV_ROUND_UP_ULL (vram_used,1024 *1024 ); + } + } + + return total_vram_used_mb < xe_vram_save_restore_threshold; +} + void xe_pm_runtime_init(struct xe_device *xe) { struct device *dev = xe->drm.dev; diff --git
a/drivers/gpu/drm/xe/xe_pm.hb/drivers/gpu/drm/xe/xe_pm.hindex6a885585f653..33515d5c2cc2100644drivers/gpu/drm/xe/xe_pm.h +1
-0@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe); void xe_pm_runtime_fini(struct xe_device *xe); int xe_pm_runtime_suspend(struct xe_device *xe); int xe_pm_runtime_resume(struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe); int xe_pm_runtime_get(struct xe_device *xe); int xe_pm_runtime_put(struct xe_device *xe); bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
Quoted 106 lines
-----Original Message-----
From: Thomas Hellström <[email protected]>
Sent: Monday, May 15, 2023 3:08 PM
To: Gupta, Anshuman <[email protected]>; Rodrigo Vivi <[email protected]>
Cc: [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policyHi!
Quoted 61 lines · Anshuman
Quoted 61 lines
-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policyQuoted 51 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta
Adding pci d3cold capable check support.
Adding avram_save_restore_thresholdmodparam, that is thePlease use imperative language as per the Linux patch-submission guidelines.
"Add pci d3cold capable check support."
Typos:
Quoted 1 line
permissisble threshold for vram save/restore. If vram used
permissible
Quoted 1 line
is more then
more than
Quoted 1 line
the threshold value then disabllow
disallow
Quoted 36 lines
the d3cold
to avoid the vram save/restore latency.TODO: At resume vram save/restore only in case card really lost power.
Cc Rodrigo Vivi <[email protected]>Signed-off-by Anshuman Gupta <[email protected]>--- drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 + 6 files changed, 75 insertions(+)drivers/gpu/drm/xe/xe_device_types.h +4
-0@@ -263,6 +263,9 @@ struct xe_device { bool hold_rpm; } mem_access; + /** d3cold_capable: Indicates if root port is d3cold +capable */ + bool d3cold_capable; + /** @d3cold_allowed: Indicates if d3cold is a valid devicestate */
boold3cold_allowed;Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
(quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_module.cindex
6860586ce7f8..1dd515ed8a65Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 26
Quoted 5 lines
100644
drivers/gpu/drm/xe/xe_module.c +0
-0@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size; module_param_named(vram_bar_size, xe_force_vram_bar_size,uint,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 28
Quoted 2 lines
drivers/gpu/drm/xe/xe_module.c +0
-00600); MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(inMiB)");
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 32
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c +3
-0+u32 xe_vram_save_restore_threshold =300 ; +module_param_named (vram_save_restore_threshold, xe_vram_save_restore_threshold, uint,0600 ); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vramused
Quoted 1 line
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0
-0threshold below which vram save/restore premissible (in MB)");Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?
probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?I was thinking about an example where you have, lets say 4 DG2s that are bound to the same xe driver instance. You want them to have different limits.
Quoted 8 lines
Quoted 4 lines
d3cold is state a for a pci device, so "
vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under/sys/bus/pci/devices/$bdfdirectory ?I'm not fully familiar with the sysfs layout used on xe, perhaps Rodrigo can fill in, but perhaps it makes sense to have them in the same directory as the gt0..gtX subdirectories? Looks like it's the same path as you write above.
Thanks thomas for review comment, I will fix this in next revision. Br,
Anshuman Gupta.
Quoted 177 lines
/Thomas
Quoted 173 lines
Thanks ,
Anshuman Gupta.Quoted 170 lines
Quoted 11 lines
Thanks,
ThomasQuoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1
-0+ int xe_guc_log_level = 5;module_param_named(guc_log_level,
xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level
(0=disable, 1..5=enable with verbosity min..max)"); diff --gita/drivers/gpu/drm/xe/xe_module.h
b/drivers/gpu/drm/xe/xe_module.hQuoted 158 lines
Quoted 158 lines
index
7169907c3365..620d1b80cab6100644drivers/gpu/drm/xe/xe_module.h +1
-0@@ -9,5 +9,6 @@ extern bool enable_guc; extern bool enable_display; extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold; extern int xe_guc_log_level; extern char *xe_param_force_probe; diff --git
a/drivers/gpu/drm/xe/xe_pci.cb/drivers/gpu/drm/xe/xe_pci.cindexee61a03d1afd..2c86a8ab727a
100644drivers/gpu/drm/xe/xe_pci.c +35
-0@@ -592,6 +592,40 @@ static void xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev) } } +static bool xe_pci_d3cold_capable (struct pci_dev *pdev) { + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability and _PR3 power resource +*/ + if (!pci_pme_capable(root_pdev, PCI_D3cold) || !pci_pr3_present (root_pdev)) + return false; + + return true; +} + +static void xe_pci_d3cold_enable_disable (struct pci_dev *pdev, +bool enable) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold_capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + if (enable) + pci_d3cold_enable(pdev); + else + pci_d3cold_disable(pdev); + +} + static int xe_pci_probe(struct pci_dev *pdev, const struct
pci_device_id*ent)
{
const structxe_device_desc*desc = (const void *)ent-Quoted 1 line
driver_data;drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; } + xe->d3cold_capable = xe_pci_d3cold_capable(pdev); + xe_pm_runtime_init(xe);return 0;
drivers/gpu/drm/xe/xe_pci.c +1
-0@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device *dev) pci_set_power_state(pdev, PCI_D3cold); } else { pci_set_power_state(pdev, PCI_D3hot); + xe_pci_d3cold_enable_disable(pdev, false); }return 0;
drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device *dev) return err; pci_set_master(pdev); + } else { + xe_pci_d3cold_enable_disable(pdev, true); }return
xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@
static int xe_pci_runtime_idle(struct device
*dev)
structpci_dev*pdev =to_pci_dev(dev); structxe_device*xe =pdev_to_xe_device(pdev);+ if (!xe->d3cold_capable) {
+ xe->d3cold_allowed = false;
+ return 0;
+ }drivers/gpu/drm/xe/xe_pci.c +3
-0+ + xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe); + /** TODO: d3cold should be allowed (true) if * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe)) (quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_pm.cindexb7b57f10ba25..523d5f89ec89100644drivers/gpu/drm/xe/xe_pm.c +20
-0@@ -16,6 +16,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h" /**@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe) return 0; } +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) { + struct ttm_resource_manager *man; + u64 vram_used; + size_t total_vram_used_mb = 0; + int i; + + /* TODO: Extend the logic to beyond XE_PL_VRAM1 */ + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = +ttm_resource_manager_usage (man); + total_vram_used_mb +=DIV_ROUND_UP_ULL (vram_used,1024 *1024 ); + } + } + + return total_vram_used_mb < xe_vram_save_restore_threshold; +} + void xe_pm_runtime_init(struct xe_device *xe) {struct device *dev =
xe->drm.dev; diff --gita/drivers/gpu/drm/xe/xe_pm.hb/drivers/gpu/drm/xe/xe_pm.hindex6a885585f653..33515d5c2cc2100644drivers/gpu/drm/xe/xe_pm.h +1
-0@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe); void xe_pm_runtime_fini(struct xe_device *xe); int xe_pm_runtime_suspend(struct xe_device *xe); int xe_pm_runtime_resume(struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe); int xe_pm_runtime_get(struct xe_device *xe); int xe_pm_runtime_put(struct xe_device *xe); bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);
Quoted 106 lines
-----Original Message-----
From: Thomas Hellström <[email protected]>
Sent: Monday, May 15, 2023 3:08 PM
To: Gupta, Anshuman <[email protected]>; Rodrigo Vivi <[email protected]>
Cc: [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policyHi!
Quoted 61 lines · Anshuman
Quoted 61 lines
-----Original Message-----
From: Rodrigo Vivi <[email protected]>
Sent: Friday, May 12, 2023 7:26 PM
To: Thomas Hellström <[email protected]>
Cc: Gupta, Anshuman <[email protected]>; intel- [email protected]; Vivi, Rodrigo <[email protected]>
Subject: Re: [Intel-xe] [RFC] drm/xe/pm: d3cold policyQuoted 51 lines · Thomas Hellström
Quoted 2 lines · Anshuman Gupta
Adding pci d3cold capable check support.
Adding avram_save_restore_thresholdmodparam, that is thePlease use imperative language as per the Linux patch-submission guidelines.
"Add pci d3cold capable check support."
Typos:
Quoted 1 line
permissisble threshold for vram save/restore. If vram used
permissible
Quoted 1 line
is more then
more than
Quoted 1 line
the threshold value then disabllow
disallow
Quoted 36 lines
the d3cold
to avoid the vram save/restore latency.TODO: At resume vram save/restore only in case card really lost power.
Cc Rodrigo Vivi <[email protected]>Signed-off-by Anshuman Gupta <[email protected]>--- drivers/gpu/drm/xe/xe_device_types.h | 3 ++ drivers/gpu/drm/xe/xe_module.c | 4 +++ drivers/gpu/drm/xe/xe_module.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 46 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pm.c | 20 ++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 + 6 files changed, 75 insertions(+)drivers/gpu/drm/xe/xe_device_types.h +4
-0@@ -263,6 +263,9 @@ struct xe_device { bool hold_rpm; } mem_access; + /** d3cold_capable: Indicates if root port is d3cold +capable */ + bool d3cold_capable; + /** @d3cold_allowed: Indicates if d3cold is a valid devicestate */
boold3cold_allowed;Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
(quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_module.cindex
6860586ce7f8..1dd515ed8a65Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 26
Quoted 5 lines
100644
drivers/gpu/drm/xe/xe_module.c +0
-0@@ -26,6 +26,10 @@ u32 xe_force_vram_bar_size; module_param_named(vram_bar_size, xe_force_vram_bar_size,uint,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 28
Quoted 2 lines
drivers/gpu/drm/xe/xe_module.c +0
-00600); MODULE_PARM_DESC(vram_bar_size, "Set the vram bar size(inMiB)");
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c u32 xe_force_vram_bar_size; · line 32
Quoted 5 lines
drivers/gpu/drm/xe/xe_module.c +3
-0+u32 xe_vram_save_restore_threshold =300 ; +module_param_named (vram_save_restore_threshold, xe_vram_save_restore_threshold, uint,0600 ); +MODULE_PARM_DESC(vram_save_restore_threshold, "Set the vramused
Quoted 1 line
Quoted 1 line
drivers/gpu/drm/xe/xe_module.c +0
-0threshold below which vram save/restore premissible (in MB)");Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
With one-driver-multiple-devices, Would we want to use this as the default template and then have an interface to set it per-device?
probably a good idea. to have a sysfs per device.
and sysfs only appears if d3cold is ever possible and supported.I have some doubts over handling of sysfs.
AFAIU this modparam is per pci device.
What are we referring here as per device ?
Is it multiple tiles ?I was thinking about an example where you have, lets say 4 DG2s that are bound to the same xe driver instance. You want them to have different limits.
Quoted 8 lines
Quoted 4 lines
d3cold is state a for a pci device, so "
vram_used" metric has to be account all vram instances in a gpu card, so sysfs for this parameter should be linked to pci device under/sys/bus/pci/devices/$bdfdirectory ?I'm not fully familiar with the sysfs layout used on xe, perhaps Rodrigo can fill in, but perhaps it makes sense to have them in the same directory as the gt0..gtX subdirectories? Looks like it's the same path as you write above.
Hi Thomas,
I have added a new sysfs called vram_d3cold_threshold and floated the patch
https://patchwork.freedesktop.org/patch/541520/?series=118035&rev=2
But am skeptical on the user space consumer for this sysfs,
I can write a IGT to use this sysfs but will that suffice the user space consumer requirement ? Do you have any idea how other GPU handles it or they do always evict entire vram ?
Thanks,
Anshuman Gupta.
Br,
Anshuman Gupta.
Quoted 177 lines
/Thomas
Quoted 173 lines
Thanks ,
Anshuman Gupta.Quoted 170 lines
Quoted 11 lines
Thanks,
ThomasQuoted 6 lines
drivers/gpu/drm/xe/xe_module.c +1
-0+ int xe_guc_log_level = 5;module_param_named(guc_log_level,
xe_guc_log_level, int, 0600); MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level
(0=disable, 1..5=enable with verbosity min..max)"); diff --gita/drivers/gpu/drm/xe/xe_module.h
b/drivers/gpu/drm/xe/xe_module.hQuoted 158 lines
Quoted 158 lines
index
7169907c3365..620d1b80cab6100644drivers/gpu/drm/xe/xe_module.h +1
-0@@ -9,5 +9,6 @@ extern bool enable_guc; extern bool enable_display; extern u32 xe_force_vram_bar_size; +extern u32 xe_vram_save_restore_threshold; extern int xe_guc_log_level; extern char *xe_param_force_probe; diff --git
a/drivers/gpu/drm/xe/xe_pci.cb/drivers/gpu/drm/xe/xe_pci.cindexee61a03d1afd..2c86a8ab727a
100644drivers/gpu/drm/xe/xe_pci.c +35
-0@@ -592,6 +592,40 @@ static void xe_pci_unbounded_bridge_disable_pm(struct pci_dev *pdev) } } +static bool xe_pci_d3cold_capable (struct pci_dev *pdev) { + struct pci_dev *root_pdev; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return false; + + /* D3Cold requires PME capability and _PR3 power resource +*/ + if (!pci_pme_capable(root_pdev, PCI_D3cold) || !pci_pr3_present (root_pdev)) + return false; + + return true; +} + +static void xe_pci_d3cold_enable_disable (struct pci_dev *pdev, +bool enable) +{ + struct xe_device *xe = pdev_to_xe_device(pdev); + struct pci_dev *root_pdev; + + if (!xe->d3cold_capable) + return; + + root_pdev = pcie_find_root_port(pdev); + if (!root_pdev) + return; + + if (enable) + pci_d3cold_enable(pdev); + else + pci_d3cold_disable(pdev); + +} + static int xe_pci_probe(struct pci_dev *pdev, const struct
pci_device_id*ent)
{
const structxe_device_desc*desc = (const void *)ent-Quoted 1 line
driver_data;drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -669,6 +703,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; } + xe->d3cold_capable = xe_pci_d3cold_capable(pdev); + xe_pm_runtime_init(xe);return 0;
drivers/gpu/drm/xe/xe_pci.c +1
-0@@ -741,6 +777,7 @@ static int xe_pci_runtime_suspend(struct device *dev) pci_set_power_state(pdev, PCI_D3cold); } else { pci_set_power_state(pdev, PCI_D3hot); + xe_pci_d3cold_enable_disable(pdev, false); }return 0;
drivers/gpu/drm/xe/xe_pci.c +2
-0@@ -764,6 +801,8 @@ static int xe_pci_runtime_resume(struct device *dev) return err; pci_set_master(pdev); + } else { + xe_pci_d3cold_enable_disable(pdev, true); }return
xe_pm_runtime_resume(xe); @@ -774,6 +813,13 @@
static int xe_pci_runtime_idle(struct device
*dev)
structpci_dev*pdev =to_pci_dev(dev); structxe_device*xe =pdev_to_xe_device(pdev);+ if (!xe->d3cold_capable) {
+ xe->d3cold_allowed = false;
+ return 0;
+ }drivers/gpu/drm/xe/xe_pci.c +3
-0+ + xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe); + /** TODO: d3cold should be allowed (true) if * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe)) (quoted diff) +0
-0
b/drivers/gpu/drm/xe/xe_pm.cindexb7b57f10ba25..523d5f89ec89100644drivers/gpu/drm/xe/xe_pm.c +20
-0@@ -16,6 +16,7 @@ #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_irq.h" +#include "xe_module.h" #include "xe_pcode.h" /**@@ -117,6 +118,25 @@ int xe_pm_resume(struct xe_device *xe) return 0; } +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe) { + struct ttm_resource_manager *man; + u64 vram_used; + size_t total_vram_used_mb = 0; + int i; + + /* TODO: Extend the logic to beyond XE_PL_VRAM1 */ + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + vram_used = +ttm_resource_manager_usage (man); + total_vram_used_mb +=DIV_ROUND_UP_ULL (vram_used,1024 *1024 ); + } + } + + return total_vram_used_mb < xe_vram_save_restore_threshold; +} + void xe_pm_runtime_init(struct xe_device *xe) {struct device *dev =
xe->drm.dev; diff --gita/drivers/gpu/drm/xe/xe_pm.hb/drivers/gpu/drm/xe/xe_pm.hindex6a885585f653..33515d5c2cc2100644drivers/gpu/drm/xe/xe_pm.h +1
-0@@ -17,6 +17,7 @@ void xe_pm_runtime_init(struct xe_device *xe); void xe_pm_runtime_fini(struct xe_device *xe); int xe_pm_runtime_suspend(struct xe_device *xe); int xe_pm_runtime_resume(struct xe_device *xe); +bool xe_pm_vram_d3cold_allowed (struct xe_device *xe); int xe_pm_runtime_get(struct xe_device *xe); int xe_pm_runtime_put(struct xe_device *xe); bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe);