drm/amdkfd: Add eviction debug messages
Subsystems (3)
Felix Kuehling
maintainerPATCH 1/1 drm/amdkfd: Add eviction debug messages
Use WARN to print messages with backtrace when evictions are triggered. This can help determine the root cause of evictions and help spot driver bugs triggering evictions unintentionally, or help with performance tuning by avoiding conditions that cause evictions in a specific workload.
The messages are controlled by a new module parameter that can be changed at runtime:
echo Y > /sys/module/amdgpu/parameters/debug_evictions echo N > /sys/module/amdgpu/parameters/debug_evictions
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 ++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 2 ++ drivers/gpu/drm/amd/amdkfd/kfd_device.c | 3 +++ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +++++ 5 files changed, 20 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
@@ -186,8 +186,10 @@ extern int amdgpu_noretry;extern int amdgpu_force_asic_type;#ifdef CONFIG_HSA_AMD extern int sched_policy; +extern bool debug_evictions;#else static const int sched_policy =KFD_SCHED_POLICY_HWS ; +static const bool debug_evictions;/* = false */ #endif extern int amdgpu_tmz;
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +8 -0
@@ -705,6 +705,14 @@ MODULE_PARM_DESC(hws_gws_support, "Assume MEC2 FW supports GWS barriers (false =int queue_preemption_timeout_ms =9000 ;module_param (queue_preemption_timeout_ms, int,0644 );MODULE_PARM_DESC (queue_preemption_timeout_ms,"queue preemption timeout in ms (1 = Minimum, 9000 = default)" ); + +/** +* DOC: debug_evictions(bool) +* Enable extra debug messages to help determine the cause of evictions +*/ +bool debug_evictions; +module_param (debug_evictions, bool,0644 ); +MODULE_PARM_DESC (debug_evictions,"enable eviction debug messages (false = default)" );#endif /**
drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +2 -0
@@ -275,6 +275,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,continue ; } +WARN (debug_evictions && fence_owner ==AMDGPU_FENCE_OWNER_KFD , +"Adding eviction fence to sync obj" ); r =amdgpu_sync_fence (sync, f, false);if (r)break ;
drivers/gpu/drm/amd/amdkfd/kfd_device.c +3 -0
@@ -942,6 +942,7 @@ int kgd2kfd_quiesce_mm(struct mm_struct *mm)if (!p)return -ESRCH ; +WARN (debug_evictions,"Evicting pid %d" , p->lead_thread->pid); r =kfd_process_evict_queues (p);kfd_unref_process (p);
@@ -1009,6 +1010,8 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,/* During process initialization eviction_work.dwork is initialized * to kfd_evict_bo_worker */ +WARN (debug_evictions,"Scheduling eviction of pid %d in %ld jiffies" , + p->lead_thread->pid, delay_jiffies);schedule_delayed_work (&p->eviction_work, delay_jiffies); out:kfd_unref_process (p);
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +5 -0
@@ -177,6 +177,11 @@ extern bool hws_gws_support;*/ extern int queue_preemption_timeout_ms; +/* +* Enable eviction debug messages +*/ +extern bool debug_evictions; +enum cache_policy { cache_policy_coherent, cache_policy_noncoherent
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
It's good idea, better to add same print in system memory eviction path amdgpu_amdkfd_evict_userptr.
Use WARN_ONCE to avoid duplicate messages.
Regards,
Philip
Quoted 94 lines · Felix Kuehling
Use WARN to print messages with backtrace when evictions are triggered. This can help determine the root cause of evictions and help spot driver bugs triggering evictions unintentionally, or help with performance tuning by avoiding conditions that cause evictions in a specific workload.
The messages are controlled by a new module parameter that can be changed at runtime:
echo Y >
/sys/module/amdgpu/parameters/debug_evictionsecho N >/sys/module/amdgpu/parameters/debug_evictionsSigned-off-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 ++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 2 ++ drivers/gpu/drm/amd/amdkfd/kfd_device.c | 3 +++ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +++++ 5 files changed, 20 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -186,8 +186,10 @@ extern int amdgpu_noretry;extern int amdgpu_force_asic_type;#ifdef CONFIG_HSA_AMD extern int sched_policy; +extern bool debug_evictions;#else static const int sched_policy =KFD_SCHED_POLICY_HWS ; +static const bool debug_evictions;/* = false */ #endif extern int amdgpu_tmz;drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +8
-0@@ -705,6 +705,14 @@ MODULE_PARM_DESC(hws_gws_support, "Assume MEC2 FW supports GWS barriers (false =int queue_preemption_timeout_ms =9000 ;module_param (queue_preemption_timeout_ms, int,0644 );MODULE_PARM_DESC (queue_preemption_timeout_ms,"queue preemption timeout in ms (1 = Minimum, 9000 = default)" ); + +/** +* DOC: debug_evictions(bool) +* Enable extra debug messages to help determine the cause of evictions +*/ +bool debug_evictions; +module_param (debug_evictions, bool,0644 ); +MODULE_PARM_DESC (debug_evictions,"enable eviction debug messages (false = default)" );#endif /** drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +2
-0@@ -275,6 +275,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,continue ; } +WARN (debug_evictions && fence_owner ==AMDGPU_FENCE_OWNER_KFD , +"Adding eviction fence to sync obj" ); r =amdgpu_sync_fence (sync, f, false);if (r)break ;drivers/gpu/drm/amd/amdkfd/kfd_device.c +3
-0@@ -942,6 +942,7 @@ int kgd2kfd_quiesce_mm(struct mm_struct *mm)if (!p)return -ESRCH ; +WARN (debug_evictions,"Evicting pid %d" , p->lead_thread->pid); r =kfd_process_evict_queues (p);kfd_unref_process (p);@@ -1009,6 +1010,8 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,/* During process initialization eviction_work.dwork is initialized * to kfd_evict_bo_worker */ +WARN (debug_evictions,"Scheduling eviction of pid %d in %ld jiffies" , + p->lead_thread->pid, delay_jiffies);schedule_delayed_work (&p->eviction_work, delay_jiffies); out:kfd_unref_process (p);drivers/gpu/drm/amd/amdkfd/kfd_priv.h +0
-0@@ -177,6 +177,11 @@ extern bool hws_gws_support;*/
drivers/gpu/drm/amd/amdkfd/kfd_priv.h extern bool hws_gws_support; · line 187
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +5
-0extern int queue_preemption_timeout_ms; +/* +* Enable eviction debug messages +*/ +extern bool debug_evictions; +enum cache_policy { cache_policy_coherent, cache_policy_noncoherent
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Felix Kuehling
maintainerAm 2020-06-12 um 6:03 p.m. schrieb philip yang:
Quoted 2 lines
It's good idea, better to add same print in system memory eviction path
amdgpu_amdkfd_evict_userptr.
That's covered by the message in kgd2kfd_quiesce_mm.
Quoted 2 lines
Use
WARN_ONCEto avoid duplicate messages.
I want duplicate messages. If many different kinds of evictions are happening I want to see them all. The module parameter is there so I can turn it on/off for short bursts while interesting things are happening. It's off by default.
I was considering WARN_RATELIMIT, but that may skip interesting evictions I actually want to see.
Regards,
Felix
Quoted 112 lines
Regards,
Philip
Quoted 105 lines · Felix Kuehling
Use WARN to print messages with backtrace when evictions are triggered. This can help determine the root cause of evictions and help spot driver bugs triggering evictions unintentionally, or help with performance tuning
by avoiding conditions that cause evictions in a specific workload.The messages are controlled by a new module parameter that can be changed
at runtime:echo Y >
/sys/module/amdgpu/parameters/debug_evictionsecho N >/sys/module/amdgpu/parameters/debug_evictionsSigned-off-by Felix Kuehling <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 ++++++++drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c| 2 ++ drivers/gpu/drm/amd/amdkfd/kfd_device.c | 3 +++ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +++++5 files changed, 20 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -186,8 +186,10 @@ extern int amdgpu_noretry; extern int amdgpu_force_asic_type; #ifdef CONFIG_HSA_AMD extern int sched_policy; +extern bool debug_evictions; #else static const int sched_policy = KFD_SCHED_POLICY_HWS; +static const bool debug_evictions;/* = false */ #endif extern int amdgpu_tmz;drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +0
-0@@ -705,6 +705,14 @@ MODULE_PARM_DESC(hws_gws_support, "Assume MEC2FW supports GWS barriers (false =
intqueue_preemption_timeout_ms= 9000;module_param(queue_preemption_timeout_ms, int, 0644);MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption timeout in ms (1 = Minimum, 9000 = default)");drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +8
-0+ +/** +* DOC: debug_evictions(bool) +* Enable extra debug messages to help determine the cause of evictions +*/ +bool debug_evictions; +module_param (debug_evictions, bool,0644 ); +MODULE_PARM_DESC(debug_evictions, "enable eviction debug messages(false = default)");
#endif
/**drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +0
-0@@ -275,6 +275,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync, continue; }+ WARN(debug_evictions &&
fence_owner==AMDGPU_FENCE_OWNER_KFD,
+ "Adding eviction fence to sync obj"); r =amdgpu_sync_fence(sync, f, false); if (r)
break;drivers/gpu/drm/amd/amdkfd/kfd_device.c +0
-0@@ -942,6 +942,7 @@ int kgd2kfd_quiesce_mm(struct mm_struct *mm) if (!p) return -ESRCH; + WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); r = kfd_process_evict_queues(p); kfd_unref_process(p);@@ -1009,6 +1010,8 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, /* During process initialization eviction_work.dwork isinitialized
* tokfd_evict_bo_worker
*/
+ WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies",
+p->lead_thread->pid,delay_jiffies);schedule_delayed_work(&p->eviction_work, delay_jiffies); out:
kfd_unref_process(p);drivers/gpu/drm/amd/amdkfd/kfd_priv.h +0
-0@@ -177,6 +177,11 @@ extern bool hws_gws_support;*/ extern int queue_preemption_timeout_ms;
Quoted 8 lines
Quoted 8 lines
+/*
+ * Enable eviction debug messages
+ */drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2
-0+extern bool debug_evictions; +enum
cache_policy{
cache_policy_coherent,
cache_policy_noncoherent
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
philip yang
Reviewed-byThanks for the explanation.
Quoted 130 lines · Felix Kuehling
Am 2020-06-12 um 6:03 p.m. schrieb philip yang:
Quoted 2 lines
It's good idea, better to add same print in system memory eviction path
amdgpu_amdkfd_evict_userptr.That's covered by the message in
kgd2kfd_quiesce_mm.Quoted 1 line
Use
WARN_ONCEto avoid duplicate messages.I want duplicate messages. If many different kinds of evictions are happening I want to see them all. The module parameter is there so I can turn it on/off for short bursts while interesting things are happening. It's off by default.
I was considering
WARN_RATELIMIT, but that may skip interesting evictions I actually want to see.Regards,
FelixQuoted 111 lines
Regards,
Philip
Quoted 105 lines · Felix Kuehling
Use WARN to print messages with backtrace when evictions are triggered. This can help determine the root cause of evictions and help spot driver bugs triggering evictions unintentionally, or help with performance tuning
by avoiding conditions that cause evictions in a specific workload.The messages are controlled by a new module parameter that can be changed
at runtime:echo Y >
/sys/module/amdgpu/parameters/debug_evictionsecho N >/sys/module/amdgpu/parameters/debug_evictionsSigned-off-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 ++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 2 ++ drivers/gpu/drm/amd/amdkfd/kfd_device.c | 3 +++ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 5 +++++ 5 files changed, 20 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -186,8 +186,10 @@ extern int amdgpu_noretry; extern int amdgpu_force_asic_type; #ifdef CONFIG_HSA_AMD extern int sched_policy; +extern bool debug_evictions; #else static const int sched_policy = KFD_SCHED_POLICY_HWS; +static const bool debug_evictions;/* = false */ #endif extern int amdgpu_tmz;drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +0
-0@@ -705,6 +705,14 @@ MODULE_PARM_DESC(hws_gws_support, "Assume MEC2FW supports GWS barriers (false =
intqueue_preemption_timeout_ms= 9000;module_param(queue_preemption_timeout_ms, int, 0644); MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption
timeout in ms (1 = Minimum, 9000 = default)");drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +8
-0+ +/** +* DOC: debug_evictions(bool) +* Enable extra debug messages to help determine the cause of evictions +*/ +bool debug_evictions; +module_param (debug_evictions, bool,0644 ); +MODULE_PARM_DESC(debug_evictions, "enable eviction debug messages(false = default)");
#endif
/**drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +1
-0@@ -275,6 +275,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,struct amdgpu_sync *sync, continue; } + WARN(debug_evictions && fence_owner ==AMDGPU_FENCE_OWNER_KFD , + "Adding eviction fence to sync obj"); r = amdgpu_sync_fence(sync, f, false); if (r) break;drivers/gpu/drm/amd/amdkfd/kfd_device.c +0
-0@@ -942,6 +942,7 @@ int kgd2kfd_quiesce_mm(struct mm_struct *mm) if (!p) return -ESRCH; + WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); r = kfd_process_evict_queues(p); kfd_unref_process(p);@@ -1009,6 +1010,8 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, /* During process initialization eviction_work.dwork isinitialized
* tokfd_evict_bo_worker
*/
+ WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies",
+p->lead_thread->pid,delay_jiffies);
schedule_delayed_work(&p->eviction_work, delay_jiffies); out:
kfd_unref_process(p);drivers/gpu/drm/amd/amdkfd/kfd_priv.h +0
-0@@ -177,6 +177,11 @@ extern bool hws_gws_support;*/ extern int queue_preemption_timeout_ms;
Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h extern bool hws_gws_support; · line 187
Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +4
-0+/* +* Enable eviction debug messages +*/ +extern bool debug_evictions; + enum cache_policy { cache_policy_coherent, cache_policy_noncoherent
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx