drm/amdkfd: option to disable system mem limit
Subsystems (3)
PATCH v2 1/1 drm/amdkfd: option to disable system mem limit
If multiple process share system memory through /dev/shm, KFD allocate memory should not fail if it reaches the system memory limit because one copy of physical system memory are shared by multiple process.
Add module parameter no_system_mem_limit to provide user option to disable system memory limit check at runtime using sysfs or during driver module init using kernel boot argument. By default the system memory limit is on.
Print out debug message to warn user if KFD allocate memory failed because system memory reaches limit.
v2: support change setting at runtime using sysfs
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 12 +++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 9 +++++++++ 3 files changed, 22 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
@@ -188,9 +188,11 @@ extern int amdgpu_force_asic_type;#ifdef CONFIG_HSA_AMD extern int sched_policy;extern bool debug_evictions; +extern bool no_system_mem_limit;#else static const int sched_policy =KFD_SCHED_POLICY_HWS ;static const bool debug_evictions;/* = false */ +static const bool no_system_mem_limit;#endif extern int amdgpu_tmz;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +11 -1
@@ -136,7 +136,10 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, ttm_mem_needed = acc_size + size; }else if (domain ==AMDGPU_GEM_DOMAIN_CPU && !sg) {/* Userptr */ - system_mem_needed = acc_size + size; +if (no_system_mem_limit) + system_mem_needed =0 ; +else + system_mem_needed = acc_size + size; ttm_mem_needed = acc_size; }else {/* VRAM and SG */
@@ -148,6 +151,10 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,spin_lock (&kfd_mem_limit.mem_limit_lock); +if (kfd_mem_limit.system_mem_used + system_mem_needed > + kfd_mem_limit.max_system_mem_limit) +pr_debug ("Set no_system_mem_limit=1 if using shared memory\n" ); + if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
@@ -204,6 +211,9 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)bool sg = (bo->preferred_domains ==AMDGPU_GEM_DOMAIN_CPU );if (bo->flags &AMDGPU_AMDKFD_USERPTR_BO ) { +if (no_system_mem_limit) +return ; + domain =AMDGPU_GEM_DOMAIN_CPU ; sg = false; }
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +9 -0
@@ -717,6 +717,15 @@ MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption timeout in ms (1bool debug_evictions;module_param (debug_evictions, bool,0644 );MODULE_PARM_DESC (debug_evictions,"enable eviction debug messages (false = default)" ); + +/** +* DOC: no_system_mem_limit(bool) +* Disable system memory limit, to support multiple process shared memory +*/ +bool no_system_mem_limit; +module_param (no_system_mem_limit, bool,0644 ); +MODULE_PARM_DESC (no_system_mem_limit,"disable system memory limit (false = default)" ); +#endif /**
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Felix Kuehling
maintainerAm 2020-08-04 um 4:42 p.m. schrieb Philip Yang:
Quoted 42 lines
If multiple process share system memory through
/dev/shm, KFD allocate memory should not fail if it reaches the system memory limit because one copy of physical system memory are shared by multiple process.Add module parameter
no_system_mem_limitto provide user option to disable system memory limit check at runtime using sysfs or during driver module init using kernel boot argument. By default the system memory limit is on.Print out debug message to warn user if KFD allocate memory failed because system memory reaches limit.
v2: support change setting at runtime using sysfs
Signed-off-by Philip Yang <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 12 +++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 9 +++++++++ 3 files changed, 22 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -188,9 +188,11 @@ extern int amdgpu_force_asic_type;#ifdef CONFIG_HSA_AMD extern int sched_policy;extern bool debug_evictions; +extern bool no_system_mem_limit;#else static const int sched_policy =KFD_SCHED_POLICY_HWS ;static const bool debug_evictions;/* = false */ +static const bool no_system_mem_limit;#endif extern int amdgpu_tmz;drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, · line 142
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +4
-1@@ -136,7 +136,10 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, ttm_mem_needed = acc_size + size; }else if (domain ==AMDGPU_GEM_DOMAIN_CPU && !sg) {/* Userptr */ - system_mem_needed = acc_size + size; +if (no_system_mem_limit) + system_mem_needed =0 ; +else + system_mem_needed = acc_size + size;
This will give incorrect results if the user changes the setting at runtime. It would be better to do all the accounting correctly but let the module parameter only affect the condition that checks whether the reservation is OK.
E.g.
if ((kfd_mem_limit.system_mem_used + system_mem_needed >
kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
/* ^--------- */
(kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
kfd_mem_limit.max_ttm_mem_limit) ||
(adev->kfd.vram_used + vram_needed >
adev->gmc.real_vram_size - reserved_for_pt)) {
ret = -ENOMEM;
} ...Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +4
-0ttm_mem_needed = acc_size; }else {/* VRAM and SG */ @@ -148,6 +151,10 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,spin_lock (&kfd_mem_limit.mem_limit_lock); +if (kfd_mem_limit.system_mem_used + system_mem_needed > + kfd_mem_limit.max_system_mem_limit) +pr_debug ("Set no_system_mem_limit=1 if using shared memory\n" ); + if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) · line 216
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +3
-0@@ -204,6 +211,9 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)bool sg = (bo->preferred_domains ==AMDGPU_GEM_DOMAIN_CPU );if (bo->flags &AMDGPU_AMDKFD_USERPTR_BO ) { +if (no_system_mem_limit) +return ; +
Same as above. This results in incorrect accounting when the user changes the setting at runtime.
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +0
-0domain =AMDGPU_GEM_DOMAIN_CPU ; sg = false; }drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +2
-0@@ -717,6 +717,15 @@ MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption timeout in ms (1bool debug_evictions;module_param (debug_evictions, bool,0644 );MODULE_PARM_DESC (debug_evictions,"enable eviction debug messages (false = default)" ); + +/**
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption timeout in ms (1 · line 731
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +7
-0+* DOC: no_system_mem_limit(bool) +* Disable system memory limit, to support multiple process shared memory +*/ +bool no_system_mem_limit; +module_param (no_system_mem_limit, bool,0644 ); +MODULE_PARM_DESC (no_system_mem_limit,"disable system memory limit (false = default)" ); +#endif /**
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx