Skip to contentSkip to conversation

drm/amdgpu: Correct amdgpu_amdkfd_total_mem_size calculation

PATCH In review Philip Yang 3 participants 4 messages Oct 4, 2022

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: Correct amdgpu_amdkfd_total_mem_size calculation

amdkfd_total_mem_size is the size of total GPUs vram plus system memory to estimate page tables memory usage and leave enough VRAM room for page tables allocation.

Calculate amdkfd_total_mem_size in amdgpu_amdkfd_device_probe is incorrect because adev->gmc.real_vram_size is still 0 called from amdgpu_device_ip_early_init. Move the calculation
to amdgpu_amdkfd_device_init to get the correct VRAM size.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -3
index 9e98f3866edc..049d192c7cdf 100644
@@ -75,9 +75,6 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
 		return;
 
 	adev->kfd.dev = kgd2kfd_probe(adev, vf);
-
-	if (adev->kfd.dev)
-		amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
 }
 
 /**
@@ -201,6 +198,8 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
 		adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev,
 						adev_to_drm(adev), &gpu_resources);
 
+		amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
+
 		INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work);
 	}
 }
Signature · 1 line
2.35.1
maintainer Reviewed-by on v1 1/1
#1 lore raw
Quoted 10 lines · Philip Yang

amdkfd_total_mem_size is the size of total GPUs vram plus system memory to estimate page tables memory usage and leave enough VRAM room for page tables allocation.

Calculate amdkfd_total_mem_size in amdgpu_amdkfd_device_probe is incorrect because adev->gmc.real_vram_size is still 0 called from amdgpu_device_ip_early_init. Move the calculation
to amdgpu_amdkfd_device_init to get the correct VRAM size.

Semi-related to this, there should probably be a reverse calculation in amdgpu_amdkfd_device_fini_sw to support hot-unplugging GPUs.

Regards,
  Felix

Quoted 27 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 5 ++---
  1 file changed, 2 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -3
index 9e98f3866edc..049d192c7cdf 100644
@@ -75,9 +75,6 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
  		return;
 
  	adev->kfd.dev = kgd2kfd_probe(adev, vf);
-
-	if (adev->kfd.dev)
-		amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
  }
 
  /**
@@ -201,6 +198,8 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
  		adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev,
  						adev_to_drm(adev), &gpu_resources);
 
+		amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
+
  		INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work);
  	}
  }
#2 lore raw
Quoted 10 lines · Felix Kuehling
Quoted 9 lines · Philip Yang

amdkfd_total_mem_size is the size of total GPUs vram plus system memory to estimate page tables memory usage and leave enough VRAM room for page tables allocation.

Calculate amdkfd_total_mem_size in amdgpu_amdkfd_device_probe is incorrect because adev->gmc.real_vram_size is still 0 called from amdgpu_device_ip_early_init. Move the calculation
to amdgpu_amdkfd_device_init to get the correct VRAM size.

Quoted 5 lines
Quoted 1 line

Semi-related to this, there should probably be a reverse calculation in amdgpu_amdkfd_device_fini_sw to support hot-unplugging GPUs.

I will add the reverse calculation then submit.

Regards,

Philip

Quoted 31 lines

Regards,
  Felix

Quoted 26 lines

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 5 ++---

  1 file changed, 2 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -3
index 9e98f3866edc..049d192c7cdf 100644
@@ -75,9 +75,6 @@ void amdgpu_amdkfd_device_probe(struct 
 amdgpu_device *adev)
           return;
         adev->kfd.dev = kgd2kfd_probe(adev, vf);
-
-    if (adev->kfd.dev)
-        amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
   }
     /**
@@ -201,6 +198,8 @@ void amdgpu_amdkfd_device_init(struct 
 amdgpu_device *adev)
           adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev,
                           adev_to_drm(adev), &gpu_resources);
   +        amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
+
           INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work);
       }

  }

maintainer
#3 lore raw

I'd prefer a separate patch and code review for the fini-case, because that addresses a different (potential) problem.

Thanks,
  Felix

Quoted 55 lines · Philip Yang
Quoted 10 lines · Felix Kuehling
Quoted 9 lines · Philip Yang

amdkfd_total_mem_size is the size of total GPUs vram plus system memory to estimate page tables memory usage and leave enough VRAM room for page tables allocation.

Calculate amdkfd_total_mem_size in amdgpu_amdkfd_device_probe is incorrect because adev->gmc.real_vram_size is still 0 called from amdgpu_device_ip_early_init. Move the calculation
to amdgpu_amdkfd_device_init to get the correct VRAM size.

Quoted 5 lines
Quoted 1 line

Semi-related to this, there should probably be a reverse calculation in amdgpu_amdkfd_device_fini_sw to support hot-unplugging GPUs.

I will add the reverse calculation then submit.

Regards,

Philip

Quoted 31 lines

Regards,
  Felix

Quoted 26 lines

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 5 ++---

  1 file changed, 2 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -3
index 9e98f3866edc..049d192c7cdf 100644
@@ -75,9 +75,6 @@ void amdgpu_amdkfd_device_probe(struct 
 amdgpu_device *adev)
           return;
         adev->kfd.dev = kgd2kfd_probe(adev, vf);
-
-    if (adev->kfd.dev)
-        amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
   }
     /**
@@ -201,6 +198,8 @@ void amdgpu_amdkfd_device_init(struct 
 amdgpu_device *adev)
           adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev,
                           adev_to_drm(adev), &gpu_resources);
   +        amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
+
           INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work);
       }

  }