Skip to contentSkip to conversation

drm/amdgpu: Allow TTM to evict svm bo from same process

PATCH Superseded Philip Yang 3 participants 7 messages Jul 25, 2022

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 drm/amdgpu: Allow TTM to evict svm bo from same process

To support SVM range VRAM overcommitment, TTM should be able to evict svm bo of same process to system memory, to get space to alloc new svm bo.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +6 -3
index 1d0dbff87d3f..e8bb32f4ca14 100644
@@ -159,11 +159,14 @@ static void amdkfd_fence_release(struct dma_fence *f)
 }
 
 /**
- * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
- *  if same return TRUE else return FALSE.
+ * amdkfd_fence_check_mm
  *
  * @f: [IN] fence
  * @mm: [IN] mm that needs to be verified
+ *
+ * Check if @mm is same as that of the fence @f, if same return TRUE else
+ * return FALSE.
+ * For svm bo, which support vram overcommitment, always return FALSE.
  */
 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
 {
@@ -171,7 +174,7 @@ bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
 
 	if (!fence)
 		return false;
-	else if (fence->mm == mm)
+	else if (fence->mm == mm  && !fence->svm_bo)
 		return true;
 
 	return false;
Signature · 1 line
2.35.1
#1 lore raw

PATCH 2/3 drm/amdkfd: Set svm range max pages

This will be used to split giant svm range into smaller ranges, to support VRAM overcommitment by giant range and improve GPU retry fault recover on giant range.

---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  2 ++
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 15 +++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h     |  3 +++
 3 files changed, 20 insertions(+)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +2 -0
index 9667015a6cbc..b1f87aa6138b 100644
@@ -1019,6 +1019,8 @@ int svm_migrate_init(struct amdgpu_device *adev)
 
 	amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
 
+	svm_range_set_max_pages(adev);
+
 	pr_info("HMM registered %ldMB device memory\n", size >> 20);
 
 	return 0;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +15 -0
index b592aee6d9d6..cf9565ddddf8 100644
@@ -46,6 +46,11 @@
  */
 #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING	(2UL * NSEC_PER_MSEC)
 
+/* Giant svm range split into smaller ranges based on this, it is decided using
+ * minimum of all dGPU/APU 1/32 VRAM size, between 2MB to 1GB and align to 2MB.
+ */
+uint64_t max_svm_range_pages;
+
 struct criu_svm_metadata {
 	struct list_head list;
 	struct kfd_criu_svm_range_priv_data data;
@@ -1869,6 +1874,16 @@ static struct svm_range *svm_range_clone(struct svm_range *old)
 
 	return new;
 }
+__init void svm_range_set_max_pages(struct amdgpu_device *adev)
+{
+	uint64_t pages;
+
+	/* 1/32 VRAM size in pages */
+	pages = adev->gmc.real_vram_size >> 17;
+	pages = clamp(pages, 1ULL << 9, 1ULL << 18);
+	max_svm_range_pages = min_not_zero(max_svm_range_pages, pages);
+	max_svm_range_pages = ALIGN(max_svm_range_pages, 1ULL << 9);
+}
 
 /**
  * svm_range_add - add svm range and handle overlap
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +3 -0
index eab7f6d3b13c..346a41bf8dbf 100644
@@ -204,6 +204,9 @@ void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_s
 #define KFD_IS_SVM_API_SUPPORTED(dev) ((dev)->pgmap.type != 0)
 
 void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
+
+__init void svm_range_set_max_pages(struct amdgpu_device *adev);
+
 #else
 
 struct kfd_process;
Signature · 1 line
2.35.1
maintainer
#2 lore raw

Re: [PATCH 2/3] drm/amdkfd: Set svm range max pages

Am 2022-07-25 um 08:23 schrieb Philip Yang:

Quoted 41 lines

This will be used to split giant svm range into smaller ranges, to support VRAM overcommitment by giant range and improve GPU retry fault recover on giant range.

---
  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  2 ++
  drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 15 +++++++++++++++
  drivers/gpu/drm/amd/amdkfd/kfd_svm.h     |  3 +++
  3 files changed, 20 insertions(+)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +2 -0
index 9667015a6cbc..b1f87aa6138b 100644
@@ -1019,6 +1019,8 @@ int svm_migrate_init(struct amdgpu_device *adev)
 
  	amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
 
+	svm_range_set_max_pages(adev);
+
  	pr_info("HMM registered %ldMB device memory\n", size >> 20);
 
  	return 0;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +5 -0
index b592aee6d9d6..cf9565ddddf8 100644
@@ -46,6 +46,11 @@
   */
  #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING	(2UL * NSEC_PER_MSEC)
 
+/* Giant svm range split into smaller ranges based on this, it is decided using
+ * minimum of all dGPU/APU 1/32 VRAM size, between 2MB to 1GB and align to 2MB.
+ */
+uint64_t max_svm_range_pages;
+
  struct criu_svm_metadata {
  	struct list_head list;
  	struct kfd_criu_svm_range_priv_data data;

drivers/gpu/drm/amd/amdkfd/kfd_svm.c static struct svm_range *svm_range_clone(struct svm_range *old) · line 1877

Quoted 4 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +1 -0
@@ -1869,6 +1874,16 @@ static struct svm_range *svm_range_clone(struct svm_range *old)
 
  	return new;
  }
+__init void svm_range_set_max_pages(struct amdgpu_device *adev)

Why is this marked as __init? This can run much later than module init.

drivers/gpu/drm/amd/amdkfd/kfd_svm.c static struct svm_range *svm_range_clone(struct svm_range *old) · line 1885

Quoted 8 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +8 -0
+{
+	uint64_t pages;
+
+	/* 1/32 VRAM size in pages */
+	pages = adev->gmc.real_vram_size >> 17;
+	pages = clamp(pages, 1ULL << 9, 1ULL << 18);
+	max_svm_range_pages = min_not_zero(max_svm_range_pages, pages);
+	max_svm_range_pages = ALIGN(max_svm_range_pages, 1ULL << 9);

I'd recommend updating max_svm_range_pages with a single WRITE_ONCE to avoid race conditions with GPU hot-plug.

Regards,
  Felix

Quoted 18 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +1 -0
+}
 
  /**
   * svm_range_add - add svm range and handle overlap
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +3 -0
index eab7f6d3b13c..346a41bf8dbf 100644
@@ -204,6 +204,9 @@ void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_s
  #define KFD_IS_SVM_API_SUPPORTED(dev) ((dev)->pgmap.type != 0)
 
  void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
+
+__init void svm_range_set_max_pages(struct amdgpu_device *adev);
+
  #else
 
  struct kfd_process;
#3 lore raw

Re: [PATCH 2/3] drm/amdkfd: Set svm range max pages

On 2022-07-25 10:34, Felix Kuehling wrote:
Am 2022-07-25 um 08:23 schrieb Philip Yang:
This will be used to split giant svm range into smaller ranges, to support VRAM overcommitment by giant range and improve GPU retry fault recover on giant range.

---
  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c |  2 ++ drivers/gpu/drm/amd/amdkfd/kfd_svm.c     | 15 +++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_svm.h     |  3 +++

  3 files changed, 20 insertions(+)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
index 9667015a6cbc..b1f87aa6138b 100644
@@ -1019,6 +1019,8 @@ int svm_migrate_init(struct amdgpu_device *adev)

        amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size)); +    svm_range_set_max_pages(adev);
+
      pr_info("HMM registered %ldMB device memory\n", size >> 20); return 0;

drivers/gpu/drm/amd/amdkfd/kfd_svm.c +0 -0
index b592aee6d9d6..cf9565ddddf8 100644
@@ -46,6 +46,11 @@

   */
  #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING    (2UL * NSEC_PER_MSEC) +/* Giant svm range split into smaller ranges based on this, it is decided using
+ * minimum of all dGPU/APU 1/32 VRAM size, between 2MB to 1GB and align to 2MB.
+ */
+uint64_t max_svm_range_pages;
+
  struct criu_svm_metadata {
      struct list_head list;
      struct kfd_criu_svm_range_priv_data data;
@@ -1869,6 +1874,16 @@ static struct svm_range *svm_range_clone(struct svm_range *old) return new;
  }
+__init void svm_range_set_max_pages(struct amdgpu_device *adev)

Why is this marked as __init? This can run much later than module init.

Thanks, this was called in amdgpu_amdkfd_init from module init, forget to remove __init after calling from svm_migrate_init, which is called from pci_probe.
Regards,
Philip

+{
+    uint64_t pages;
+
+    /* 1/32 VRAM size in pages */
+    pages = adev->gmc.real_vram_size >> 17;
+    pages = clamp(pages, 1ULL << 9, 1ULL << 18);
+    max_svm_range_pages = min_not_zero(max_svm_range_pages, pages); +    max_svm_range_pages = ALIGN(max_svm_range_pages, 1ULL << 9);

I'd recommend updating max_svm_range_pages with a single WRITE_ONCE to avoid race conditions with GPU hot-plug.

Regards,
  Felix

+}
    /**
   * svm_range_add - add svm range and handle overlap

drivers/gpu/drm/amd/amdkfd/kfd_svm.h +0 -0
index eab7f6d3b13c..346a41bf8dbf 100644
@@ -204,6 +204,9 @@ void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_s

  #define KFD_IS_SVM_API_SUPPORTED(dev) ((dev)->pgmap.type != 0) void svm_range_bo_unref_async(struct svm_range_bo *svm_bo); +
+__init void svm_range_set_max_pages(struct amdgpu_device *adev); +
  #else
    struct kfd_process;

#4 lore raw

PATCH 3/3 drm/amdkfd: Split giant svm range

In reply to Philip Yang

Giant svm range split to smaller ranges, align the range start address to max svm range pages to improve MMU TLB usage.

---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 52 +++++++++++++++++++---------
 1 file changed, 36 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +36 -16
index cf9565ddddf8..044bb99f88ea 100644
@@ -1885,6 +1885,37 @@ __init void svm_range_set_max_pages(struct amdgpu_device *adev)
 	max_svm_range_pages = ALIGN(max_svm_range_pages, 1ULL << 9);
 }
 
+static int
+__svm_range_add(struct svm_range_list *svms, uint64_t start, uint64_t last,
+	       struct list_head *insert_list, struct list_head *update_list)
+{
+	struct svm_range *prange;
+	uint64_t l;
+
+	pr_debug("max_svm_range_pages 0x%llx adding [0x%llx 0x%llx]\n",
+		 max_svm_range_pages, start, last);
+
+	while (last >= start) {
+		if (last - start + 1 > max_svm_range_pages) {
+			if (start % max_svm_range_pages)
+				l = ALIGN(start, max_svm_range_pages) - 1;
+			else
+				l = start + max_svm_range_pages - 1;
+		} else {
+			l = last;
+		}
+
+		prange = svm_range_new(svms, start, l);
+		if (!prange)
+			return -ENOMEM;
+		list_add(&prange->list, insert_list);
+		list_add(&prange->update_list, update_list);
+
+		start = l + 1;
+	}
+	return 0;
+}
+
 /**
  * svm_range_add - add svm range and handle overlap
  * @p: the range add to this process svms
@@ -1987,14 +2018,10 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
 
 		/* insert a new node if needed */
 		if (node->start > start) {
-			prange = svm_range_new(svms, start, node->start - 1);
-			if (!prange) {
-				r = -ENOMEM;
+			r = __svm_range_add(svms, start, node->start - 1,
+					    insert_list, update_list);
+			if (r)
 				goto out;
-			}
-
-			list_add(&prange->list, insert_list);
-			list_add(&prange->update_list, update_list);
 		}
 
 		node = next;
@@ -2002,15 +2029,8 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
 	}
 
 	/* add a final range at the end if needed */
-	if (start <= last) {
-		prange = svm_range_new(svms, start, last);
-		if (!prange) {
-			r = -ENOMEM;
-			goto out;
-		}
-		list_add(&prange->list, insert_list);
-		list_add(&prange->update_list, update_list);
-	}
+	if (start <= last)
+		r = __svm_range_add(svms, start, last, insert_list, update_list);
 
 out:
 	if (r)
Signature · 1 line
2.35.1
maintainer
#5 lore raw

Re: [PATCH 3/3] drm/amdkfd: Split giant svm range

Am 2022-07-25 um 08:23 schrieb Philip Yang:

Quoted 13 lines

Giant svm range split to smaller ranges, align the range start address to max svm range pages to improve MMU TLB usage.

---
  drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 52 +++++++++++++++++++---------
  1 file changed, 36 insertions(+), 16 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +0 -0
index cf9565ddddf8..044bb99f88ea 100644

drivers/gpu/drm/amd/amdkfd/kfd_svm.c __init void svm_range_set_max_pages(struct amdgpu_device *adev) · line 1890

Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +3 -0
@@ -1885,6 +1885,37 @@ __init void svm_range_set_max_pages(struct amdgpu_device *adev)
  	max_svm_range_pages = ALIGN(max_svm_range_pages, 1ULL << 9);
  }
 
+static int
+__svm_range_add(struct svm_range_list *svms, uint64_t start, uint64_t last,
+	       struct list_head *insert_list, struct list_head *update_list)

It would be nice to find a better name for this. Maybe svm_range_split_new. Maybe make the max size a parameter of the function for better clarity.

drivers/gpu/drm/amd/amdkfd/kfd_svm.c __init void svm_range_set_max_pages(struct amdgpu_device *adev) · line 1899

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +9 -0
+{
+	struct svm_range *prange;
+	uint64_t l;
+
+	pr_debug("max_svm_range_pages 0x%llx adding [0x%llx 0x%llx]\n",
+		 max_svm_range_pages, start, last);
+
+	while (last >= start) {
+		if (last - start + 1 > max_svm_range_pages) {

Use a single READ_ONCE in this function to read max_svm_range_pages into a local variable. This should avoid race conditions with GPU hotplug. If you make the max size a parameter of this function, that also works if the caller uses READ_ONCE.

drivers/gpu/drm/amd/amdkfd/kfd_svm.c __init void svm_range_set_max_pages(struct amdgpu_device *adev) · line 1905

Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +6 -0
+			if (start % max_svm_range_pages)
+				l = ALIGN(start, max_svm_range_pages) - 1;
+			else
+				l = start + max_svm_range_pages - 1;
+		} else {
+			l = last;

I think this whole if block could be written as

    l = min(last, ALIGN_DOWN(start + max_svm_range_pages, max_svm_range_pages) - 1);

Regards,
  Felix

Quoted 52 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +18 -16
+		}
+
+		prange = svm_range_new(svms, start, l);
+		if (!prange)
+			return -ENOMEM;
+		list_add(&prange->list, insert_list);
+		list_add(&prange->update_list, update_list);
+
+		start = l + 1;
+	}
+	return 0;
+}
+
  /**
   * svm_range_add - add svm range and handle overlap
   * @p: the range add to this process svms
@@ -1987,14 +2018,10 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
 
  		/* insert a new node if needed */
  		if (node->start > start) {
-			prange = svm_range_new(svms, start, node->start - 1);
-			if (!prange) {
-				r = -ENOMEM;
+			r = __svm_range_add(svms, start, node->start - 1,
+					    insert_list, update_list);
+			if (r)
  				goto out;
-			}
-
-			list_add(&prange->list, insert_list);
-			list_add(&prange->update_list, update_list);
  		}
 
  		node = next;
@@ -2002,15 +2029,8 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
  	}
 
  	/* add a final range at the end if needed */
-	if (start <= last) {
-		prange = svm_range_new(svms, start, last);
-		if (!prange) {
-			r = -ENOMEM;
-			goto out;
-		}
-		list_add(&prange->list, insert_list);
-		list_add(&prange->update_list, update_list);
-	}
+	if (start <= last)
+		r = __svm_range_add(svms, start, last, insert_list, update_list);
 
  out:
  	if (r)
maintainer Reviewed-by on v1 1/3
#6 lore raw
In reply to Philip Yang

Am 2022-07-25 um 08:23 schrieb Philip Yang:

Quoted 14 lines

To support SVM range VRAM overcommitment, TTM should be able to evict svm bo of same process to system memory, to get space to alloc new svm bo.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 9 ++++++---
  1 file changed, 6 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +0 -0
index 1d0dbff87d3f..e8bb32f4ca14 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c static void amdkfd_fence_release(struct dma_fence *f) · line 162

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +1 -2
@@ -159,11 +159,14 @@ static void amdkfd_fence_release(struct dma_fence *f)
  }
 
  /**
- * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
- *  if same return TRUE else return FALSE.
+ * amdkfd_fence_check_mm

I think we still need a brief description here. How about "Check whether to prevent eviction of @f by @mm".

With that fixed, the patch is

Quoted 19 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +5 -1
   *
   * @f: [IN] fence
   * @mm: [IN] mm that needs to be verified
+ *
+ * Check if @mm is same as that of the fence @f, if same return TRUE else
+ * return FALSE.
+ * For svm bo, which support vram overcommitment, always return FALSE.
   */
  bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
  {
@@ -171,7 +174,7 @@ bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
 
  	if (!fence)
  		return false;
-	else if (fence->mm == mm)
+	else if (fence->mm == mm  && !fence->svm_bo)
  		return true;
 
  	return false;