Skip to contentSkip to conversation

drm/amdkfd: Fix partial migrate issue

PATCH Superseded Emily Deng 4 participants 18 messages Jan 3, 2025 – Jan 8, 2025

Subsystems (3)

v1 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Emily Deng

author
#0 lore raw

PATCH 1/1 drm/amdkfd: Fix partial migrate issue

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +4 -4
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 			 struct migrate_vma *migrate, struct dma_fence **mfence,
 			 dma_addr_t *scratch, uint64_t ttm_res_offset)
 {
-	uint64_t npages = migrate->cpages;
+	uint64_t npages = migrate->npages;
 	struct amdgpu_device *adev = node->adev;
 	struct device *dev = adev->dev;
 	struct amdgpu_res_cursor cursor;
@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 		struct page *spage;
 
 		dst[i] = cursor.start + (j << PAGE_SHIFT);
-		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-		svm_migrate_get_vram_page(prange, migrate->dst[i]);
-		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
 		spage = migrate_pfn_to_page(migrate->src[i]);
 		if (spage && !is_zone_device_page(spage)) {
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 		} else {
 			j++;
 		}
+		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+		svm_migrate_get_vram_page(prange, migrate->dst[i]);
+		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 	}
 
 	r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
Signature · 1 line
2.36.1
maintainer
#1 lore raw
Quoted 8 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

Good catch. But I think it's still not quite right. See inline.

Quoted 18 lines
---
  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
  1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  			 struct migrate_vma *migrate, struct dma_fence **mfence,
  			 dma_addr_t *scratch, uint64_t ttm_res_offset)
  {
-	uint64_t npages = migrate->cpages;
+	uint64_t npages = migrate->npages;
  	struct amdgpu_device *adev = node->adev;
  	struct device *dev = adev->dev;
  	struct amdgpu_res_cursor cursor;

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 304

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  		struct page *spage;
 
  		dst[i] = cursor.start + (j << PAGE_SHIFT);
-		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-		svm_migrate_get_vram_page(prange, migrate->dst[i]);
-		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
  		spage = migrate_pfn_to_page(migrate->src[i]);
  		if (spage && !is_zone_device_page(spage)) {

We should also check (migrate->src[i] & MIGRATE_PFN_MIGRATE) to catch only the pages that are ready to migrate (i.e. not already in device memory and not pinned by someone else).

Regards,
  Felix

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  		} else {
  			j++;
  		}
+		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+		svm_migrate_get_vram_page(prange, migrate->dst[i]);
+		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
  	}
 
  	r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
#2 lore raw

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 52 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Saturday, January 4, 2025 7:10 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

Quoted 9 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate- dst[i].

Good catch. But I think it's still not quite right. See inline.

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct

Quoted 10 lines
---
  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
  1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
                      struct migrate_vma *migrate, struct dma_fence **mfence,
                      dma_addr_t *scratch, uint64_t ttm_res_offset)
  {
-    uint64_t npages = migrate->cpages;
+    uint64_t npages = migrate->npages;
     struct amdgpu_device *adev = node->adev;
     struct device *dev = adev->dev;
     struct amdgpu_res_cursor cursor;
@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 304

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
             struct page *spage;
 
             dst[i] = cursor.start + (j << PAGE_SHIFT);
-            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-            svm_migrate_get_vram_page(prange, migrate->dst[i]);
-            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
             spage = migrate_pfn_to_page(migrate->src[i]);
             if (spage && !is_zone_device_page(spage)) {
Quoted 7 lines

We should also check (migrate->src[i] & MIGRATE_PFN_MIGRATE) to catch only the pages that are ready to migrate (i.e. not already in device memory and not pinned by someone else).

Regards,
Felix
The code " if (spage && !is_zone_device_page(spage)) {" is already checked whether the page could be migrated or not. Because after called migrate_vma_setup, for those pages couldn't be migrated it will be set to NULL.

Emily Deng
Best Wishes

Quoted 12 lines

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct

Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
             } else {
                     j++;
             }
+            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+            svm_migrate_get_vram_page(prange, migrate->dst[i]);
+            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
     }
 
     r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
maintainer
#3 lore raw
Quoted 59 lines · Emily

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 57 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Saturday, January 4, 2025 7:10 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

Quoted 9 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate- dst[i].

Good catch. But I think it's still not quite right. See inline.

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct

Quoted 10 lines
---
   drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
   1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
                       struct migrate_vma *migrate, struct dma_fence **mfence,
                       dma_addr_t *scratch, uint64_t ttm_res_offset)
   {
-    uint64_t npages = migrate->cpages;
+    uint64_t npages = migrate->npages;
      struct amdgpu_device *adev = node->adev;
      struct device *dev = adev->dev;
      struct amdgpu_res_cursor cursor;
@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 304

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
              struct page *spage;
 
              dst[i] = cursor.start + (j << PAGE_SHIFT);
-            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-            svm_migrate_get_vram_page(prange, migrate->dst[i]);
-            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
              spage = migrate_pfn_to_page(migrate->src[i]);
              if (spage && !is_zone_device_page(spage)) {

We should also check (migrate->src[i] & MIGRATE_PFN_MIGRATE) to catch only the pages that are ready to migrate (i.e. not already in device memory and not pinned by someone else).

Regards,
Felix
The code " if (spage && !is_zone_device_page(spage)) {" is already checked whether the page could be migrated or not. Because after called migrate_vma_setup, for those pages couldn't be migrated it will be set to NULL.

The page may be non-NULL. If the MIGRATE_PFN_MIGRATE is not set, the page could not be collected for migration by migrate_vma_setup and it should not be migrated. See https://elixir.bootlin.com/linux/v6.12.6/source/mm/migrate_device.c#L470

Regards,
  Felix

Quoted 15 lines

Emily Deng
Best Wishes

Quoted 11 lines

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
              } else {
                      j++;
              }
+            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+            svm_migrate_get_vram_page(prange, migrate->dst[i]);
+            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
      }
 
      r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
#4 lore raw

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 71 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Tuesday, January 7, 2025 4:53 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

Quoted 60 lines · Emily

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 58 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Saturday, January 4, 2025 7:10 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

Quoted 9 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate- dst[i].

Good catch. But I think it's still not quite right. See inline.

Quoted 11 lines
---
   drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
   1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node,

struct

svm_range *prange,

Quoted 9 lines

struct migrate_vma *migrate, struct dma_fence **mfence, dma_addr_t *scratch, uint64_t ttm_res_offset)
{

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
-    uint64_t npages = migrate->cpages;
+    uint64_t npages = migrate->npages;
      struct amdgpu_device *adev = node->adev;
      struct device *dev = adev->dev;
      struct amdgpu_res_cursor cursor; @@ -299,9 +299,6 @@

svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 304

Quoted 9 lines
              struct page *spage;

              dst[i] = cursor.start + (j << PAGE_SHIFT);
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
-            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-            svm_migrate_get_vram_page(prange, migrate->dst[i]);
-            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
              spage = migrate_pfn_to_page(migrate->src[i]);
              if (spage && !is_zone_device_page(spage)) {

We should also check (migrate->src[i] & MIGRATE_PFN_MIGRATE) to catch only the pages that are ready to migrate (i.e. not already in device memory and not pinned by someone else).

Regards,
Felix
The code " if (spage && !is_zone_device_page(spage)) {" is already checked

whether the page could be migrated or not. Because after called migrate_vma_setup, for those pages couldn't be migrated it will be set to NULL.

Quoted 7 lines

The page may be non-NULL. If the MIGRATE_PFN_MIGRATE is not set, the page could not be collected for migration by migrate_vma_setup and it should not be migrated. See
https://elixir.bootlin.com/linux/v6.12.6/source/mm/migrate_device.c#L470

Regards,
Felix

Sorry, don't understand clearly. If a page couldn't be migrated, the src should be set NULL, if not, I think it is the migrate_vma_collect's bug. So could you point which code path it will not be NULL.

Quoted 18 lines
Quoted 18 lines

Emily Deng
Best Wishes

Quoted 13 lines
Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node,

struct

svm_range *prange,

Quoted 10 lines
              } else {
                      j++;
              }
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
+            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+            svm_migrate_get_vram_page(prange, migrate->dst[i]);
+            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
      }
 
      r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i -

j, j,

maintainer
#5 lore raw
Quoted 78 lines · Emily

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 76 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Tuesday, January 7, 2025 4:53 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

Quoted 60 lines · Emily

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 58 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Saturday, January 4, 2025 7:10 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

Quoted 9 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate- dst[i].

Good catch. But I think it's still not quite right. See inline.

Quoted 11 lines
---
    drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
    1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node,

struct

svm_range *prange,

Quoted 9 lines

struct migrate_vma *migrate, struct dma_fence **mfence, dma_addr_t *scratch, uint64_t ttm_res_offset)
{

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
-    uint64_t npages = migrate->cpages;
+    uint64_t npages = migrate->npages;
       struct amdgpu_device *adev = node->adev;
       struct device *dev = adev->dev;
       struct amdgpu_res_cursor cursor; @@ -299,9 +299,6 @@

svm_migrate_copy_to_vram(struct kfd_node *node, struct

svm_range *prange,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 304

Quoted 9 lines
               struct page *spage;

               dst[i] = cursor.start + (j << PAGE_SHIFT);
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
-            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-            svm_migrate_get_vram_page(prange, migrate->dst[i]);
-            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
               spage = migrate_pfn_to_page(migrate->src[i]);
               if (spage && !is_zone_device_page(spage)) {

We should also check (migrate->src[i] & MIGRATE_PFN_MIGRATE) to catch only the pages that are ready to migrate (i.e. not already in device memory and not pinned by someone else).

Regards,
Felix
The code " if (spage && !is_zone_device_page(spage)) {" is already checked

whether the page could be migrated or not. Because after called migrate_vma_setup, for those pages couldn't be migrated it will be set to NULL.

The page may be non-NULL. If the MIGRATE_PFN_MIGRATE is not set, the page could not be collected for migration by migrate_vma_setup and it should not be migrated. See
https://elixir.bootlin.com/linux/v6.12.6/source/mm/migrate_device.c#L470

Quoted 4 lines
Quoted 2 lines

Regards,
Felix

Sorry, don't understand clearly. If a page couldn't be migrated, the src should be set NULL, if not, I think it is the migrate_vma_collect's bug. So could you point which code path it will not be NULL.

Looks like I misread the documentation. I thought there could be cases where a valid page would have the MIGRATE_PFN_MIGRATE flag unset. But I can't find that in the code in mm/migrate_device.c. In the cases where a page cannot be collected, the src entry will be 0. The only case I see where the MIGRATE_PFN_MIGRATE flag is significant is in cases where host memory was not allocated yet (PFN is 0) but the migration can allocate 0-initialized device memory for it.

Regards,
  Felix

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines

Emily Deng
Best Wishes

Quoted 13 lines
Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node,

struct

svm_range *prange,

Quoted 10 lines
               } else {
                       j++;
               }
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
+            migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+            svm_migrate_get_vram_page(prange, migrate->dst[i]);
+            migrate->dst[i] = migrate_pfn(migrate->dst[i]);
       }
 
       r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i -

j, j,

#6 lore raw
In reply to Emily Deng
Quoted 17 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---
  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
  1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
index 4b275937d05e..5c96c2d425e3 100644

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 281

Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  			 struct migrate_vma *migrate, struct dma_fence **mfence,
  			 dma_addr_t *scratch, uint64_t ttm_res_offset)
  {
-	uint64_t npages = migrate->cpages;
+	uint64_t npages = migrate->npages;

I agree this part.

Quoted 14 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
  	struct amdgpu_device *adev = node->adev;
  	struct device *dev = adev->dev;
  	struct amdgpu_res_cursor cursor;
@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  		struct page *spage;
 
  		dst[i] = cursor.start + (j << PAGE_SHIFT);
-		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-		svm_migrate_get_vram_page(prange, migrate->dst[i]);
-		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
  		spage = migrate_pfn_to_page(migrate->src[i]);
  		if (spage && !is_zone_device_page(spage)) {

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 347

Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  		} else {
  			j++;
  		}
+		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+		svm_migrate_get_vram_page(prange, migrate->dst[i]);
+		migrate->dst[i] = migrate_pfn(migrate->dst[i]);

I think what current code misses here is that migrate->dst[i] should match migrate->src[i]: migrate->dst[i](vram page) got set for page that will be migrated from system ram, otherwise migrate->dst[i] should be zero. Your change makes migrates->dst[i] not set though its page has been migrated by svm_migrate_copy_memory_gart for the case that cpages != npages, because you set migrate->dst[i] at end of loop and use 'continue' after migration.

The page migration happens at svm_migrate_copy_memory_gart, not migrate_vma_pages that migrates struct page meta-data from source struct page to destination struct page and has mmu notification.

Regards

Xiaogang

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
  	}
 
  	r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
#7 lore raw

[AMD Official Use Only - AMD Internal Distribution Only]

From: Chen, Xiaogang <[email protected]>
Sent: Monday, January 6, 2025 11:27 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 1/2/2025 6:06 PM, Emily Deng wrote:

For partial migrate from ram to vram, the migrate->cpages is not

equal to migrate->npages, should use migrate->npages to check all needed

migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or

the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---

 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0

index 4b275937d05e..5c96c2d425e3 100644

--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

struct migrate_vma *migrate, struct dma_fence **mfence,

dma_addr_t *scratch, uint64_t ttm_res_offset)

{

- uint64_t npages = migrate->cpages;

+ uint64_t npages = migrate->npages;
I agree this part.

        struct amdgpu_device *adev = node->adev;

        struct device *dev = adev->dev;

        struct amdgpu_res_cursor cursor;

@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                struct page *spage;



                dst[i] = cursor.start + (j << PAGE_SHIFT);

- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

- svm_migrate_get_vram_page(prange, migrate->dst[i]);

- migrate->dst[i] = migrate_pfn(migrate->dst[i]);

                spage = migrate_pfn_to_page(migrate->src[i]);

                if (spage && !is_zone_device_page(spage)) {

@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                } else {

                        j++;

                }

+ migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

+ svm_migrate_get_vram_page(prange, migrate->dst[i]);

+ migrate->dst[i] = migrate_pfn(migrate->dst[i]);

I think what current code misses here is that migrate->dst[i] should match migrate->src[i]: migrate->dst[i](vram page) got set for page that will be migrated from system ram, otherwise migrate->dst[i] should be zero. Your change makes migrates->dst[i] not set though its page has been migrated by svm_migrate_copy_memory_gart for the case that cpages != npages, because you set migrate->dst[i] at end of loop and use 'continue' after migration.

The page migration happens at svm_migrate_copy_memory_gart, not migrate_vma_pages that migrates struct page meta-data from source struct page to destination struct page and has mmu notification.

Regards

Xiaogang

The migrate->dst[i] default value is 0. The purpose of the change is to match the migrate->dst[i] and migrate->src[i]. And for svm_migrate_copy_to_vram, it only will call svm_migrate_copy_memory_gart to copy those pages which need to be migrated, for those don’t need migrate pages, it won’t call svm_migrate_copy_memory_gart.

}

r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#8 lore raw
Quoted 93 lines · Emily

[AMD Official Use Only - AMD Internal Distribution Only]

*From:*Chen, Xiaogang <[email protected]>
*Sent:* Monday, January 6, 2025 11:27 PM
*To:* Deng, Emily <[email protected]>; [email protected] *Subject:* Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 1/2/2025 6:06 PM, Emily Deng wrote:

    For partial migrate from ram to vram, the migrate->cpages is not

    equal to migrate->npages, should use migrate->npages to check all needed

    migrate pages which could be copied or not.

    And only need to set those pages could be migrated to migrate->dst[i], or

    the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

    Signed-off-by: Emily Deng<[email protected]> <mailto:[email protected]>

    ---

      drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----

      1 file changed, 4 insertions(+), 4 deletions(-)

    diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

    index 4b275937d05e..5c96c2d425e3 100644

    --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

    +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

    @@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                              struct migrate_vma *migrate, struct dma_fence **mfence,

                              dma_addr_t *scratch, uint64_t ttm_res_offset)

      {

    -       uint64_t npages = migrate->cpages;

    +       uint64_t npages = migrate->npages;

I agree this part.

             struct amdgpu_device *adev = node->adev;

             struct device *dev = adev->dev;

             struct amdgpu_res_cursor cursor;

    @@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                     struct page *spage;

      

                     dst[i] = cursor.start + (j << PAGE_SHIFT);

    -               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

    -        svm_migrate_get_vram_page(prange, migrate->dst[i]);

    -               migrate->dst[i] = migrate_pfn(migrate->dst[i]);

      

                     spage = migrate_pfn_to_page(migrate->src[i]);

                     if (spage && !is_zone_device_page(spage)) {

    @@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                     } else {

                             j++;

                     }

    +               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

    +        svm_migrate_get_vram_page(prange, migrate->dst[i]);

    +               migrate->dst[i] = migrate_pfn(migrate->dst[i]);

I think what current code misses here is that migrate->dst[i] should match migrate->src[i]: migrate->dst[i](vram page) got set for page that will be migrated from system ram, otherwise migrate->dst[i] should be zero. Your change makes migrates->dst[i] not set though its page has been migrated by svm_migrate_copy_memory_gart for the case that cpages != npages, because you set migrate->dst[i] at end of loop and use 'continue' after migration.

Quoted 8 lines

The page migration happens at svm_migrate_copy_memory_gart, not migrate_vma_pages that migrates struct page meta-data from source struct page to destination struct page and has mmu notification.

Regards

Xiaogang

The migrate->dst[i] default value is 0. The purpose of the change is to match the migrate->dst[i] and migrate->src[i]. And for svm_migrate_copy_to_vram, it only will call svm_migrate_copy_memory_gart to copy those pages which need to be migrated, for those don’t need migrate pages, it won’t call svm_migrate_copy_memory_gart.

Yes, the issue here is we need match migrate->dst[i] and migrate->src[i]: migrate->dst[i] need be set only when its correspondent migrate->src[i] page will be migrated or the src page has dma address setup by dma_map_page. Then why not set migrate->dst[i] at same time when have dma_map_page for src page? That way will simplify the logic and not need use amdgpu_res_next to jump vram cursor.

Regards

Xiaogang

Quoted 6 lines

       }

        r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#9 lore raw

On 2025-01-07 10:50, Chen, Xiaogang wrote:

On 1/6/2025 8:02 PM, Deng, Emily wrote:
[AMD Official Use Only - AMD Internal Distribution Only]



From:Chen, Xiaogang <[email protected]>
Sent:Monday, January 6, 2025 11:27 PM
To:Deng, Emily <[email protected]>; [email protected]
Subject:Re: [PATCH] drm/amdkfd: Fix partial migrate issue


On 1/2/2025 6:06 PM, Emily Deng wrote:
For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.   And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].   Signed-off-by: Emily Deng <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)   diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 4b275937d05e..5c96c2d425e3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                         struct migrate_vma *migrate, struct dma_fence **mfence,                         dma_addr_t *scratch, uint64_t ttm_res_offset) { -       uint64_t npages = migrate->cpages; +       uint64_t npages = migrate->npages; I agree this part.

         struct amdgpu_device *adev = node->adev;        struct device *dev = adev->dev;        struct amdgpu_res_cursor cursor; @@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                struct page *spage;                 dst[i] = cursor.start + (j << PAGE_SHIFT); -               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); -       svm_migrate_get_vram_page(prange, migrate->dst[i]); -               migrate->dst[i] = migrate_pfn(migrate->dst[i]);                 spage = migrate_pfn_to_page(migrate->src[i]);                if (spage && !is_zone_device_page(spage)) { @@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                } else {                        j++;                } +               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); +       svm_migrate_get_vram_page(prange, migrate->dst[i]); +               migrate->dst[i] = migrate_pfn(migrate->dst[i]); I think what current code misses here is that migrate->dst[i] should match migrate->src[i]: migrate->dst[i](vram page) got set for page that will be migrated from system ram, otherwise migrate->dst[i] should be zero. Your change makes migrates->dst[i] not set though its page has been migrated by svm_migrate_copy_memory_gart for the case that cpages != npages, because you set migrate->dst[i] at end of loop and use 'continue' after migration. The page migration happens at svm_migrate_copy_memory_gart, not migrate_vma_pages that migrates struct page meta-data from source struct page to destination struct page and has mmu notification.
Regards
Xiaogang
The migrate->dst[i] default value is 0. The purpose of the change is to match the migrate->dst[i] and migrate->src[i]. And for svm_migrate_copy_to_vram, it only will call svm_migrate_copy_memory_gart to copy those pages which need to be migrated, for those don’t need migrate pages, it won’t call svm_migrate_copy_memory_gart.
Yes, the issue here is we need match migrate->dst[i] and migrate->src[i]: migrate->dst[i] need be set only when its correspondent migrate->src[i] page will be migrated or the src page has dma address setup by dma_map_page. Then why not set migrate->dst[i] at same time when have dma_map_page for src page?
yes, agree, I will reply the v2 patch with this change.
That way will simplify the logic and not need use amdgpu_res_next to jump vram cursor.

if spage is not contiguous or dst is the last page of cursor (vram address is not contiguous), we need setup sdma copy and then call amdgpu_res_next to update cursor->start.
Regards,
Philip

Regards
Xiaogang

         }         r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#10 lore raw
Quoted 107 lines · Philip Yang
Quoted 104 lines · Xiaogang
Quoted 101 lines · Emily

[AMD Official Use Only - AMD Internal Distribution Only]

*From:*Chen, Xiaogang <[email protected]>
*Sent:* Monday, January 6, 2025 11:27 PM
*To:* Deng, Emily <[email protected]>; [email protected] *Subject:* Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 1/2/2025 6:06 PM, Emily Deng wrote:

    For partial migrate from ram to vram, the migrate->cpages is not

    equal to migrate->npages, should use migrate->npages to check all needed

    migrate pages which could be copied or not.

    And only need to set those pages could be migrated to migrate->dst[i], or

    the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

    Signed-off-by: Emily Deng<[email protected]> <mailto:[email protected]>

    ---

      drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----

      1 file changed, 4 insertions(+), 4 deletions(-)

    diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

    index 4b275937d05e..5c96c2d425e3 100644

    --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

    +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

    @@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                              struct migrate_vma *migrate, struct dma_fence **mfence,

                              dma_addr_t *scratch, uint64_t ttm_res_offset)

      {

    -       uint64_t npages = migrate->cpages;

    +       uint64_t npages = migrate->npages;

I agree this part.

             struct amdgpu_device *adev = node->adev;

             struct device *dev = adev->dev;

             struct amdgpu_res_cursor cursor;

    @@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                     struct page *spage;

      

                     dst[i] = cursor.start + (j << PAGE_SHIFT);

    -               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

    -        svm_migrate_get_vram_page(prange, migrate->dst[i]);

    -               migrate->dst[i] = migrate_pfn(migrate->dst[i]);

      

                     spage = migrate_pfn_to_page(migrate->src[i]);

                     if (spage && !is_zone_device_page(spage)) {

    @@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                     } else {

                             j++;

                     }

    +               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

    +        svm_migrate_get_vram_page(prange, migrate->dst[i]);

    +               migrate->dst[i] = migrate_pfn(migrate->dst[i]);

I think what current code misses here is that migrate->dst[i] should match migrate->src[i]: migrate->dst[i](vram page) got set for page that will be migrated from system ram, otherwise migrate->dst[i] should be zero. Your change makes migrates->dst[i] not set though its page has been migrated by svm_migrate_copy_memory_gart for the case that cpages != npages, because you set migrate->dst[i] at end of loop and use 'continue' after migration.

The page migration happens at svm_migrate_copy_memory_gart, not migrate_vma_pages that migrates struct page meta-data from source struct page to destination struct page and has mmu notification.

Regards

Xiaogang

The migrate->dst[i] default value is 0. The purpose of the change is to match the migrate->dst[i] and migrate->src[i]. And for svm_migrate_copy_to_vram, it only will call svm_migrate_copy_memory_gart to copy those pages which need to be migrated, for those don’t need migrate pages, it won’t call svm_migrate_copy_memory_gart.

Quoted 8 lines
Quoted 2 lines

Yes, the issue here is we need match migrate->dst[i] and migrate->src[i]: migrate->dst[i] need be set only when its correspondent migrate->src[i] page will be migrated or the src page has dma address setup by dma_map_page. Then why not set migrate->dst[i] at same time when have dma_map_page for src page?

yes, agree, I will reply the v2 patch with this change.

Quoted 3 lines

That way will simplify the logic and not need use amdgpu_res_next to jump vram cursor.

if spage is not contiguous or dst is the last page of cursor (vram address is not contiguous), we need setup sdma copy and then call amdgpu_res_next to update cursor->start.

I think we only need use amdgpu_res_nex at one place where update migrate->dst[i] together with dma_map_page for src page, but need to remember the jump between two dma_map_page to adapt to cursor->start update.

Regards

Xiaogang

Quoted 14 lines

Regards,

Philip

Quoted 10 lines

Regards

Xiaogang

Quoted 6 lines

       }

        r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#11 lore raw
In reply to Emily Deng

On 2025-01-02 19:06, Emily Deng wrote:
For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not. And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i]. Signed-off-by: Emily Deng <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 4b275937d05e..5c96c2d425e3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, struct migrate_vma *migrate, struct dma_fence **mfence, dma_addr_t *scratch, uint64_t ttm_res_offset) { - uint64_t npages = migrate->cpages; + uint64_t npages = migrate->npages; As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect.

struct amdgpu_device *adev = node->adev; struct device *dev = adev->dev; struct amdgpu_res_cursor cursor; @@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, struct page *spage; dst[i] = cursor.start + (j << PAGE_SHIFT); - migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); - svm_migrate_get_vram_page(prange, migrate->dst[i]); - migrate->dst[i] = migrate_pfn(migrate->dst[i]); spage = migrate_pfn_to_page(migrate->src[i]); if (spage && !is_zone_device_page(spage)) { @@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, } else { j++; } + migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); + svm_migrate_get_vram_page(prange, migrate->dst[i]); + migrate->dst[i] = migrate_pfn(migrate->dst[i]); This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor. Please check this change, to add mpages accounting to break the loop earlier.

-       uint64_t npages = migrate->cpages;
+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;
@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
        amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                         npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                struct page *spage;

-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT); j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n", src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
(END)
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT); j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n", src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {

} r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#12 lore raw
Quoted 15 lines · Philip Yang
Quoted 12 lines · Emily Deng

For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---
  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
  1 file changed, 4 insertions(+), 4 deletions(-)
Quoted 13 lines

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 281

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
index 4b275937d05e..5c96c2d425e3 100644
@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  			 struct migrate_vma *migrate, struct dma_fence **mfence,
  			 dma_addr_t *scratch, uint64_t ttm_res_offset)
  {
-	uint64_t npages = migrate->cpages;
+	uint64_t npages = migrate->npages;

As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect.

The commit message is somehow confusing. It is not partial migration from page fault recovery. It is the case that migrate->cpages != migrate->npages after migrate_vma_setup due to some pages cannot be moved for some reasons. For that only part of src pages can be migrated. Ex: some system ram pages got locked by kernel. Usually this case does not happen during normal tests.

Regards

Xiaogang

Quoted 122 lines
Quoted 14 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -3
  	struct amdgpu_device *adev = node->adev;
  	struct device *dev = adev->dev;
  	struct amdgpu_res_cursor cursor;
@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  		struct page *spage;
 
  		dst[i] = cursor.start + (j << PAGE_SHIFT);
-		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-		svm_migrate_get_vram_page(prange, migrate->dst[i]);
-		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 
  		spage = migrate_pfn_to_page(migrate->src[i]);
  		if (spage && !is_zone_device_page(spage)) {

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, · line 347

Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -0
@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
  		} else {
  			j++;
  		}
+		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+		svm_migrate_get_vram_page(prange, migrate->dst[i]);
+		migrate->dst[i] = migrate_pfn(migrate->dst[i]);

This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor.

Please check this change, to add mpages accounting to break the loop earlier.

-       uint64_t npages = migrate->cpages;
+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;

(quoted diff) +1 -6
@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
 
         amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                          npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                 struct page *spage;
 
-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                 spage = migrate_pfn_to_page(migrate->src[i]);
                 if (spage && !is_zone_device_page(spage)) {

                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,

(quoted diff) +1 -0
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
                                                 mfence);
                                 if (r)
                                         goto out_free_vram_pages;
+                               mpages += j;
                                 amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                 j = 0;

                        } else {

(quoted diff) +5 -0
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
                 pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                      src[i] >> PAGE_SHIFT, page_to_pfn(spage));
 
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+

                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages
- 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i
- j,
                                                         dst + i - j, j + 1,

(quoted diff) +1 -0
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
 mfence);
                         if (r)
                                 goto out_free_vram_pages;
+                       mpages += j + 1;
                         amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                         j = 0;

                } else {
(END)

(quoted diff) +1 -0
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
                                                 mfence);
                                 if (r)
                                         goto out_free_vram_pages;
+                               mpages += j;
                                 amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                 j = 0;

                        } else {

(quoted diff) +5 -0
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
                 pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                      src[i] >> PAGE_SHIFT, page_to_pfn(spage));
 
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+

                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages
- 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i
- j,
                                                         dst + i - j, j + 1,

(quoted diff) +1 -0
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, 
 struct svm_range *prange,
 mfence);
                         if (r)
                                 goto out_free_vram_pages;
+                       mpages += j + 1;
                         amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                         j = 0;

                } else {

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0
  	}
 
  	r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
#13 lore raw

[AMD Official Use Only - AMD Internal Distribution Only]

From: Chen, Xiaogang <[email protected]>
Sent: Tuesday, January 7, 2025 8:20 AM
To: Yang, Philip <[email protected]>; Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 1/6/2025 5:50 PM, Philip Yang wrote:

On 2025-01-02 19:06, Emily Deng wrote:

For partial migrate from ram to vram, the migrate->cpages is not

equal to migrate->npages, should use migrate->npages to check all needed

migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or

the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---

 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0

index 4b275937d05e..5c96c2d425e3 100644

--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

struct migrate_vma *migrate, struct dma_fence **mfence,

dma_addr_t *scratch, uint64_t ttm_res_offset)

{

- uint64_t npages = migrate->cpages;

+ uint64_t npages = migrate->npages;

As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect.

The commit message is somehow confusing. It is not partial migration from page fault recovery. It is the case that migrate->cpages != migrate->npages after migrate_vma_setup due to some pages cannot be moved for some reasons. For that only part of src pages can be migrated. Ex: some system ram pages got locked by kernel. Usually this case does not happen during normal tests.

Regards

Xiaogang

You are right, issue happens while some system ram pages are swapped out and couldn’t be migrated. But it is only one reason. It has lots of reason that migrate->cpages != migrate->npages. Anyway, it is not full migrate, and just migrate partial pages. So, I think partial migrate will be more suitable for.

  struct amdgpu_device *adev = node->adev;

  struct device *dev = adev->dev;

struct amdgpu_res_cursor cursor;

@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

          struct page *spage;



          dst[i] = cursor.start + (j << PAGE_SHIFT);

- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

- svm_migrate_get_vram_page(prange, migrate->dst[i]);

- migrate->dst[i] = migrate_pfn(migrate->dst[i]);

          spage = migrate_pfn_to_page(migrate->src[i]);

          if (spage && !is_zone_device_page(spage)) {

@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

          } else {

                  j++;

          }

+ migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

+ svm_migrate_get_vram_page(prange, migrate->dst[i]);

+ migrate->dst[i] = migrate_pfn(migrate->dst[i]);

This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor.

Please check this change, to add mpages accounting to break the loop earlier.

- uint64_t npages = migrate->cpages;

+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;

@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

        amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                         npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                struct page *spage;
-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
(END)
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
  }

r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#14 lore raw
In reply to Philip Yang

[AMD Official Use Only - AMD Internal Distribution Only]

From: Yang, Philip <[email protected]>
Sent: Tuesday, January 7, 2025 7:51 AM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 2025-01-02 19:06, Emily Deng wrote:

For partial migrate from ram to vram, the migrate->cpages is not

equal to migrate->npages, should use migrate->npages to check all needed

migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or

the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---

 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0

index 4b275937d05e..5c96c2d425e3 100644

--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

struct migrate_vma *migrate, struct dma_fence **mfence,

dma_addr_t *scratch, uint64_t ttm_res_offset)

{

- uint64_t npages = migrate->cpages;

+ uint64_t npages = migrate->npages;

As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect.

        struct amdgpu_device *adev = node->adev;

        struct device *dev = adev->dev;

        struct amdgpu_res_cursor cursor;

@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                struct page *spage;



                dst[i] = cursor.start + (j << PAGE_SHIFT);

- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

- svm_migrate_get_vram_page(prange, migrate->dst[i]);

- migrate->dst[i] = migrate_pfn(migrate->dst[i]);

                spage = migrate_pfn_to_page(migrate->src[i]);

                if (spage && !is_zone_device_page(spage)) {

@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                } else {

                        j++;

                }

+ migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

+ svm_migrate_get_vram_page(prange, migrate->dst[i]);

+ migrate->dst[i] = migrate_pfn(migrate->dst[i]);

This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor.

Sorry, don’t understand what is the corner case? And the place you put has not much different with mine, as it doesn’t have any jump or continue in the code between your

code place and mine. But it has “if (r) goto out_free_vram_pages;” to handle the error case. For error case, I think the migrate->dst[i] also need to be set to null.

Please check this change, to add mpages accounting to break the loop earlier.

Good suggestion, I will add mpages in change.

Emily Deng
Best Wishes

- uint64_t npages = migrate->cpages;

+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;

@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

        amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                         npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                struct page *spage;
-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
(END)
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {



        }



        r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
#15 lore raw

On 2025-01-06 21:31, Deng, Emily wrote:
[AMD Official Use Only - AMD Internal Distribution Only]


From:Yang, Philip <[email protected]>
Sent:Tuesday, January 7, 2025 7:51 AM
To:Deng, Emily <[email protected]>; [email protected]
Subject:Re: [PATCH] drm/amdkfd: Fix partial migrate issue


On 2025-01-02 19:06, Emily Deng wrote:
For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.   And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].   Signed-off-by: Emily Deng <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)   diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 4b275937d05e..5c96c2d425e3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                         struct migrate_vma *migrate, struct dma_fence **mfence,                         dma_addr_t *scratch, uint64_t ttm_res_offset) { -       uint64_t npages = migrate->cpages; +       uint64_t npages = migrate->npages; As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect. struct amdgpu_device *adev = node->adev;        struct device *dev = adev->dev;        struct amdgpu_res_cursor cursor; @@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                struct page *spage;                 dst[i] = cursor.start + (j << PAGE_SHIFT); -               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); -       svm_migrate_get_vram_page(prange, migrate->dst[i]); -               migrate->dst[i] = migrate_pfn(migrate->dst[i]);                 spage = migrate_pfn_to_page(migrate->src[i]);                if (spage && !is_zone_device_page(spage)) { @@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                } else {                        j++;                } +               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); +       svm_migrate_get_vram_page(prange, migrate->dst[i]); +               migrate->dst[i] = migrate_pfn(migrate->dst[i]); This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor.
Sorry, don’t understand what is the corner case? And the place you put has not much different with mine, as it doesn’t have any jump or continue in the code between your
migrate->dst[i] should set before this if condition (not after if) to setup sdma copy migrate->src to migrate->dst, if condition is true for the case migrating to the last page of VRAM res cusor, as the VRAM physical address is not contiguous.

        if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
Regards,
Philip

code place and mine. But it has “if (r) goto out_free_vram_pages;” to handle the error case. For error case, I think the migrate->dst[i] also need to be set to null. Please check this change, to add mpages accounting to break the loop earlier.
Good suggestion, I will add mpages in change.

Emily Deng
Best Wishes


-       uint64_t npages = migrate->cpages;
+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;
@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

        amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                         npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                struct page *spage;

-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));

+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
(END)
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));

+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
         }         r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,

#16 lore raw

[AMD Official Use Only - AMD Internal Distribution Only]

From: Yang, Philip <[email protected]>
Sent: Tuesday, January 7, 2025 10:04 PM
To: Deng, Emily <[email protected]>; Yang, Philip <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 2025-01-06 21:31, Deng, Emily wrote:

[AMD Official Use Only - AMD Internal Distribution Only]

From: Yang, Philip <[email protected]><mailto:[email protected]>
Sent: Tuesday, January 7, 2025 7:51 AM
To: Deng, Emily <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]>
Subject: Re: [PATCH] drm/amdkfd: Fix partial migrate issue

On 2025-01-02 19:06, Emily Deng wrote:

For partial migrate from ram to vram, the migrate->cpages is not

equal to migrate->npages, should use migrate->npages to check all needed

migrate pages which could be copied or not.

And only need to set those pages could be migrated to migrate->dst[i], or

the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].

---

 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +0 -0

index 4b275937d05e..5c96c2d425e3 100644

--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

@@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

struct migrate_vma *migrate, struct dma_fence **mfence,

dma_addr_t *scratch, uint64_t ttm_res_offset)

{

- uint64_t npages = migrate->cpages;

+ uint64_t npages = migrate->npages;

As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect.

        struct amdgpu_device *adev = node->adev;

        struct device *dev = adev->dev;

        struct amdgpu_res_cursor cursor;

@@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                struct page *spage;



                dst[i] = cursor.start + (j << PAGE_SHIFT);

- migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

- svm_migrate_get_vram_page(prange, migrate->dst[i]);

- migrate->dst[i] = migrate_pfn(migrate->dst[i]);

                spage = migrate_pfn_to_page(migrate->src[i]);

                if (spage && !is_zone_device_page(spage)) {

@@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

                } else {

                        j++;

                }

+ migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);

+ svm_migrate_get_vram_page(prange, migrate->dst[i]);

+ migrate->dst[i] = migrate_pfn(migrate->dst[i]);

This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor.

Sorry, don’t understand what is the corner case? And the place you put has not much different with mine, as it doesn’t have any jump or continue in the code between your
migrate->dst[i] should set before this if condition (not after if) to setup sdma copy migrate->src to migrate->dst, if condition is true for the case migrating to the last page of VRAM res cusor, as the VRAM physical address is not contiguous.

        if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {

Regards,

Philip

Maybe I am wrong, but I couldn’t find anywhere in svm_migrate_copy_memory_gart will use migrate->dst[i], it uses the dst[i] directly. So, I think it will have no effect. The reason why I put it to the tail is because if have some exceptions, for example “if (r) goto out_free_vram_pages;”, for this case, I think we need to treat the copy as fail, and couldn’t migrate these pages, so also couldn’t set the migrate->dst[i].

Emily Deng
Best Wishes

code place and mine. But it has “if (r) goto out_free_vram_pages;” to handle the error case. For error case, I think the migrate->dst[i] also need to be set to null.

Please check this change, to add mpages accounting to break the loop earlier.

Good suggestion, I will add mpages in change.

Emily Deng
Best Wishes

- uint64_t npages = migrate->cpages;

+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;

@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

        amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                         npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                struct page *spage;
-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
(END)
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {



        }



        r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
#17 lore raw

On 2025-01-07 19:31, Deng, Emily wrote:
[AMD Official Use Only - AMD Internal Distribution Only]



From:Yang, Philip <[email protected]>
Sent:Tuesday, January 7, 2025 10:04 PM
To:Deng, Emily <[email protected]>; Yang, Philip <[email protected]>; [email protected]
Subject:Re: [PATCH] drm/amdkfd: Fix partial migrate issue


On 2025-01-06 21:31, Deng, Emily wrote:
[AMD Official Use Only - AMD Internal Distribution Only]


From:Yang, Philip <[email protected]>
Sent:Tuesday, January 7, 2025 7:51 AM
To:Deng, Emily <[email protected]>; [email protected]
Subject:Re: [PATCH] drm/amdkfd: Fix partial migrate issue


On 2025-01-02 19:06, Emily Deng wrote:
For partial migrate from ram to vram, the migrate->cpages is not equal to migrate->npages, should use migrate->npages to check all needed migrate pages which could be copied or not.   And only need to set those pages could be migrated to migrate->dst[i], or the migrate_vma_pages will migrate the wrong pages based on the migrate->dst[i].   Signed-off-by: Emily Deng <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)   diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 4b275937d05e..5c96c2d425e3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -278,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                         struct migrate_vma *migrate, struct dma_fence **mfence,                         dma_addr_t *scratch, uint64_t ttm_res_offset) { -       uint64_t npages = migrate->cpages; +       uint64_t npages = migrate->npages; As partial migration size is based on prange granularity, by default 2MB, maybe always migrate->cpages equal to migrate->npages, that's why we didn't trigger this bug. Wondering how do you catch this bug? This bug will leak svm_bo too, as svm_migrate_get_vram_page reference counter is incorrect. struct amdgpu_device *adev = node->adev;        struct device *dev = adev->dev;        struct amdgpu_res_cursor cursor; @@ -299,9 +299,6 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                struct page *spage;                 dst[i] = cursor.start + (j << PAGE_SHIFT); -               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); -        svm_migrate_get_vram_page(prange, migrate->dst[i]); -               migrate->dst[i] = migrate_pfn(migrate->dst[i]);                 spage = migrate_pfn_to_page(migrate->src[i]);                if (spage && !is_zone_device_page(spage)) { @@ -345,6 +342,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,                } else {                        j++;                } +               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); +        svm_migrate_get_vram_page(prange, migrate->dst[i]); +               migrate->dst[i] = migrate_pfn(migrate->dst[i]); This should move forward, to handle the corner case to migrate 1 page to the last page of VRAM res cursor.
Sorry, don’t understand what is the corner case? And the place you put has not much different with mine, as it doesn’t have any jump or continue in the code between your
migrate->dst[i] should set before this if condition (not after if) to setup sdma copy migrate->src to migrate->dst, if condition is true for the case migrating to the last page of VRAM res cusor, as the VRAM physical address is not contiguous.
        if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
Regards,
Philip
Maybe I am wrong, but I couldn’t find anywhere in svm_migrate_copy_memory_gart will use migrate->dst[i], it uses the dst[i] directly. So, I think it will have no effect. The reason why I put it to the tail is because if have some exceptions, for example “if (r) goto out_free_vram_pages;”, for this case, I think we need to treat the copy as fail, and couldn’t migrate these pages, so also couldn’t set the migrate->dst[i].
I missed your change to always set the dst[i] at beginning of the each loop, you are right, this works too. dst[i] is vram physical address for sdma copy, migrate->dst[i] is pfn plus flags for hmm migrate helper, I think it is better to keep/move them together.
out_free_vram_pages path is incorrect now, I will put fix in v2 patch review.
Regards,
Philip


Emily Deng
Best Wishes


code place and mine. But it has “if (r) goto out_free_vram_pages;” to handle the error case. For error case, I think the migrate->dst[i] also need to be set to null. Please check this change, to add mpages accounting to break the loop earlier.
Good suggestion, I will add mpages in change.

Emily Deng
Best Wishes


-       uint64_t npages = migrate->cpages;
+       uint64_t npages = migrate->npages;
        struct amdgpu_device *adev = node->adev;
        struct device *dev = adev->dev;
        struct amdgpu_res_cursor cursor;
+       uint64_t mpages = 0;
        dma_addr_t *src;
        uint64_t *dst;
        uint64_t i, j;
@@ -295,14 +296,9 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

        amdgpu_res_first(prange->ttm_res, ttm_res_offset,
                         npages << PAGE_SHIFT, &cursor);
-       for (i = j = 0; i < npages; i++) {
+       for (i = j = 0; i < npages && mpages < migrate->cpages; i++) {
                struct page *spage;

-               dst[i] = cursor.start + (j << PAGE_SHIFT);
-               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-               svm_migrate_get_vram_page(prange, migrate->dst[i]);
-               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
-
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));

+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
(END)
@@ -322,6 +318,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                mfence);
                                if (r)
                                        goto out_free_vram_pages;
+                               mpages += j;
                                amdgpu_res_next(&cursor, (j + 1) << PAGE_SHIFT);
                                j = 0;
                        } else {
@@ -333,6 +330,11 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
                                     src[i] >> PAGE_SHIFT, page_to_pfn(spage));

+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
                        r = svm_migrate_copy_memory_gart(adev, src + i - j,
                                                         dst + i - j, j + 1,
@@ -340,6 +342,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
                                                         mfence);
                        if (r)
                                goto out_free_vram_pages;
+                       mpages += j + 1;
                        amdgpu_res_next(&cursor, (j + 1) * PAGE_SIZE);
                        j = 0;
                } else {
         }         r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,