Skip to contentSkip to conversation

drm/amdgpu: Optimize recursion in amdgpu_vm_update_level

PATCH In review Felix Kuehling 3 participants 7 messages Jul 12, 2017 – Jul 13, 2017 5 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: Optimize recursion in amdgpu_vm_update_level

When lots of virtual address spaces is used, there can be thousands of page table BOs. amdgpu_vm_update_level iterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.

This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++----------------
 1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +55 -54
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 {
 	struct amdgpu_bo *shadow;
 	struct amdgpu_ring *ring = NULL;
-	uint64_t pd_addr, shadow_addr = 0;
+	uint64_t pd_addr = 0, shadow_addr = 0;
 	uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
 	uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
 	unsigned count = 0, pt_idx, ndw = 0;
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
 	WARN_ON(vm->use_cpu_for_update && shadow);
 	if (vm->use_cpu_for_update && !shadow) {
-		r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-		if (r)
-			return r;
-		r = amdgpu_vm_bo_wait(adev, parent->bo);
-		if (unlikely(r)) {
-			amdgpu_bo_kunmap(parent->bo);
-			return r;
-		}
+		/* Defer kmapping until it's actually needed. Some
+		 * PDBs may need no update at all
+		 */
 		params.func = amdgpu_vm_cpu_set_ptes;
+		params.ib = (void *)(long)-1;
 	} else {
-		if (shadow) {
-			r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-		ring = container_of(vm->entity.sched, struct amdgpu_ring,
-				    sched);
-
-		/* padding, etc. */
-		ndw = 64;
-
-		/* assume the worst case */
-		ndw += parent->last_entry_used * 6;
-
-		pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-		if (shadow) {
-			shadow_addr = amdgpu_bo_gpu_offset(shadow);
-			ndw *= 2;
-		} else {
-			shadow_addr = 0;
-		}
-
-		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-		if (r)
-			return r;
-
-		params.ib = &job->ibs[0];
+		/* Defer IB allocation until it's actually
+		 * needed. Some PDBs may need no update at all
+		 */
+		params.ib = NULL;
 		params.func = amdgpu_vm_do_set_ptes;
 	}
 
-
 	/* walk over the address space and update the directory */
 	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
 		struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 		if (bo == NULL)
 			continue;
 
-		if (bo->shadow) {
-			struct amdgpu_bo *pt_shadow = bo->shadow;
-
-			r = amdgpu_ttm_bind(&pt_shadow->tbo,
-					    &pt_shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-
-		pt = amdgpu_bo_gpu_offset(bo);
-		pt = amdgpu_gart_get_vm_pde(adev, pt);
+		pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);
 		if (parent->entries[pt_idx].addr == pt)
 			continue;
 
 		parent->entries[pt_idx].addr = pt;
 
+		if (!params.ib) {
+			if (shadow) {
+				r = amdgpu_ttm_bind(&shadow->tbo,
+						    &shadow->tbo.mem);
+				if (r)
+					return r;
+			}
+
+			ring = container_of(vm->entity.sched,
+					    struct amdgpu_ring, sched);
+
+			/* padding, etc. */
+			ndw = 64;
+
+			/* assume the worst case */
+			ndw += (parent->last_entry_used - pt_idx) * 6;
+
+			pd_addr = parent->bo->tbo.offset;
+
+			if (shadow) {
+				shadow_addr = shadow->tbo.offset;
+				ndw *= 2;
+			} else {
+				shadow_addr = 0;
+			}
+			r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+			if (r)
+				return r;
+
+			params.ib = &job->ibs[0];
+		} else if (!pd_addr) {
+			r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+			if (r)
+				return r;
+			r = amdgpu_vm_bo_wait(adev, parent->bo);
+			if (unlikely(r)) {
+				amdgpu_bo_kunmap(parent->bo);
+				return r;
+			}
+		}
+
 		pde = pd_addr + pt_idx * 8;
 		if (((last_pde + 8 * count) != pde) ||
 		    ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
 	if (params.func == amdgpu_vm_cpu_set_ptes)
 		amdgpu_bo_kunmap(parent->bo);
-	else if (params.ib->length_dw == 0) {
+	else if (params.ib && params.ib->length_dw == 0) {
 		amdgpu_job_free(job);
-	} else {
+	} else if (params.ib) {
 		amdgpu_ring_pad_ib(ring, params.ib);
 		amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
 				 AMDGPU_FENCE_OWNER_VM);
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
 		amdgpu_bo_fence(parent->bo, fence, true);
 		dma_fence_put(vm->last_dir_update);
-		vm->last_dir_update = dma_fence_get(fence);
-		dma_fence_put(fence);
+		vm->last_dir_update = fence;
 	}
 	/*
 	 * Recurse into the subdirectories. This recursion is harmless because
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
 		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
-		if (!entry->bo)
+		if (!entry->bo || !entry->entries)
 			continue;
 
 		r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
Signature · 6 lines
1.9.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1499831769-3764-1-git-send-email-Felix.Kuehling-5C7GfCeVMHo@public.gmane.org

#2 lore raw

nice improvement, just some nitpick inline, otherwise the patch is Reviewed-by: Chunming Zhou <[email protected]>.

Quoted 90 lines · Felix Kuehling

When lots of virtual address spaces is used, there can be thousands of page table BOs. amdgpu_vm_update_level iterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.

This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++----------------
  1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +9 -40
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  {
  	struct amdgpu_bo *shadow;
  	struct amdgpu_ring *ring = NULL;
-	uint64_t pd_addr, shadow_addr = 0;
+	uint64_t pd_addr = 0, shadow_addr = 0;
  	uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
  	uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
  	unsigned count = 0, pt_idx, ndw = 0;
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
  	WARN_ON(vm->use_cpu_for_update && shadow);
  	if (vm->use_cpu_for_update && !shadow) {
-		r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-		if (r)
-			return r;
-		r = amdgpu_vm_bo_wait(adev, parent->bo);
-		if (unlikely(r)) {
-			amdgpu_bo_kunmap(parent->bo);
-			return r;
-		}
+		/* Defer kmapping until it's actually needed. Some
+		 * PDBs may need no update at all
+		 */
  		params.func = amdgpu_vm_cpu_set_ptes;
+		params.ib = (void *)(long)-1;
  	} else {
-		if (shadow) {
-			r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-		ring = container_of(vm->entity.sched, struct amdgpu_ring,
-				    sched);
-
-		/* padding, etc. */
-		ndw = 64;
-
-		/* assume the worst case */
-		ndw += parent->last_entry_used * 6;
-
-		pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-		if (shadow) {
-			shadow_addr = amdgpu_bo_gpu_offset(shadow);
-			ndw *= 2;
-		} else {
-			shadow_addr = 0;
-		}
-
-		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-		if (r)
-			return r;
-
-		params.ib = &job->ibs[0];
+		/* Defer IB allocation until it's actually
+		 * needed. Some PDBs may need no update at all
+		 */
+		params.ib = NULL;
  		params.func = amdgpu_vm_do_set_ptes;
  	}
 
-
  	/* walk over the address space and update the directory */
  	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  		struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  		if (bo == NULL)
  			continue;
 
-		if (bo->shadow) {
-			struct amdgpu_bo *pt_shadow = bo->shadow;

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1068

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -9
-
-			r = amdgpu_ttm_bind(&pt_shadow->tbo,
-					    &pt_shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-
-		pt = amdgpu_bo_gpu_offset(bo);
-		pt = amdgpu_gart_get_vm_pde(adev, pt);
+		pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);

Getting offset from amdgpu_bo_gpu_offset(bo) is better I think, although we can assume pt bo is a known domain.

Quoted 62 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +43 -2
  		if (parent->entries[pt_idx].addr == pt)
  			continue;
 
  		parent->entries[pt_idx].addr = pt;
 
+		if (!params.ib) {
+			if (shadow) {
+				r = amdgpu_ttm_bind(&shadow->tbo,
+						    &shadow->tbo.mem);
+				if (r)
+					return r;
+			}
+
+			ring = container_of(vm->entity.sched,
+					    struct amdgpu_ring, sched);
+
+			/* padding, etc. */
+			ndw = 64;
+
+			/* assume the worst case */
+			ndw += (parent->last_entry_used - pt_idx) * 6;
+
+			pd_addr = parent->bo->tbo.offset;
+
+			if (shadow) {
+				shadow_addr = shadow->tbo.offset;
+				ndw *= 2;
+			} else {
+				shadow_addr = 0;
+			}
+			r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+			if (r)
+				return r;
+
+			params.ib = &job->ibs[0];
+		} else if (!pd_addr) {
+			r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+			if (r)
+				return r;
+			r = amdgpu_vm_bo_wait(adev, parent->bo);
+			if (unlikely(r)) {
+				amdgpu_bo_kunmap(parent->bo);
+				return r;
+			}
+		}
+
  		pde = pd_addr + pt_idx * 8;
  		if (((last_pde + 8 * count) != pde) ||
  		    ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
  	if (params.func == amdgpu_vm_cpu_set_ptes)
  		amdgpu_bo_kunmap(parent->bo);
-	else if (params.ib->length_dw == 0) {
+	else if (params.ib && params.ib->length_dw == 0) {
  		amdgpu_job_free(job);
-	} else {
+	} else if (params.ib) {
  		amdgpu_ring_pad_ib(ring, params.ib);
  		amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
  				 AMDGPU_FENCE_OWNER_VM);

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1171

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -2
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
  		amdgpu_bo_fence(parent->bo, fence, true);
  		dma_fence_put(vm->last_dir_update);
-		vm->last_dir_update = dma_fence_get(fence);
-		dma_fence_put(fence);
+		vm->last_dir_update = fence;

Here I prefer to keep previous code, the logic is more clear.:)

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1180

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
  	}
  	/*
  	 * Recurse into the subdirectories. This recursion is harmless because
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
-		if (!entry->bo)
+		if (!entry->bo || !entry->entries)

In fact, !entry->entries would be checked at the begin of this function in next loop, but either way is ok to me here.

Regards,
David Zhou

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1183

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
  			continue;
 
  		r = amdgpu_vm_update_level(adev, vm, entry, level + 1);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

#4 lore raw

Hi David,

Responses inline ...

Quoted 100 lines · On 17-07-12 12:30 AM, zhoucm1 wrote:

nice improvement, just some nitpick inline, otherwise the patch is

Quoted 96 lines · Felix Kuehling

When lots of virtual address spaces is used, there can be thousands of page table BOs. amdgpu_vm_update_level iterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.

This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109
+++++++++++++++++----------------
  1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
  {
      struct amdgpu_bo *shadow;
      struct amdgpu_ring *ring = NULL;
-    uint64_t pd_addr, shadow_addr = 0;
+    uint64_t pd_addr = 0, shadow_addr = 0;
      uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
      uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
      unsigned count = 0, pt_idx, ndw = 0;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8 -41
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
        WARN_ON(vm->use_cpu_for_update && shadow);
      if (vm->use_cpu_for_update && !shadow) {
-        r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-        if (r)
-            return r;
-        r = amdgpu_vm_bo_wait(adev, parent->bo);
-        if (unlikely(r)) {
-            amdgpu_bo_kunmap(parent->bo);
-            return r;
-        }
+        /* Defer kmapping until it's actually needed. Some
+         * PDBs may need no update at all
+         */
          params.func = amdgpu_vm_cpu_set_ptes;
+        params.ib = (void *)(long)-1;
      } else {
-        if (shadow) {
-            r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-            if (r)
-                return r;
-        }
-        ring = container_of(vm->entity.sched, struct amdgpu_ring,
-                    sched);
-
-        /* padding, etc. */
-        ndw = 64;
-
-        /* assume the worst case */
-        ndw += parent->last_entry_used * 6;
-
-        pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-        if (shadow) {
-            shadow_addr = amdgpu_bo_gpu_offset(shadow);
-            ndw *= 2;
-        } else {
-            shadow_addr = 0;
-        }
-
-        r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-        if (r)
-            return r;
-
-        params.ib = &job->ibs[0];
+        /* Defer IB allocation until it's actually
+         * needed. Some PDBs may need no update at all
+         */
+        params.ib = NULL;
          params.func = amdgpu_vm_do_set_ptes;
      }
  -
      /* walk over the address space and update the directory */
      for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
          struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
          if (bo == NULL)
              continue;
  -        if (bo->shadow) {
-            struct amdgpu_bo *pt_shadow = bo->shadow;
-
-            r = amdgpu_ttm_bind(&pt_shadow->tbo,
-                        &pt_shadow->tbo.mem);
-            if (r)
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1068

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -5
-                return r;
-        }
-
-        pt = amdgpu_bo_gpu_offset(bo);
-        pt = amdgpu_gart_get_vm_pde(adev, pt);
+        pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);

Getting offset from amdgpu_bo_gpu_offset(bo) is better I think, although we can assume pt bo is a known domain.

amdgpu_bo_gpu_offset does a bunch of sanity checks (WARN_ON_ONCE). They are redundant here, because we can assume that the page table BOs have been properly created and validated by kernel code we can trust more than user mode. I saw amdgpu_bo_gpu_offset near the top with "perf top" when running my memory mapping benchmark, so I think it makes sense to eliminate the function call and redundant sanity checks in this code path.

Quoted 60 lines
Quoted 60 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +42 -2
          if (parent->entries[pt_idx].addr == pt)
              continue;
            parent->entries[pt_idx].addr = pt;
  +        if (!params.ib) {
+            if (shadow) {
+                r = amdgpu_ttm_bind(&shadow->tbo,
+                            &shadow->tbo.mem);
+                if (r)
+                    return r;
+            }
+
+            ring = container_of(vm->entity.sched,
+                        struct amdgpu_ring, sched);
+
+            /* padding, etc. */
+            ndw = 64;
+
+            /* assume the worst case */
+            ndw += (parent->last_entry_used - pt_idx) * 6;
+
+            pd_addr = parent->bo->tbo.offset;
+
+            if (shadow) {
+                shadow_addr = shadow->tbo.offset;
+                ndw *= 2;
+            } else {
+                shadow_addr = 0;
+            }
+            r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+            if (r)
+                return r;
+
+            params.ib = &job->ibs[0];
+        } else if (!pd_addr) {
+            r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+            if (r)
+                return r;
+            r = amdgpu_vm_bo_wait(adev, parent->bo);
+            if (unlikely(r)) {
+                amdgpu_bo_kunmap(parent->bo);
+                return r;
+            }
+        }
+
          pde = pd_addr + pt_idx * 8;
          if (((last_pde + 8 * count) != pde) ||
              ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
        if (params.func == amdgpu_vm_cpu_set_ptes)
          amdgpu_bo_kunmap(parent->bo);
-    else if (params.ib->length_dw == 0) {
+    else if (params.ib && params.ib->length_dw == 0) {
          amdgpu_job_free(job);
-    } else {
+    } else if (params.ib) {
          amdgpu_ring_pad_ib(ring, params.ib);
          amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
                   AMDGPU_FENCE_OWNER_VM);
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1171

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -2
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
            amdgpu_bo_fence(parent->bo, fence, true);
          dma_fence_put(vm->last_dir_update);
-        vm->last_dir_update = dma_fence_get(fence);
-        dma_fence_put(fence);
+        vm->last_dir_update = fence;

Here I prefer to keep previous code, the logic is more clear.:)

Again, this is a code path that can be called very frequently. So I'd like to eliminate redundant atomic operations here.

Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
      }
      /*
       * Recurse into the subdirectories. This recursion is harmless

because

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1181

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -0
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
      for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
          struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  -        if (!entry->bo)
+        if (!entry->bo || !entry->entries)

In fact, !entry->entries would be checked at the begin of this function in next loop, but either way is ok to me here.

Doing the check here eliminates the lowest level of recursive function calls. That's where it counts the most. Each successive level of the recursion runs 512 times as often as the level above it. So completely eliminating the function call overhead of the lowest level is a good thing.

Regards,
Felix

Quoted 10 lines

Regards,
David Zhou

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1183

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
              continue;
            r = amdgpu_vm_update_level(adev, vm, entry, level + 1);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 4e7dee07-81d0-381d-579b-52e7e5969112-5C7GfCeVMHo@public.gmane.org

#6 lore raw
Quoted 3 lines · Felix Kuehling

Hi David,

Responses inline ...

Yeah, I see your mean, you mainly want to save some cpu overhead. If others have no object, feel free add my RB then.

Regards,
David Zhou

Quoted 211 lines · On 17-07-12 12:30 AM, zhoucm1 wrote:
Quoted 100 lines

nice improvement, just some nitpick inline, otherwise the patch is

Quoted 96 lines · Felix Kuehling

When lots of virtual address spaces is used, there can be thousands of page table BOs. amdgpu_vm_update_level iterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.

This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109
+++++++++++++++++----------------
   1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
   {
       struct amdgpu_bo *shadow;
       struct amdgpu_ring *ring = NULL;
-    uint64_t pd_addr, shadow_addr = 0;
+    uint64_t pd_addr = 0, shadow_addr = 0;
       uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
       uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
       unsigned count = 0, pt_idx, ndw = 0;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8 -41
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
         WARN_ON(vm->use_cpu_for_update && shadow);
       if (vm->use_cpu_for_update && !shadow) {
-        r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-        if (r)
-            return r;
-        r = amdgpu_vm_bo_wait(adev, parent->bo);
-        if (unlikely(r)) {
-            amdgpu_bo_kunmap(parent->bo);
-            return r;
-        }
+        /* Defer kmapping until it's actually needed. Some
+         * PDBs may need no update at all
+         */
           params.func = amdgpu_vm_cpu_set_ptes;
+        params.ib = (void *)(long)-1;
       } else {
-        if (shadow) {
-            r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-            if (r)
-                return r;
-        }
-        ring = container_of(vm->entity.sched, struct amdgpu_ring,
-                    sched);
-
-        /* padding, etc. */
-        ndw = 64;
-
-        /* assume the worst case */
-        ndw += parent->last_entry_used * 6;
-
-        pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-        if (shadow) {
-            shadow_addr = amdgpu_bo_gpu_offset(shadow);
-            ndw *= 2;
-        } else {
-            shadow_addr = 0;
-        }
-
-        r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-        if (r)
-            return r;
-
-        params.ib = &job->ibs[0];
+        /* Defer IB allocation until it's actually
+         * needed. Some PDBs may need no update at all
+         */
+        params.ib = NULL;
           params.func = amdgpu_vm_do_set_ptes;
       }
   -
       /* walk over the address space and update the directory */
       for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
           struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
           if (bo == NULL)
               continue;
   -        if (bo->shadow) {
-            struct amdgpu_bo *pt_shadow = bo->shadow;
-
-            r = amdgpu_ttm_bind(&pt_shadow->tbo,
-                        &pt_shadow->tbo.mem);
-            if (r)
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1068

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -5
-                return r;
-        }
-
-        pt = amdgpu_bo_gpu_offset(bo);
-        pt = amdgpu_gart_get_vm_pde(adev, pt);
+        pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);

Getting offset from amdgpu_bo_gpu_offset(bo) is better I think, although we can assume pt bo is a known domain.

amdgpu_bo_gpu_offset does a bunch of sanity checks (WARN_ON_ONCE). They are redundant here, because we can assume that the page table BOs have been properly created and validated by kernel code we can trust more than user mode. I saw amdgpu_bo_gpu_offset near the top with "perf top" when running my memory mapping benchmark, so I think it makes sense to eliminate the function call and redundant sanity checks in this code path.

Quoted 59 lines
Quoted 59 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +42 -2
           if (parent->entries[pt_idx].addr == pt)
               continue;
             parent->entries[pt_idx].addr = pt;
   +        if (!params.ib) {
+            if (shadow) {
+                r = amdgpu_ttm_bind(&shadow->tbo,
+                            &shadow->tbo.mem);
+                if (r)
+                    return r;
+            }
+
+            ring = container_of(vm->entity.sched,
+                        struct amdgpu_ring, sched);
+
+            /* padding, etc. */
+            ndw = 64;
+
+            /* assume the worst case */
+            ndw += (parent->last_entry_used - pt_idx) * 6;
+
+            pd_addr = parent->bo->tbo.offset;
+
+            if (shadow) {
+                shadow_addr = shadow->tbo.offset;
+                ndw *= 2;
+            } else {
+                shadow_addr = 0;
+            }
+            r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+            if (r)
+                return r;
+
+            params.ib = &job->ibs[0];
+        } else if (!pd_addr) {
+            r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+            if (r)
+                return r;
+            r = amdgpu_vm_bo_wait(adev, parent->bo);
+            if (unlikely(r)) {
+                amdgpu_bo_kunmap(parent->bo);
+                return r;
+            }
+        }
+
           pde = pd_addr + pt_idx * 8;
           if (((last_pde + 8 * count) != pde) ||
               ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
         if (params.func == amdgpu_vm_cpu_set_ptes)
           amdgpu_bo_kunmap(parent->bo);
-    else if (params.ib->length_dw == 0) {
+    else if (params.ib && params.ib->length_dw == 0) {
           amdgpu_job_free(job);
-    } else {
+    } else if (params.ib) {
           amdgpu_ring_pad_ib(ring, params.ib);
           amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
                    AMDGPU_FENCE_OWNER_VM);
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1171

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -2
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
             amdgpu_bo_fence(parent->bo, fence, true);
           dma_fence_put(vm->last_dir_update);
-        vm->last_dir_update = dma_fence_get(fence);
-        dma_fence_put(fence);
+        vm->last_dir_update = fence;

Here I prefer to keep previous code, the logic is more clear.:)

Again, this is a code path that can be called very frequently. So I'd like to eliminate redundant atomic operations here.

Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
       }
       /*
        * Recurse into the subdirectories. This recursion is harmless

because

Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1181

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -0
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
       for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
           struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
   -        if (!entry->bo)
+        if (!entry->bo || !entry->entries)

In fact, !entry->entries would be checked at the begin of this function in next loop, but either way is ok to me here.

Doing the check here eliminates the lowest level of recursive function calls. That's where it counts the most. Each successive level of the recursion runs 512 times as often as the level above it. So completely eliminating the function call overhead of the lowest level is a good thing.

Regards,
Felix

Quoted 4 lines

Regards,
David Zhou

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0 -0
               continue;
             r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
Quoted 8 lines
Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#7 lore raw
In reply to a message not in the archive

Am 12.07.2017 um 05:56 schrieb Felix Kuehling:

Quoted 10 lines

When lots of virtual address spaces is used, there can be thousands of page table BOs. amdgpu_vm_update_level iterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.

This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.

That optimization is unnecessary.

I have a patch in the pipeline that makes the VM BO permanently CPU mapped and also fixes a couple of bugs in those code path.

Give me a moment to clean that up and send it out.

Regards,
Christian.

Quoted 162 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++----------------
  1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +54 -53
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  {
  	struct amdgpu_bo *shadow;
  	struct amdgpu_ring *ring = NULL;
-	uint64_t pd_addr, shadow_addr = 0;
+	uint64_t pd_addr = 0, shadow_addr = 0;
  	uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
  	uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
  	unsigned count = 0, pt_idx, ndw = 0;
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
  	WARN_ON(vm->use_cpu_for_update && shadow);
  	if (vm->use_cpu_for_update && !shadow) {
-		r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-		if (r)
-			return r;
-		r = amdgpu_vm_bo_wait(adev, parent->bo);
-		if (unlikely(r)) {
-			amdgpu_bo_kunmap(parent->bo);
-			return r;
-		}
+		/* Defer kmapping until it's actually needed. Some
+		 * PDBs may need no update at all
+		 */
  		params.func = amdgpu_vm_cpu_set_ptes;
+		params.ib = (void *)(long)-1;
  	} else {
-		if (shadow) {
-			r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-		ring = container_of(vm->entity.sched, struct amdgpu_ring,
-				    sched);
-
-		/* padding, etc. */
-		ndw = 64;
-
-		/* assume the worst case */
-		ndw += parent->last_entry_used * 6;
-
-		pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-		if (shadow) {
-			shadow_addr = amdgpu_bo_gpu_offset(shadow);
-			ndw *= 2;
-		} else {
-			shadow_addr = 0;
-		}
-
-		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-		if (r)
-			return r;
-
-		params.ib = &job->ibs[0];
+		/* Defer IB allocation until it's actually
+		 * needed. Some PDBs may need no update at all
+		 */
+		params.ib = NULL;
  		params.func = amdgpu_vm_do_set_ptes;
  	}
 
-
  	/* walk over the address space and update the directory */
  	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  		struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  		if (bo == NULL)
  			continue;
 
-		if (bo->shadow) {
-			struct amdgpu_bo *pt_shadow = bo->shadow;
-
-			r = amdgpu_ttm_bind(&pt_shadow->tbo,
-					    &pt_shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-
-		pt = amdgpu_bo_gpu_offset(bo);
-		pt = amdgpu_gart_get_vm_pde(adev, pt);
+		pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);
  		if (parent->entries[pt_idx].addr == pt)
  			continue;
 
  		parent->entries[pt_idx].addr = pt;
 
+		if (!params.ib) {
+			if (shadow) {
+				r = amdgpu_ttm_bind(&shadow->tbo,
+						    &shadow->tbo.mem);
+				if (r)
+					return r;
+			}
+
+			ring = container_of(vm->entity.sched,
+					    struct amdgpu_ring, sched);
+
+			/* padding, etc. */
+			ndw = 64;
+
+			/* assume the worst case */
+			ndw += (parent->last_entry_used - pt_idx) * 6;
+
+			pd_addr = parent->bo->tbo.offset;
+
+			if (shadow) {
+				shadow_addr = shadow->tbo.offset;
+				ndw *= 2;
+			} else {
+				shadow_addr = 0;
+			}
+			r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+			if (r)
+				return r;
+
+			params.ib = &job->ibs[0];
+		} else if (!pd_addr) {
+			r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+			if (r)
+				return r;
+			r = amdgpu_vm_bo_wait(adev, parent->bo);
+			if (unlikely(r)) {
+				amdgpu_bo_kunmap(parent->bo);
+				return r;
+			}
+		}
+
  		pde = pd_addr + pt_idx * 8;
  		if (((last_pde + 8 * count) != pde) ||
  		    ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
  	if (params.func == amdgpu_vm_cpu_set_ptes)
  		amdgpu_bo_kunmap(parent->bo);
-	else if (params.ib->length_dw == 0) {
+	else if (params.ib && params.ib->length_dw == 0) {
  		amdgpu_job_free(job);
-	} else {
+	} else if (params.ib) {
  		amdgpu_ring_pad_ib(ring, params.ib);
  		amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
  				 AMDGPU_FENCE_OWNER_VM);
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 
  		amdgpu_bo_fence(parent->bo, fence, true);
  		dma_fence_put(vm->last_dir_update);
-		vm->last_dir_update = dma_fence_get(fence);
-		dma_fence_put(fence);
+		vm->last_dir_update = fence;
  	}
  	/*
  	 * Recurse into the subdirectories. This recursion is harmless because

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1183

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
-		if (!entry->bo)
+		if (!entry->bo || !entry->entries)
  			continue;
 
  		r = amdgpu_vm_update_level(adev, vm, entry, level + 1);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): c50326a0-6813-d484-b289-5681b306d723-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#9 lore raw
Quoted 5 lines · Christian König

That optimization is unnecessary.

I have a patch in the pipeline that makes the VM BO permanently CPU mapped and also fixes a couple of bugs in those code path.

You mean just for the CPU-update path? I think deferring the SDMA IB allocation still makes sense. In fact that's the code path I tested.

Quoted 2 lines

Give me a moment to clean that up and send it out.

Sure. BTW, I found that the most expensive operation turned out to be amdgpu_vm_move_pt_bos_in_lru. It has to recurse through all PT BOs and does lots of redundant list pointer manipulations for two lists (LRU and swap), and kref get/put.

For KFD VMs I'm removing the call to it. I think it's useless for us.

amdgpu_vm_validate_pt_bos is also expensive, but not as bad because it's only done if there was an eviction.

I was thinking about ways to optimize this. It should be possible to allocate multiple PTBs and PDBs in a single BO in amdgpu_vm_alloc_pts. That would potentially save lots of time in vm_move_pt_bos and vm_validate. What do you think?

Regards,
Felix

Quoted 166 lines

Regards,
Christian.

Quoted 162 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109
+++++++++++++++++----------------
  1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
  {
      struct amdgpu_bo *shadow;
      struct amdgpu_ring *ring = NULL;
-    uint64_t pd_addr, shadow_addr = 0;
+    uint64_t pd_addr = 0, shadow_addr = 0;
      uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
      uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
      unsigned count = 0, pt_idx, ndw = 0;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +52 -50
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
        WARN_ON(vm->use_cpu_for_update && shadow);
      if (vm->use_cpu_for_update && !shadow) {
-        r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-        if (r)
-            return r;
-        r = amdgpu_vm_bo_wait(adev, parent->bo);
-        if (unlikely(r)) {
-            amdgpu_bo_kunmap(parent->bo);
-            return r;
-        }
+        /* Defer kmapping until it's actually needed. Some
+         * PDBs may need no update at all
+         */
          params.func = amdgpu_vm_cpu_set_ptes;
+        params.ib = (void *)(long)-1;
      } else {
-        if (shadow) {
-            r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-            if (r)
-                return r;
-        }
-        ring = container_of(vm->entity.sched, struct amdgpu_ring,
-                    sched);
-
-        /* padding, etc. */
-        ndw = 64;
-
-        /* assume the worst case */
-        ndw += parent->last_entry_used * 6;
-
-        pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-        if (shadow) {
-            shadow_addr = amdgpu_bo_gpu_offset(shadow);
-            ndw *= 2;
-        } else {
-            shadow_addr = 0;
-        }
-
-        r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-        if (r)
-            return r;
-
-        params.ib = &job->ibs[0];
+        /* Defer IB allocation until it's actually
+         * needed. Some PDBs may need no update at all
+         */
+        params.ib = NULL;
          params.func = amdgpu_vm_do_set_ptes;
      }
  -
      /* walk over the address space and update the directory */
      for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
          struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
          if (bo == NULL)
              continue;
  -        if (bo->shadow) {
-            struct amdgpu_bo *pt_shadow = bo->shadow;
-
-            r = amdgpu_ttm_bind(&pt_shadow->tbo,
-                        &pt_shadow->tbo.mem);
-            if (r)
-                return r;
-        }
-
-        pt = amdgpu_bo_gpu_offset(bo);
-        pt = amdgpu_gart_get_vm_pde(adev, pt);
+        pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);
          if (parent->entries[pt_idx].addr == pt)
              continue;
            parent->entries[pt_idx].addr = pt;
  +        if (!params.ib) {
+            if (shadow) {
+                r = amdgpu_ttm_bind(&shadow->tbo,
+                            &shadow->tbo.mem);
+                if (r)
+                    return r;
+            }
+
+            ring = container_of(vm->entity.sched,
+                        struct amdgpu_ring, sched);
+
+            /* padding, etc. */
+            ndw = 64;
+
+            /* assume the worst case */
+            ndw += (parent->last_entry_used - pt_idx) * 6;
+
+            pd_addr = parent->bo->tbo.offset;
+
+            if (shadow) {
+                shadow_addr = shadow->tbo.offset;
+                ndw *= 2;
+            } else {
+                shadow_addr = 0;
+            }
+            r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+            if (r)
+                return r;
+
+            params.ib = &job->ibs[0];
+        } else if (!pd_addr) {
+            r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+            if (r)
+                return r;
+            r = amdgpu_vm_bo_wait(adev, parent->bo);
+            if (unlikely(r)) {
+                amdgpu_bo_kunmap(parent->bo);
+                return r;
+            }
+        }
+
          pde = pd_addr + pt_idx * 8;
          if (((last_pde + 8 * count) != pde) ||
              ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
        if (params.func == amdgpu_vm_cpu_set_ptes)
          amdgpu_bo_kunmap(parent->bo);
-    else if (params.ib->length_dw == 0) {
+    else if (params.ib && params.ib->length_dw == 0) {
          amdgpu_job_free(job);
-    } else {
+    } else if (params.ib) {
          amdgpu_ring_pad_ib(ring, params.ib);
          amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
                   AMDGPU_FENCE_OWNER_VM);
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
            amdgpu_bo_fence(parent->bo, fence, true);
          dma_fence_put(vm->last_dir_update);
-        vm->last_dir_update = dma_fence_get(fence);
-        dma_fence_put(fence);
+        vm->last_dir_update = fence;
      }
      /*
       * Recurse into the subdirectories. This recursion is harmless

because

Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1183

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -0
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
      for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
          struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  -        if (!entry->bo)
+        if (!entry->bo || !entry->entries)
              continue;
            r = amdgpu_vm_update_level(adev, vm, entry, level + 1);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 083c3f4e-72b3-1ad1-6d11-a9eba48e3c9c-5C7GfCeVMHo@public.gmane.org

#11 lore raw

Am 12.07.2017 um 21:38 schrieb Felix Kuehling:

Quoted 8 lines · Christian König
Quoted 5 lines

That optimization is unnecessary.

I have a patch in the pipeline that makes the VM BO permanently CPU mapped and also fixes a couple of bugs in those code path.

You mean just for the CPU-update path? I think deferring the SDMA IB allocation still makes sense. In fact that's the code path I tested.

I've added tracking if we need to update anything to all paths, not just the CPU update path.

Quoted 5 lines
Quoted 1 line

Give me a moment to clean that up and send it out.

Sure. BTW, I found that the most expensive operation turned out to be amdgpu_vm_move_pt_bos_in_lru. It has to recurse through all PT BOs and does lots of redundant list pointer manipulations for two lists (LRU and swap), and kref get/put.

Yeah, that's why I've stopped using the separate remove/add functions and wanted to optimize just moving everything to the end.

But that turned out to be more problematic than I thought.

Quoted 9 lines

For KFD VMs I'm removing the call to it. I think it's useless for us.

amdgpu_vm_validate_pt_bos is also expensive, but not as bad because it's only done if there was an eviction.

I was thinking about ways to optimize this. It should be possible to allocate multiple PTBs and PDBs in a single BO in amdgpu_vm_alloc_pts. That would potentially save lots of time in vm_move_pt_bos and vm_validate. What do you think?

I think allocating separate BOs is still the right approach, a single BO is tricky to allocate and probably won't worth it.

Instead I move the LRU handling after the validation. So BOs are only moved in the LRU when there actually was some eviction.

Since we don't care what kind of eviction it was this basically turns LRU handling on/off based on memory pressure (e.g. evictions).

Regards,
Christian.

Quoted 178 lines

Regards,
Felix

Quoted 174 lines

Regards,
Christian.

Quoted 170 lines
---
   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109
+++++++++++++++++----------------
   1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index ff5de3a..23b899b 100644
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
   {
       struct amdgpu_bo *shadow;
       struct amdgpu_ring *ring = NULL;
-    uint64_t pd_addr, shadow_addr = 0;
+    uint64_t pd_addr = 0, shadow_addr = 0;
       uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
       uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
       unsigned count = 0, pt_idx, ndw = 0;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +52 -50
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
         WARN_ON(vm->use_cpu_for_update && shadow);
       if (vm->use_cpu_for_update && !shadow) {
-        r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-        if (r)
-            return r;
-        r = amdgpu_vm_bo_wait(adev, parent->bo);
-        if (unlikely(r)) {
-            amdgpu_bo_kunmap(parent->bo);
-            return r;
-        }
+        /* Defer kmapping until it's actually needed. Some
+         * PDBs may need no update at all
+         */
           params.func = amdgpu_vm_cpu_set_ptes;
+        params.ib = (void *)(long)-1;
       } else {
-        if (shadow) {
-            r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-            if (r)
-                return r;
-        }
-        ring = container_of(vm->entity.sched, struct amdgpu_ring,
-                    sched);
-
-        /* padding, etc. */
-        ndw = 64;
-
-        /* assume the worst case */
-        ndw += parent->last_entry_used * 6;
-
-        pd_addr = amdgpu_bo_gpu_offset(parent->bo);
-
-        if (shadow) {
-            shadow_addr = amdgpu_bo_gpu_offset(shadow);
-            ndw *= 2;
-        } else {
-            shadow_addr = 0;
-        }
-
-        r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-        if (r)
-            return r;
-
-        params.ib = &job->ibs[0];
+        /* Defer IB allocation until it's actually
+         * needed. Some PDBs may need no update at all
+         */
+        params.ib = NULL;
           params.func = amdgpu_vm_do_set_ptes;
       }
   -
       /* walk over the address space and update the directory */
       for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
           struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
           if (bo == NULL)
               continue;
   -        if (bo->shadow) {
-            struct amdgpu_bo *pt_shadow = bo->shadow;
-
-            r = amdgpu_ttm_bind(&pt_shadow->tbo,
-                        &pt_shadow->tbo.mem);
-            if (r)
-                return r;
-        }
-
-        pt = amdgpu_bo_gpu_offset(bo);
-        pt = amdgpu_gart_get_vm_pde(adev, pt);
+        pt = amdgpu_gart_get_vm_pde(adev, bo->tbo.offset);
           if (parent->entries[pt_idx].addr == pt)
               continue;
             parent->entries[pt_idx].addr = pt;
   +        if (!params.ib) {
+            if (shadow) {
+                r = amdgpu_ttm_bind(&shadow->tbo,
+                            &shadow->tbo.mem);
+                if (r)
+                    return r;
+            }
+
+            ring = container_of(vm->entity.sched,
+                        struct amdgpu_ring, sched);
+
+            /* padding, etc. */
+            ndw = 64;
+
+            /* assume the worst case */
+            ndw += (parent->last_entry_used - pt_idx) * 6;
+
+            pd_addr = parent->bo->tbo.offset;
+
+            if (shadow) {
+                shadow_addr = shadow->tbo.offset;
+                ndw *= 2;
+            } else {
+                shadow_addr = 0;
+            }
+            r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+            if (r)
+                return r;
+
+            params.ib = &job->ibs[0];
+        } else if (!pd_addr) {
+            r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
+            if (r)
+                return r;
+            r = amdgpu_vm_bo_wait(adev, parent->bo);
+            if (unlikely(r)) {
+                amdgpu_bo_kunmap(parent->bo);
+                return r;
+            }
+        }
+
           pde = pd_addr + pt_idx * 8;
           if (((last_pde + 8 * count) != pde) ||
               ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
         if (params.func == amdgpu_vm_cpu_set_ptes)
           amdgpu_bo_kunmap(parent->bo);
-    else if (params.ib->length_dw == 0) {
+    else if (params.ib && params.ib->length_dw == 0) {
           amdgpu_job_free(job);
-    } else {
+    } else if (params.ib) {
           amdgpu_ring_pad_ib(ring, params.ib);
           amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
                    AMDGPU_FENCE_OWNER_VM);
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
             amdgpu_bo_fence(parent->bo, fence, true);
           dma_fence_put(vm->last_dir_update);
-        vm->last_dir_update = dma_fence_get(fence);
-        dma_fence_put(fence);
+        vm->last_dir_update = fence;
       }
       /*
        * Recurse into the subdirectories. This recursion is harmless

because

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -0
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct
 amdgpu_device *adev,
       for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
           struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
   -        if (!entry->bo)
+        if (!entry->bo || !entry->entries)
               continue;
             r = amdgpu_vm_update_level(adev, vm, entry, level + 1);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx