drm/amdgpu: Optimize recursion in amdgpu_vm_update_level
Subsystems (2)
PATCH 1/1 drm/amdgpu: Optimize recursion in amdgpu_vm_update_level
When lots of virtual address spaces is used, there can be thousands of page table BOs. amdgpu_vm_update_level iterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.
This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +55 -54
@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count =0 , pt_idx, ndw =0 ;
@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx =0 ; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,if (bo ==NULL )continue ; -if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow; - - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r) -return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; +if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + }else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||
@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); -else if (params.ib->length_dw ==0 ) { +else if (params.ib && params.ib->length_dw ==0 ) {amdgpu_job_free (job); - }else { + }else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );
@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence; }/* * Recurse into the subdirectories. This recursion is harmless because
@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,for (pt_idx =0 ; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; -if (!entry->bo) +if (!entry->bo || !entry->entries)continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );
Signature · 6 lines
1.9.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 1499831769-3764-1-git-send-email-Felix.Kuehling-5C7GfCeVMHo@public.gmane.org
nice improvement, just some nitpick inline, otherwise the patch is Reviewed-by: Chunming Zhou <[email protected]>.
Quoted 90 lines · Felix Kuehling
When lots of virtual address spaces is used, there can be thousands of page table BOs.
amdgpu_vm_update_leveliterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.
Signed-off-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +9
-40@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count =0 , pt_idx, ndw =0 ;@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx =0 ; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,if (bo ==NULL )continue ; -if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1068
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-9- - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r) -return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);
Getting offset from amdgpu_bo_gpu_offset(bo) is better I think, although we can assume pt bo is a known domain.
Quoted 62 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +43
-2if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; +if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + }else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); -else if (params.ib->length_dw ==0 ) { +else if (params.ib && params.ib->length_dw ==0 ) {amdgpu_job_free (job); - }else { + }else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1171
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-2@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence;
Here I prefer to keep previous code, the logic is more clear.:)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1180
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1}/* * Recurse into the subdirectories. This recursion is harmless because @@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,for (pt_idx =0 ; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; -if (!entry->bo) +if (!entry->bo || !entry->entries)
In fact, !entry->entries would be checked at the begin of this function in next loop, but either way is ok to me here.
Regards,
David Zhou
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1183
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): [email protected]
Hi David,
Responses inline ...
Quoted 100 lines · On 17-07-12 12:30 AM, zhoucm1 wrote:
nice improvement, just some nitpick inline, otherwise the patch is
Reviewed-by Chunming Zhou <[email protected]>.Quoted 96 lines · Felix Kuehling
When lots of virtual address spaces is used, there can be thousands of page table BOs.
amdgpu_vm_update_leveliterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.
Signed-off-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count = 0, pt_idx, ndw = 0;drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8
-41@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (bo == NULL) continue; - if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow; - - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r)
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1068
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-5-return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);Getting offset from
amdgpu_bo_gpu_offset(bo)is better I think, although we can assume pt bo is a known domain.
amdgpu_bo_gpu_offset does a bunch of sanity checks (WARN_ON_ONCE). They are redundant here, because we can assume that the page table BOs have been properly created and validated by kernel code we can trust more than user mode. I saw amdgpu_bo_gpu_offset near the top with "perf top" when running my memory mapping benchmark, so I think it makes sense to eliminate the function call and redundant sanity checks in this code path.
Quoted 60 lines
Quoted 60 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +42
-2if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; + if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + } else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); - else if (params.ib->length_dw == 0) { + else if (params.ib && params.ib->length_dw == 0) {amdgpu_job_free (job); - } else { + } else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1171
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-2@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence;Here I prefer to keep previous code, the logic is more clear.:)
Again, this is a code path that can be called very frequently. So I'd like to eliminate redundant atomic operations here.
Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0}/* * Recurse into the subdirectories. This recursion is harmless because
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1181
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-0@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; - if (!entry->bo) + if (!entry->bo || !entry->entries)In fact, !entry->entries would be checked at the begin of this function in next loop, but either way is ok to me here.
Doing the check here eliminates the lowest level of recursive function calls. That's where it counts the most. Each successive level of the recursion runs 512 times as often as the level above it. So completely eliminating the function call overhead of the lowest level is a good thing.
Regards,
Felix
Quoted 10 lines
Regards,
David Zhoudrivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1183
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 4e7dee07-81d0-381d-579b-52e7e5969112-5C7GfCeVMHo@public.gmane.org
Quoted 3 lines · Felix Kuehling
Hi David,
Responses inline ...
Yeah, I see your mean, you mainly want to save some cpu overhead. If others have no object, feel free add my RB then.
Regards,
David Zhou
Quoted 211 lines · On 17-07-12 12:30 AM, zhoucm1 wrote:
Quoted 100 lines
nice improvement, just some nitpick inline, otherwise the patch is
Reviewed-by Chunming Zhou <[email protected]>.Quoted 96 lines · Felix Kuehling
When lots of virtual address spaces is used, there can be thousands of page table BOs.
amdgpu_vm_update_leveliterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.
Signed-off-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count = 0, pt_idx, ndw = 0;drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8
-41@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (bo == NULL) continue; - if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow; - - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r)Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1068
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-5-return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);Getting offset from
amdgpu_bo_gpu_offset(bo)is better I think, although we can assume pt bo is a known domain.
amdgpu_bo_gpu_offsetdoes a bunch of sanity checks (WARN_ON_ONCE). They are redundant here, because we can assume that the page table BOs have been properly created and validated by kernel code we can trust more than user mode. I sawamdgpu_bo_gpu_offsetnear the top with "perf top" when running my memory mapping benchmark, so I think it makes sense to eliminate the function call and redundant sanity checks in this code path.Quoted 59 lines
Quoted 59 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +42
-2if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; + if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + } else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); - else if (params.ib->length_dw == 0) { + else if (params.ib && params.ib->length_dw == 0) {amdgpu_job_free (job); - } else { + } else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1171
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-2@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence;Here I prefer to keep previous code, the logic is more clear.:)
Again, this is a code path that can be called very frequently. So I'd like to eliminate redundant atomic operations here.
Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0}/* * Recurse into the subdirectories. This recursion is harmless because
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1181
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-0@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; - if (!entry->bo) + if (!entry->bo || !entry->entries)In fact, !entry->entries would be checked at the begin of this function in next loop, but either way is ok to me here.
Doing the check here eliminates the lowest level of recursive function calls. That's where it counts the most. Each successive level of the recursion runs 512 times as often as the level above it. So completely eliminating the function call overhead of the lowest level is a good thing.
Regards,
FelixQuoted 4 lines
Regards,
David ZhouQuoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );
Quoted 8 lines
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 12.07.2017 um 05:56 schrieb Felix Kuehling:
Quoted 10 lines
When lots of virtual address spaces is used, there can be thousands of page table BOs.
amdgpu_vm_update_leveliterates over all of them recursively. In many cases only a few or none at all need to be updated. Minimize unnecessary code execution and memory usage in those cases.This speeds up memory mapping in a synthetic KFD memory mapping benchmark by roughly a factor two.
Signed-off-by Felix Kuehling <[email protected]>
That optimization is unnecessary.
I have a patch in the pipeline that makes the VM BO permanently CPU mapped and also fixes a couple of bugs in those code path.
Give me a moment to clean that up and send it out.
Regards,
Christian.
Quoted 162 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +54
-53@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count =0 , pt_idx, ndw =0 ;@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx =0 ; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,if (bo ==NULL )continue ; -if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow; - - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r) -return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; +if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + }else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); -else if (params.ib->length_dw ==0 ) { +else if (params.ib && params.ib->length_dw ==0 ) {amdgpu_job_free (job); - }else { + }else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence; }/* * Recurse into the subdirectories. This recursion is harmless because
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct amdgpu_device *adev, · line 1183
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,for (pt_idx =0 ; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; -if (!entry->bo) +if (!entry->bo || !entry->entries)continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): c50326a0-6813-d484-b289-5681b306d723-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Quoted 5 lines · Christian König
That optimization is unnecessary.
I have a patch in the pipeline that makes the VM BO permanently CPU mapped and also fixes a couple of bugs in those code path.
You mean just for the CPU-update path? I think deferring the SDMA IB allocation still makes sense. In fact that's the code path I tested.
Quoted 2 lines
Give me a moment to clean that up and send it out.
Sure. BTW, I found that the most expensive operation turned out to be amdgpu_vm_move_pt_bos_in_lru. It has to recurse through all PT BOs and does lots of redundant list pointer manipulations for two lists (LRU and swap), and kref get/put.
For KFD VMs I'm removing the call to it. I think it's useless for us.
amdgpu_vm_validate_pt_bos is also expensive, but not as bad because it's only done if there was an eviction.
I was thinking about ways to optimize this. It should be possible to allocate multiple PTBs and PDBs in a single BO in amdgpu_vm_alloc_pts. That would potentially save lots of time in vm_move_pt_bos and vm_validate. What do you think?
Regards,
Felix
Quoted 166 lines
Regards,
Christian.Quoted 162 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count = 0, pt_idx, ndw = 0;drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +52
-50@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (bo == NULL) continue; - if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow; - - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r) -return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; + if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + } else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); - else if (params.ib->length_dw == 0) { + else if (params.ib && params.ib->length_dw == 0) {amdgpu_job_free (job); - } else { + } else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence; }/* * Recurse into the subdirectories. This recursion is harmless because
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c static int amdgpu_vm_update_level(struct · line 1183
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-0@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; - if (!entry->bo) +if (!entry->bo || !entry->entries)continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 083c3f4e-72b3-1ad1-6d11-a9eba48e3c9c-5C7GfCeVMHo@public.gmane.org
Am 12.07.2017 um 21:38 schrieb Felix Kuehling:
Quoted 8 lines · Christian König
Quoted 5 lines
That optimization is unnecessary.
I have a patch in the pipeline that makes the VM BO permanently CPU mapped and also fixes a couple of bugs in those code path.
You mean just for the CPU-update path? I think deferring the SDMA IB allocation still makes sense. In fact that's the code path I tested.
I've added tracking if we need to update anything to all paths, not just the CPU update path.
Quoted 5 lines
Quoted 1 line
Give me a moment to clean that up and send it out.
Sure. BTW, I found that the most expensive operation turned out to be
amdgpu_vm_move_pt_bos_in_lru. It has to recurse through all PT BOs and does lots of redundant list pointer manipulations for two lists (LRU and swap), and kref get/put.
Yeah, that's why I've stopped using the separate remove/add functions and wanted to optimize just moving everything to the end.
But that turned out to be more problematic than I thought.
Quoted 9 lines
For KFD VMs I'm removing the call to it. I think it's useless for us.
amdgpu_vm_validate_pt_bosis also expensive, but not as bad because it's only done if there was an eviction.I was thinking about ways to optimize this. It should be possible to allocate multiple PTBs and PDBs in a single BO in
amdgpu_vm_alloc_pts. That would potentially save lots of time invm_move_pt_bosandvm_validate. What do you think?
I think allocating separate BOs is still the right approach, a single BO is tricky to allocate and probably won't worth it.
Instead I move the LRU handling after the validation. So BOs are only moved in the LRU when there actually was some eviction.
Since we don't care what kind of eviction it was this basically turns LRU handling on/off based on memory pressure (e.g. evictions).
Regards,
Christian.
Quoted 178 lines
Regards,
FelixQuoted 174 lines
Regards,
Christian.Quoted 170 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 109 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 54 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -1025,7 +1025,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, {struct amdgpu_bo *shadow;struct amdgpu_ring *ring =NULL ; -uint64_t pd_addr, shadow_addr =0 ; +uint64_t pd_addr =0 , shadow_addr =0 ;uint32_t incr =amdgpu_vm_bo_size (adev, level +1 );uint64_t last_pde = ~0 , last_pt = ~0 , last_shadow = ~0 ;unsigned count = 0, pt_idx, ndw = 0;drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +52
-50@@ -1044,48 +1044,19 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,WARN_ON (vm->use_cpu_for_update && shadow);if (vm->use_cpu_for_update && !shadow) { - r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); -if (r) -return r; - r =amdgpu_vm_bo_wait (adev, parent->bo); -if (unlikely (r)) { -amdgpu_bo_kunmap (parent->bo); -return r; - } +/* Defer kmapping until it's actually needed. Some +* PDBs may need no update at all +*/ params.func = amdgpu_vm_cpu_set_ptes; + params.ib = (void *)(long )-1 ; }else { -if (shadow) { - r =amdgpu_ttm_bind (&shadow->tbo, &shadow->tbo.mem); -if (r) -return r; - } - ring = container_of(vm->entity.sched, struct amdgpu_ring, - sched); - -/* padding, etc. */ - ndw =64 ; - -/* assume the worst case */ - ndw += parent->last_entry_used *6 ; - - pd_addr =amdgpu_bo_gpu_offset (parent->bo); - -if (shadow) { - shadow_addr =amdgpu_bo_gpu_offset (shadow); - ndw *=2 ; - }else { - shadow_addr =0 ; - } - - r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); -if (r) -return r; - - params.ib = &job->ibs[0 ]; +/* Defer IB allocation until it's actually +* needed. Some PDBs may need no update at all +*/ + params.ib =NULL ; params.func = amdgpu_vm_do_set_ptes; } -/* walk over the address space and update the directory */ for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_bo *bo = parent->entries[pt_idx].bo;@@ -1094,22 +1065,53 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (bo == NULL) continue; - if (bo->shadow) { -struct amdgpu_bo *pt_shadow = bo->shadow; - - r =amdgpu_ttm_bind (&pt_shadow->tbo, - &pt_shadow->tbo.mem); -if (r) -return r; - } - - pt =amdgpu_bo_gpu_offset (bo); - pt =amdgpu_gart_get_vm_pde (adev, pt); + pt =amdgpu_gart_get_vm_pde (adev, bo->tbo.offset);if (parent->entries[pt_idx].addr == pt)continue ; parent->entries[pt_idx].addr = pt; + if (!params.ib) { +if (shadow) { + r =amdgpu_ttm_bind (&shadow->tbo, + &shadow->tbo.mem); +if (r) +return r; + } + + ring =container_of (vm->entity.sched, + struct amdgpu_ring, sched); + +/* padding, etc. */ + ndw =64 ; + +/* assume the worst case */ + ndw += (parent->last_entry_used - pt_idx) *6 ; + + pd_addr = parent->bo->tbo.offset; + +if (shadow) { + shadow_addr = shadow->tbo.offset; + ndw *=2 ; + }else { + shadow_addr =0 ; + } + r =amdgpu_job_alloc_with_ib (adev, ndw *4 , &job); +if (r) +return r; + + params.ib = &job->ibs[0 ]; + } else if (!pd_addr) { + r =amdgpu_bo_kmap (parent->bo, (void **)&pd_addr); +if (r) +return r; + r =amdgpu_vm_bo_wait (adev, parent->bo); +if (unlikely (r)) { +amdgpu_bo_kunmap (parent->bo); +return r; + } + } + pde = pd_addr + pt_idx *8 ; if (((last_pde + 8 * count) != pde) || ((last_pt + incr * count) != pt) ||@@ -1148,9 +1150,9 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, if (params.func == amdgpu_vm_cpu_set_ptes)amdgpu_bo_kunmap (parent->bo); - else if (params.ib->length_dw == 0) { + else if (params.ib && params.ib->length_dw == 0) {amdgpu_job_free (job); - } else { + } else if (params.ib) {amdgpu_ring_pad_ib (ring, params.ib);amdgpu_sync_resv (adev, &job->sync, parent->bo->tbo.resv,AMDGPU_FENCE_OWNER_VM );@@ -1166,8 +1168,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,amdgpu_bo_fence (parent->bo, fence, true);dma_fence_put (vm->last_dir_update); - vm->last_dir_update =dma_fence_get (fence); -dma_fence_put (fence); + vm->last_dir_update = fence; }/* * Recurse into the subdirectories. This recursion is harmless because
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-0@@ -1176,7 +1177,7 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev, for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; - if (!entry->bo) +if (!entry->bo || !entry->entries)continue ; r =amdgpu_vm_update_level (adev, vm, entry, level +1 );
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx