drm/amdkfd: Add helper function svm_range_need_access_gpus
Subsystems (3)
PATCH v4 1/7 drm/amdkfd: Add helper function svm_range_need_access_gpus
Add the helper function to get all GPUs bitmap that need access the svm range. This helper will be used in the following patch to check if prange is mapped to all gpus.
Refactor svm_range_validate_and_map to use the helper function, no functional change.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 74 ++++++++++++++++------------ 1 file changed, 43 insertions(+), 31 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +43 -31
@@ -1169,6 +1169,44 @@ svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,list_add_tail (&pchild->child_list, &prange->child_list); } +static int +svm_range_need_access_gpus (unsigned long *bitmap,struct svm_range *prange) +{ + struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); +u32 gpuidx; + +if (p->xnack_enabled) { +bitmap_copy (bitmap, prange->bitmap_aip,MAX_GPU_INSTANCE ); + +/* If prefetch range to GPU, or GPU retry fault migrate range to +* GPU, which has ACCESS attribute to the range, create mapping +* on that GPU. +*/ +if (prange->actual_loc) { + gpuidx =kfd_process_gpuidx_from_gpuid (p, prange->actual_loc); +if (gpuidx <0 ) +return -EINVAL ; + +if (test_bit (gpuidx, prange->bitmap_access)) +bitmap_set (bitmap, gpuidx,1 ); + } + +/* +* If prange is already mapped or with always mapped flag, +* update mapping on GPUs with ACCESS attribute +*/ +if (bitmap_empty (bitmap,MAX_GPU_INSTANCE )) { +if (prange->mapped_to_gpu || + prange->flags &KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED ) +bitmap_copy (bitmap, prange->bitmap_access,MAX_GPU_INSTANCE ); + } + }else { +bitmap_or (bitmap, prange->bitmap_access, + prange->bitmap_aip,MAX_GPU_INSTANCE ); + } +return 0 ; +} +static bool svm_nodes_in_same_hive (struct kfd_node *node_a,struct kfd_node *node_b) {
@@ -1609,38 +1647,12 @@ static int svm_range_validate_and_map(struct mm_struct *mm,if (gpuidx <MAX_GPU_INSTANCE ) {bitmap_zero (ctx->bitmap,MAX_GPU_INSTANCE );bitmap_set (ctx->bitmap, gpuidx,1 ); - }else if (ctx->process->xnack_enabled) { -bitmap_copy (ctx->bitmap, prange->bitmap_aip,MAX_GPU_INSTANCE ); - -/* If prefetch range to GPU, or GPU retry fault migrate range to -* GPU, which has ACCESS attribute to the range, create mapping -* on that GPU. -*/ -if (prange->actual_loc) { - gpuidx =kfd_process_gpuidx_from_gpuid (ctx->process, - prange->actual_loc); -if (gpuidx <0 ) { -WARN_ONCE (1 ,"failed get device by id 0x%x\n" , - prange->actual_loc); - r = -EINVAL ; - goto free_ctx; - } -if (test_bit (gpuidx, prange->bitmap_access)) -bitmap_set (ctx->bitmap, gpuidx,1 ); - } - -/* -* If prange is already mapped or with always mapped flag, -* update mapping on GPUs with ACCESS attribute -*/ -if (bitmap_empty (ctx->bitmap,MAX_GPU_INSTANCE )) { -if (prange->mapped_to_gpu || - prange->flags &KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED ) -bitmap_copy (ctx->bitmap, prange->bitmap_access,MAX_GPU_INSTANCE ); - } }else { -bitmap_or (ctx->bitmap, prange->bitmap_access, - prange->bitmap_aip,MAX_GPU_INSTANCE ); + r =svm_range_need_access_gpus (ctx->bitmap, prange); +if (r) { +WARN_ONCE (1 ,"failed get device by id 0x%x\n" , prange->actual_loc); + goto free_ctx; + } }if (bitmap_empty (ctx->bitmap,MAX_GPU_INSTANCE )) {
Signature · 1 line
2.35.1
PATCH v4 2/7 drm/amdkfd: Add helper function align range start last
Calculate range start, last address aligned to the range granularity size. This removes the duplicate code, and the helper function will be used in the future patch to handle map, unmap to GPU based on range granularity. No functional change.
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 4 ++-- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 10 ++++------ drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 10 ++++++++++ 3 files changed, 16 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +2 -2
@@ -986,8 +986,8 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)/* Align migration range start and size to granularity size */ size =1UL << prange->granularity; - start =max (ALIGN_DOWN (addr, size), prange->start); - last =min (ALIGN (addr +1 , size) -1 , prange->last); + start =svm_range_align_start (addr, prange->start, size); + last =svm_range_align_last (addr, prange->last, size); r =svm_migrate_vram_to_ram (prange, vmf->vma->vm_mm, start, last,KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU , vmf->page);
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +4 -6
@@ -2698,10 +2698,8 @@ svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr, (vma->vm_start <= vma->vm_mm->start_stack && vma->vm_end >= vma->vm_mm->start_stack); - start_limit =max (vma->vm_start >>PAGE_SHIFT , - (unsigned long )ALIGN_DOWN (addr,2UL <<8 )); - end_limit =min (vma->vm_end >>PAGE_SHIFT , - (unsigned long )ALIGN (addr +1 ,2UL <<8 )); + start_limit =svm_range_align_start (addr, vma->vm_start >>PAGE_SHIFT ,2UL <<8 ); + end_limit =svm_range_align_last (addr, (vma->vm_end >>PAGE_SHIFT ) -1 ,2UL <<8 ) +1 ;/* First range that starts after the fault address */ node =interval_tree_iter_first (&p->svms.objects, addr +1 ,ULONG_MAX );if (node) {
@@ -3043,8 +3041,8 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,/* Align migration range start and size to granularity size */ size =1UL << prange->granularity; - start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start); - last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last); + start =svm_range_align_start (addr, prange->start, size); + last =svm_range_align_last (addr, prange->last, size);if (prange->actual_loc !=0 || best_loc !=0 ) { migration = true;
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +10 -0
@@ -159,6 +159,16 @@ static inline struct svm_range_bo *svm_range_bo_ref(struct svm_range_bo *svm_bo)return svm_bo; } +static inline u64 svm_range_align_start (u64 addr,u64 range_start,u64 align_size) +{ +return max (ALIGN_DOWN (addr, align_size), range_start); +} + +static inline u64 svm_range_align_last (u64 addr,u64 range_last,u64 align_size) +{ +return min (ALIGN (addr +1 , align_size) -1 , range_last); +} +int svm_range_list_init (struct kfd_process *p);void svm_range_list_fini (struct kfd_process *p); int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
Signature · 1 line
2.35.1
PATCH v4 3/7 drm/amdkfd: Add granularity size based bitmap map flag
In reply to Philip YangReplace prange->mapped_to_gpu with prange->bitmap_map[], which is per GPU flag and use bitmap bits based on prange granularity. Align map to GPU or unmap from GPU range size to granularity size and update the corresponding bitmap_map flag bits. This will optimize multiple GPU map, unmap and retry fault recover.
svm_range_partial_mapped is false only if no part of the range mapping on any GPUs.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 258 ++++++++++++++++++++++----- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 7 +- 2 files changed, 219 insertions(+), 46 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +213 -45
@@ -307,12 +307,12 @@ static void svm_range_free(struct svm_range *prange, bool do_unmap) KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, 0); } -/* free dma_addr array for each gpu */ +/* free dma_addr array, bitmap_map for each gpu */ for (gpuidx =0 ; gpuidx <MAX_GPU_INSTANCE ; gpuidx++) { -if (prange->dma_addr[gpuidx]) { +if (prange->dma_addr[gpuidx])kvfree (prange->dma_addr[gpuidx]); - prange->dma_addr[gpuidx] =NULL ; - } +if (prange->bitmap_map[gpuidx]) +bitmap_free (prange->bitmap_map[gpuidx]); }mutex_destroy (&prange->lock);
@@ -338,19 +338,38 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,uint64_t size = last - start +1 ;struct svm_range *prange;struct kfd_process *p; - - prange =kzalloc (sizeof (*prange),GFP_KERNEL ); -if (!prange) -return NULL ; +unsigned int nbits; +u32 gpuidx; p = container_of(svms, struct kfd_process, svms);if (!p->xnack_enabled && update_mem_usage &&amdgpu_amdkfd_reserve_mem_limit (NULL , size <<PAGE_SHIFT ,KFD_IOC_ALLOC_MEM_FLAGS_USERPTR ,0 )) {pr_info ("SVM mapping failed, exceeds resident system memory limit\n" ); -kfree (prange);return NULL ; } + + prange =kzalloc (sizeof (*prange),GFP_KERNEL ); +if (!prange) +return NULL ; + +svm_range_set_default_attributes (&prange->preferred_loc, + &prange->prefetch_loc, + &prange->granularity, &prange->flags); + + nbits =svm_range_map_nbits (start, last, prange->granularity); +pr_debug ("prange 0x%p [0x%llx 0x%llx] bitmap_map nbits %d\n" , prange, + start, last, nbits); +for_each_set_bit (gpuidx, p->svms.bitmap_supported, p->n_pdds) { + prange->bitmap_map[gpuidx] =bitmap_zalloc (nbits,GFP_KERNEL ); +if (!prange->bitmap_map[gpuidx]) { +while (gpuidx--) +bitmap_free (prange->bitmap_map[gpuidx]); +kfree (prange); +return NULL ; + } + } + prange->npages = size; prange->svms = svms; prange->start = start;
@@ -369,10 +388,6 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,bitmap_copy (prange->bitmap_access, svms->bitmap_supported,MAX_GPU_INSTANCE ); -svm_range_set_default_attributes (&prange->preferred_loc, - &prange->prefetch_loc, - &prange->granularity, &prange->flags); -pr_debug ("svms 0x%p [0x%llx 0x%llx]\n" , svms, start, last);return prange;
@@ -1017,6 +1032,51 @@ svm_range_split_nodes(struct svm_range *new, struct svm_range *old,return 0 ; } +static int +svm_range_split_bitmap_map (struct svm_range *new,struct svm_range *old, +u64 start,u64 last) +{ + struct kfd_process *p = container_of(new->svms, struct kfd_process, svms); +u32 new_nbits, old_nbits, old_nbits2; +unsigned long *bits; +u32 gpuidx; + + new_nbits =svm_range_map_nbits (new->start, new->last, new->granularity); + old_nbits =svm_range_map_nbits (old->start, old->last, old->granularity); + old_nbits2 =svm_range_map_nbits (start, last, old->granularity); + +pr_debug ("old 0x%p [0x%lx 0x%lx] => [0x%llx 0x%llx] nbits %d => %d\n" , + old, old->start, old->last, start, last, old_nbits, old_nbits2); +pr_debug ("new 0x%p [0x%lx 0x%lx] nbits %d\n" , new, new->start, new->last, + new_nbits); + +for_each_set_bit (gpuidx, p->svms.bitmap_supported, p->n_pdds) { + bits =bitmap_alloc (old_nbits2,GFP_KERNEL ); +if (!bits) +return -ENOMEM ; + +if (start == old->start) { +bitmap_copy (bits, old->bitmap_map[gpuidx], old_nbits2); +bitmap_shift_right (old->bitmap_map[gpuidx], + old->bitmap_map[gpuidx], + old_nbits - new_nbits, old_nbits); +bitmap_copy (new->bitmap_map[gpuidx], + old->bitmap_map[gpuidx], new_nbits); + }else { +bitmap_copy (new->bitmap_map[gpuidx], + old->bitmap_map[gpuidx], new_nbits); +bitmap_shift_right (old->bitmap_map[gpuidx], + old->bitmap_map[gpuidx], + old_nbits - old_nbits2, old_nbits); +bitmap_copy (bits, old->bitmap_map[gpuidx], old_nbits2); + } +bitmap_free (old->bitmap_map[gpuidx]); + old->bitmap_map[gpuidx] = bits; + } + +return 0 ; +} +/** * svm_range_split_adjust - split range and adjust *
@@ -1057,6 +1117,10 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old,return r; } + r =svm_range_split_bitmap_map (new, old, start, last); +if (r) +return r; + old->npages = last - start +1 ; old->start = start; old->last = last;
@@ -1064,7 +1128,6 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old, new->preferred_loc = old->preferred_loc; new->prefetch_loc = old->prefetch_loc; new->granularity = old->granularity; - new->mapped_to_gpu = old->mapped_to_gpu;bitmap_copy (new->bitmap_access, old->bitmap_access,MAX_GPU_INSTANCE );bitmap_copy (new->bitmap_aip, old->bitmap_aip,MAX_GPU_INSTANCE );
@@ -1169,6 +1232,65 @@ svm_range_add_child(struct svm_range *prange, struct mm_struct *mm,list_add_tail (&pchild->child_list, &prange->child_list); } +/** +* svm_range_partial_mapped_dev - check if prange mapped on the specific GPU +* +* @gpuidx: the GPU index to check +* @prange: prange to check +* @start: the start address in pages +* @last: the last address in pages +* +* Return: +* true: if any part of the range within [start, last] mapped on the GPU +* false: if the entire range [start, last] not mapped on the GPU +*/ +static bool +svm_range_partial_mapped_dev (u32 gpuidx,struct svm_range *prange, +u64 start,u64 last) +{ +u64 start_index, last_index; + + start =max_t (u64, start, prange->start); + last =min_t (u64, last, prange->last); +if (last < start) +return false; + + start_index = (start >> prange->granularity) - (prange->start >> prange->granularity); + last_index = (last >> prange->granularity) - (prange->start >> prange->granularity); +return find_next_bit (prange->bitmap_map[gpuidx], last_index +1 , + start_index) <= last_index; +} + +/** +* svm_range_partial_mapped - check if prange mapped on any GPU +* +* @prange: prange to check +* @start: the start address in pages +* @last: the last address in pages +* +* Return: +* true: if any part of prange mapped on any GPU currently +* false: if the entire range is not mapped on any GPU +*/ +static bool +svm_range_partial_mapped (struct svm_range *prange,u64 start,u64 last) +{ + struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); +struct svm_range *pchild; +u32 gpuidx; + +for_each_set_bit (gpuidx, p->svms.bitmap_supported, p->n_pdds) { +list_for_each_entry (pchild, &prange->child_list, child_list) { +if (svm_range_partial_mapped_dev (gpuidx, pchild, start, last)) +return true; + } + +if (svm_range_partial_mapped_dev (gpuidx, prange, start, last)) +return true; + } +return false; +} +static int svm_range_need_access_gpus (unsigned long *bitmap,struct svm_range *prange) {
@@ -1196,7 +1318,7 @@ svm_range_need_access_gpus(unsigned long *bitmap, struct svm_range *prange)* update mapping on GPUs with ACCESS attribute */ if (bitmap_empty (bitmap,MAX_GPU_INSTANCE )) { -if (prange->mapped_to_gpu || +if (svm_range_partial_mapped (prange, prange->start, prange->last) || prange->flags &KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED )bitmap_copy (bitmap, prange->bitmap_access,MAX_GPU_INSTANCE ); }
@@ -1336,6 +1458,48 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, fence); } +/** +* svm_range_complete_mapped - check if prange mapped on all GPUs completely +* +* @prange: prange to check +* +* Return: +* true: if the entire prange mapped completely on all GPUs that need access +* otherwise return false +*/ +static bool svm_range_complete_mapped (struct svm_range *prange) +{ +int nbits =svm_range_map_nbits (prange->start, prange->last, prange->granularity); +DECLARE_BITMAP (bitmap,MAX_GPU_INSTANCE ); +u32 gpuidx; +int r; + + r =svm_range_need_access_gpus (bitmap, prange); +if (r) +return false; + +for_each_set_bit (gpuidx, bitmap,MAX_GPU_INSTANCE ) +if (!bitmap_full (prange->bitmap_map[gpuidx], nbits)) +return false; +return true; +} + +static void +svm_range_update_mapped (u32 gpuidx,struct svm_range *prange, +u64 start,u64 last,bool mapped) +{ +u64 index, nbits; + + index = (start >> prange->granularity) - (prange->start >> prange->granularity); + nbits =svm_range_map_nbits (start, last, prange->granularity); +if (mapped) +bitmap_set (prange->bitmap_map[gpuidx], index, nbits); +else +bitmap_clear (prange->bitmap_map[gpuidx], index, nbits); +pr_debug ("prange 0x%p [0x%llx 0x%llx] update mapped %d nbits %lld gpu %d\n" , + prange, start, last, mapped, nbits, gpuidx); +} +static int svm_range_unmap_from_gpus (struct svm_range *prange,unsigned long start,unsigned long last,uint32_t trigger)
@@ -1347,29 +1511,28 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,uint32_t gpuidx;int r =0 ; -if (!prange->mapped_to_gpu) { -pr_debug ("prange 0x%p [0x%lx 0x%lx] not mapped to GPU\n" , - prange, prange->start, prange->last); -return 0 ; - } - -if (prange->start == start && prange->last == last) { -pr_debug ("unmap svms 0x%p prange 0x%p\n" , prange->svms, prange); - prange->mapped_to_gpu = false; - } -bitmap_or (bitmap, prange->bitmap_access, prange->bitmap_aip,MAX_GPU_INSTANCE ); p = container_of(prange->svms, struct kfd_process, svms);for_each_set_bit (gpuidx, bitmap,MAX_GPU_INSTANCE ) { -pr_debug ("unmap from gpu idx 0x%x\n" , gpuidx); pdd =kfd_process_device_from_gpuidx (p, gpuidx);if (!pdd) {pr_debug ("failed to find device idx %d\n" , gpuidx); -return -EINVAL ; +continue ; } +if (!svm_range_partial_mapped_dev (gpuidx, prange, start, last)) { +pr_debug ("svms 0x%p prange 0x%p [0x%lx 0x%lx] not mapped on gpu %d\n" , + prange->svms, prange, start, last, gpuidx); +continue ; + } + +svm_range_update_mapped (gpuidx, prange, start, last, false); + +pr_debug ("unmap svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu %d\n" , + prange->svms, prange, start, last, gpuidx); +kfd_smi_event_unmap_from_gpu (pdd->dev, p->lead_thread->pid, start, last, trigger);
@@ -1510,6 +1673,10 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,if (r)break ; +if (!r) +svm_range_update_mapped (gpuidx, prange, prange->start + offset, + prange->start + offset + npages -1 , true); +if (fence) { r =dma_fence_wait (fence, false);dma_fence_put (fence);
@@ -1745,9 +1912,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm, } } -if (!r && next == end) - prange->mapped_to_gpu = true; -svm_range_unlock (prange); addr = next;
@@ -1916,12 +2080,11 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,if (!p->xnack_enabled || (prange->flags &KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED )) {int evicted_ranges; -bool mapped = prange->mapped_to_gpu; + +if (!svm_range_partial_mapped (prange, start, last)) +return r;list_for_each_entry (pchild, &prange->child_list, child_list) { -if (!pchild->mapped_to_gpu) -continue ; - mapped = true;mutex_lock_nested (&pchild->lock,1 );if (pchild->start <= last && pchild->last >= start) { pr_debug("increment pchild invalid [0x%lx 0x%lx]\n",
@@ -1930,10 +2093,6 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm, }mutex_unlock (&pchild->lock); } - -if (!mapped) -return r; -if (prange->start <= last && prange->last >= start)atomic_inc (&prange->invalid);
@@ -1982,7 +2141,10 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,static struct svm_range *svm_range_clone (struct svm_range *old) { + struct kfd_process *p = container_of(old->svms, struct kfd_process, svms);struct svm_range *new; +u32 gpuidx; +int nbits; new =svm_range_new (old->svms, old->start, old->last, false);if (!new)
@@ -2004,8 +2166,13 @@ static struct svm_range *svm_range_clone(struct svm_range *old) new->prefetch_loc = old->prefetch_loc; new->actual_loc = old->actual_loc; new->granularity = old->granularity; - new->mapped_to_gpu = old->mapped_to_gpu; new->vram_pages = old->vram_pages; + nbits =svm_range_map_nbits (new->start, new->last, new->granularity); +for_each_set_bit (gpuidx, p->svms.bitmap_supported, p->n_pdds) { +bitmap_copy (new->bitmap_map[gpuidx], + old->bitmap_map[gpuidx], + nbits); + };bitmap_copy (new->bitmap_access, old->bitmap_access,MAX_GPU_INSTANCE );bitmap_copy (new->bitmap_aip, old->bitmap_aip,MAX_GPU_INSTANCE );
@@ -2126,7 +2293,7 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, next_start =min (node->last, last) +1 ;if (svm_range_is_same_attrs (p, prange, nattr, attrs) && - prange->mapped_to_gpu) { +svm_range_complete_mapped (prange)) {/* nothing to do */ }else if (node->start < start || node->last > last) {/* node intersects the update range and its attributes
@@ -3611,7 +3778,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,if (migrated && (!p->xnack_enabled || (prange->flags &KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED )) && - prange->mapped_to_gpu) { +svm_range_partial_mapped (prange, prange->start, prange->last)) {pr_debug ("restore_work will update mappings of GPUs\n" );mutex_unlock (&prange->migrate_mutex);continue ;
@@ -3622,8 +3789,8 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,continue ; } - flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu; - + flush_tlb = !migrated && update_mapping && +svm_range_partial_mapped (prange, prange->start, prange->last); r =svm_range_validate_and_map (mm, prange->start, prange->last, prange,MAX_GPU_INSTANCE , true, true, flush_tlb);if (r)
@@ -3639,8 +3806,9 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,pr_debug ("Remapping prange 0x%p [0x%lx 0x%lx]\n" , prange, prange->start, prange->last);mutex_lock (&prange->migrate_mutex); + flush_tlb =svm_range_partial_mapped (prange, prange->start, prange->last); r =svm_range_validate_and_map (mm, prange->start, prange->last, prange, -MAX_GPU_INSTANCE , true, true, prange->mapped_to_gpu); +MAX_GPU_INSTANCE , true, true, flush_tlb);if (r)pr_debug ("failed %d on remap svm range\n" , r);mutex_unlock (&prange->migrate_mutex);
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +6 -1
@@ -136,7 +136,7 @@ struct svm_range {struct list_head child_list;DECLARE_BITMAP (bitmap_access,MAX_GPU_INSTANCE );DECLARE_BITMAP (bitmap_aip,MAX_GPU_INSTANCE ); -bool mapped_to_gpu; +unsigned long *bitmap_map[MAX_GPU_INSTANCE ]; };static inline void svm_range_lock (struct svm_range *prange)
@@ -169,6 +169,11 @@ static inline u64 svm_range_align_last(u64 addr, u64 range_last, u64 align_size)return min (ALIGN (addr +1 , align_size) -1 , range_last); } +static inline u32 svm_range_map_nbits (u64 start,u64 last,u8 granularity) +{ +return (last >> granularity) - (start >> granularity) +1 ; +} +int svm_range_list_init (struct kfd_process *p);void svm_range_list_fini (struct kfd_process *p); int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
Signature · 1 line
2.35.1
PATCH v4 4/7 amd/amdkfd: Unmap range from GPU based on granularity
In reply to Philip YangWhen MMU notifier invalidate the range, align the start and last address to range granularity to unmap from GPU and update bitmap_map flag. Skip unmap from GPU if range is already unmapped based on bitmap_map flag. This avoids unmap 1 page from GPU and flush TLB, also solve the rocgdb CWSR migration related issue.
Unmap the range from cpu will remove the range and split the range, this cannot align the start and last address to range granularity. Change to split the range and bitmap_map flag first, then unmap the range from GPU. If unmapping from GPU first, the bitmap_map flag is updated, split range may get incorrect bitmap_map for the remaining ranges.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 42 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +29 -13
@@ -2114,6 +2114,13 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm, }else {unsigned long s, l;uint32_t trigger; +u64 size =1UL << prange->granularity; + +if (!svm_range_partial_mapped (prange, start, last)) { +pr_debug ("svms 0x%p [0x%lx 0x%lx] unmapped already\n" , + prange->svms, start, last); +return 0 ; + }if (event ==MMU_NOTIFY_MIGRATE ) trigger =KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE ;
@@ -2122,16 +2129,17 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,pr_debug ("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n" , prange->svms, start, last); +list_for_each_entry (pchild, &prange->child_list, child_list) {mutex_lock_nested (&pchild->lock,1 ); - s =max (start, pchild->start); - l =min (last, pchild->last); + s =svm_range_align_start (start, pchild->start, size); + l =svm_range_align_last (last, pchild->last, size);if (l >= s)svm_range_unmap_from_gpus (pchild, s, l, trigger);mutex_unlock (&pchild->lock); } - s =max (start, prange->start); - l =min (last, prange->last); + s =svm_range_align_start (start, prange->start, size); + l =svm_range_align_last (last, prange->last, size);if (l >= s)svm_range_unmap_from_gpus (prange, s, l, trigger); }
@@ -2645,24 +2653,32 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,list_for_each_entry (pchild, &prange->child_list, child_list) {mutex_lock_nested (&pchild->lock,1 ); - s =max (start, pchild->start); - l =min (last, pchild->last); -if (l >= s) -svm_range_unmap_from_gpus (pchild, s, l, trigger);svm_range_unmap_split (mm, prange, pchild, start, last);mutex_unlock (&pchild->lock); } - s =max (start, prange->start); - l =min (last, prange->last); -if (l >= s) -svm_range_unmap_from_gpus (prange, s, l, trigger);svm_range_unmap_split (mm, prange, prange, start, last); -if (unmap_parent)svm_range_add_list_work (svms, prange, mm,SVM_OP_UNMAP_RANGE );else svm_range_add_list_work (svms, prange, mm,SVM_OP_UPDATE_RANGE_NOTIFIER ); + +list_for_each_entry (pchild, &prange->child_list, child_list) { +if (pchild->work_item.op !=SVM_OP_UNMAP_RANGE ) +continue ; + + s =max (start, pchild->start); + l =min (last, pchild->last); +if (l >= s) +svm_range_unmap_from_gpus (pchild, s, l, trigger); + } +if (prange->work_item.op ==SVM_OP_UNMAP_RANGE ) { + s =max (start, prange->start); + l =min (last, prange->last); +if (l >= s) +svm_range_unmap_from_gpus (prange, s, l, trigger); + } +schedule_deferred_list_work (svms);kfd_unref_process (p);
Signature · 1 line
2.35.1
PATCH v4 5/7 drm/amdkfd: Change range granularity update bitmap_map
In reply to Philip YangWhen changing the svm range granularity, update the svm range bitmap_map based on new range granularity.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +48 -1
@@ -757,6 +757,53 @@ svm_range_check_attr(struct kfd_process *p,return 0 ; } +static void +svm_range_change_granularity (struct svm_range *prange,u8 value) +{ + struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); +u32 new_nbits, old_nbits, i, n; +unsigned long *new_bits, *old_bits; +u32 gpuidx; + +if (prange->granularity == value) +return ; + + old_nbits =svm_range_map_nbits (prange->start, prange->last, prange->granularity); + new_nbits =svm_range_map_nbits (prange->start, prange->last, value); +if (new_nbits > old_nbits) { + n = new_nbits / old_nbits; +if (new_nbits % old_nbits) + n++; + }else { + n = old_nbits / new_nbits; +if (old_nbits % new_nbits) + n++; + } + +pr_debug ("prange 0x%p [0x%lx 0x%lx] bitmap_map nbits %d -> %d\n" , + prange, prange->start, prange->last, old_nbits, new_nbits); + +for_each_set_bit (gpuidx, p->svms.bitmap_supported, p->n_pdds) { + old_bits = prange->bitmap_map[gpuidx]; +if (bitmap_empty (old_bits, old_nbits)) +continue ; + + new_bits =bitmap_zalloc (new_nbits,GFP_KERNEL ); +if (!new_bits) +return ; + +for_each_set_bit (i, old_bits, old_nbits) { +if (new_nbits > old_nbits) +bitmap_set (new_bits, i * n, n); +else +bitmap_set (new_bits, i / n,1 ); + } + prange->bitmap_map[gpuidx] = new_bits; +bitmap_free (old_bits); + } + prange->granularity = value; +} + static void svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange, uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
@@ -801,7 +848,7 @@ svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange, prange->flags &= ~attrs[i].value;break ;case KFD_IOCTL_SVM_ATTR_GRANULARITY : - prange->granularity =min_t (uint32_t, attrs[i].value,0x3F ); +svm_range_change_granularity (prange,min_t (u32, attrs[i].value,0x3F ));break ;default :WARN_ONCE (1 ,"svm_range_check_attrs wasn't called?" );
Signature · 1 line
2.35.1
PATCH v4 6/7 drm/amdkfd: Check bitmap_map flag to skip retry fault
In reply to Philip YangRemove prange validate_timestamp which is not accurate for multiple GPUs.
Use the bitmap_map flag to skip the retry fault from different pages of the same granularity range if the granularity range is already mapped on the specific GPU.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 24 ++++++++---------------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 1 - 2 files changed, 8 insertions(+), 17 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +8 -16
@@ -45,10 +45,6 @@#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1 -/* Long enough to ensure no retry fault comes after svm range is restored and -* page table is updated. -*/ -#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING (2UL * NSEC_PER_MSEC)#if IS_ENABLED (CONFIG_DYNAMIC_DEBUG )#define dynamic_svm_range_dump (svms) \ _dynamic_func_call_no_desc("svm_range_dump", svm_range_debug_dump, svms)
@@ -380,7 +376,6 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,INIT_LIST_HEAD (&prange->deferred_list);INIT_LIST_HEAD (&prange->child_list);atomic_set (&prange->invalid,0 ); - prange->validate_timestamp =0 ;mutex_init (&prange->migrate_mutex);mutex_init (&prange->lock);
@@ -1965,8 +1960,6 @@ static int svm_range_validate_and_map(struct mm_struct *mm, }svm_range_unreserve_bos (ctx); -if (!r) - prange->validate_timestamp =ktime_get_boottime (); free_ctx:kfree (ctx);
@@ -3226,15 +3219,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, goto out_unlock_mm; } -/* skip duplicate vm fault on different pages of same range */ -if (ktime_before (timestamp,ktime_add_ns (prange->validate_timestamp, -AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING ))) { -pr_debug ("svms 0x%p [0x%lx %lx] already restored\n" , - svms, prange->start, prange->last); - r =0 ; - goto out_unlock_mm; - } -/* __do_munmap removed VMA, return success as we are handling stale * retry fault. */
@@ -3260,6 +3244,14 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, goto out_unlock_mm; } +/* skip duplicate vm fault on different pages of same granularity range */ +if (svm_range_partial_mapped_dev (gpuidx, prange, addr, addr)) { +pr_debug ("svms 0x%p [0x%lx %lx] addr 0x%llx already mapped on gpu %d\n" , + svms, prange->start, prange->last, addr, gpuidx); + r =0 ; + goto out_unlock_mm; + } +mutex_lock (&prange->migrate_mutex); pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n",
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +0 -1
@@ -129,7 +129,6 @@ struct svm_range {uint32_t actual_loc;uint8_t granularity;atomic_t invalid; -ktime_t validate_timestamp;struct mmu_interval_notifier notifier;struct svm_work_list_item work_item;struct list_head deferred_list;
Signature · 1 line
2.35.1
PATCH v4 7/7 drm/amdkfd: Wait update sdma fence before tlb flush
In reply to Philip YangIf using sdma update GPU page table, kfd flush tlb does nothing if vm update fence callback doesn't update vm->tlb_seq. This works now because retry fault will come and update page table again and flush tlb finally.
With the bitmap_map flag, the retry fault recover will only update GPU page table once, have to wait sdma udate fence and then flush tlb.
No change if using CPU update GPU page table for large bar because no vm update fence.
Remove wait parameter in svm_range_validate_and_map because it is always called with true now.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +8 -9
@@ -1677,7 +1677,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,static int svm_range_map_to_gpus (struct svm_range *prange,unsigned long offset,unsigned long npages,bool readonly, -unsigned long *bitmap,bool wait,bool flush_tlb) +unsigned long *bitmap,bool flush_tlb) {struct kfd_process_device *pdd;struct amdgpu_device *bo_adev =NULL ;
@@ -1710,8 +1710,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset, r =svm_range_map_to_gpu (pdd, prange, offset, npages, readonly, prange->dma_addr[gpuidx], - bo_adev, wait ? &fence :NULL , - flush_tlb); + bo_adev, &fence, flush_tlb);if (r)break ;
@@ -1837,7 +1836,7 @@ static void *kfd_svm_page_owner(struct kfd_process *p, int32_t gpuidx)static int svm_range_validate_and_map (struct mm_struct *mm,unsigned long map_start,unsigned long map_last,struct svm_range *prange,int32_t gpuidx, -bool intr,bool wait,bool flush_tlb) +bool intr,bool flush_tlb) {struct svm_validate_context *ctx;unsigned long start, end, addr;
@@ -1950,7 +1949,7 @@ static int svm_range_validate_and_map(struct mm_struct *mm, offset = map_start_vma - prange->start; npages = map_last_vma - map_start_vma +1 ; r =svm_range_map_to_gpus (prange, offset, npages, readonly, - ctx->bitmap, wait, flush_tlb); + ctx->bitmap, flush_tlb); } }
@@ -2041,7 +2040,7 @@ static void svm_range_restore_work(struct work_struct *work)mutex_lock (&prange->migrate_mutex); r =svm_range_validate_and_map (mm, prange->start, prange->last, prange, -MAX_GPU_INSTANCE , false, true, false); +MAX_GPU_INSTANCE , false, false);if (r)pr_debug ("failed %d to map 0x%lx to gpus\n" , r, prange->start);
@@ -3303,7 +3302,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,mmap_read_lock (mm); r =svm_range_validate_and_map (mm, start, last, prange, gpuidx, false, - false, false); + false);if (r)pr_debug ("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n" , r, svms, start, last);
@@ -3847,7 +3846,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm, flush_tlb = !migrated && update_mapping &&svm_range_partial_mapped (prange, prange->start, prange->last); r =svm_range_validate_and_map (mm, prange->start, prange->last, prange, -MAX_GPU_INSTANCE , true, true, flush_tlb); +MAX_GPU_INSTANCE , true, flush_tlb);if (r)pr_debug ("failed %d to map svm range\n" , r);
@@ -3863,7 +3862,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,mutex_lock (&prange->migrate_mutex); flush_tlb =svm_range_partial_mapped (prange, prange->start, prange->last); r =svm_range_validate_and_map (mm, prange->start, prange->last, prange, -MAX_GPU_INSTANCE , true, true, flush_tlb); +MAX_GPU_INSTANCE , true, flush_tlb);if (r)pr_debug ("failed %d on remap svm range\n" , r);mutex_unlock (&prange->migrate_mutex);
Signature · 1 line
2.35.1