drm/amdkfd: device pgmap owner at the svm migrate init
Subsystems (3)
PATCH 1/11 drm/amdkfd: device pgmap owner at the svm migrate init
pgmap owner member at the svm migrate init could be referenced to either adev or hive, depending on device topology.
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3 -3
@@ -427,7 +427,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, migrate.start = start; migrate.end = end; migrate.flags =MIGRATE_VMA_SELECT_SYSTEM ; - migrate.pgmap_owner = adev; + migrate.pgmap_owner =SVM_ADEV_PGMAP_OWNER (adev); size =2 *sizeof (*migrate.src) +sizeof (uint64_t ) +sizeof (dma_addr_t); size *= npages;
@@ -649,7 +649,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, migrate.start = start; migrate.end = end; migrate.flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE ; - migrate.pgmap_owner = adev; + migrate.pgmap_owner =SVM_ADEV_PGMAP_OWNER (adev); size =2 *sizeof (*migrate.src) +sizeof (uint64_t ) +sizeof (dma_addr_t); size *= npages;
@@ -921,7 +921,7 @@ int svm_migrate_init(struct amdgpu_device *adev) pgmap->range.start = res->start; pgmap->range.end = res->end; pgmap->ops = &svm_migrate_pgmap_ops; - pgmap->owner = adev; + pgmap->owner =SVM_ADEV_PGMAP_OWNER (adev); pgmap->flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE ; r =devm_memremap_pages (adev->dev, pgmap);if (IS_ERR (r)) {
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +3 -0
@@ -35,6 +35,9 @@#include "amdgpu.h" #include "kfd_priv.h" +#define SVM_ADEV_PGMAP_OWNER (adev)\ + ((adev)->hive ? (void *)(adev)->hive : (void *)(adev)) +struct svm_range_bo {struct amdgpu_bo *bo;struct kref kref;
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/11 drm/amdkfd: add owner ref param to get hmm pages
The parameter is used in the dev_private_owner to decide if device pages in the range require to be migrated back to system memory, based if they are or not in the same memory domain.
In this case, this reference could come from the same memory domain with devices connected to the same hive.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +2 -1
@@ -160,7 +160,7 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, struct mm_struct *mm, struct page **pages, uint64_t start, uint64_t npages, struct hmm_range **phmm_range, bool readonly, - bool mmap_locked) + bool mmap_locked, void *owner) { struct hmm_range *hmm_range;unsigned long timeout;
@@ -185,6 +185,7 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, hmm_range->hmm_pfns = pfns; hmm_range->start = start; hmm_range->end = start + npages *PAGE_SIZE ; + hmm_range->dev_private_owner = owner;/* Assuming 512MB takes maxmium 1 second to fault page address */ timeout =max (npages >>17 ,1ULL ) *HMM_RANGE_DEFAULT_TIMEOUT ;
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +1 -1
@@ -34,7 +34,7 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, struct mm_struct *mm, struct page **pages, uint64_t start, uint64_t npages, struct hmm_range **phmm_range, bool readonly, - bool mmap_locked); + bool mmap_locked, void *owner);int amdgpu_hmm_range_get_pages_done (struct hmm_range *hmm_range);#if defined(CONFIG_HMM_MIRROR )
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
@@ -712,7 +712,7 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) readonly =amdgpu_ttm_tt_is_readonly (ttm); r =amdgpu_hmm_range_get_pages (&bo->notifier, mm, pages, start, ttm->num_pages, >t->range, readonly, - false); + false,NULL ); out_putmm:mmput (mm);
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2 -2
@@ -1416,7 +1416,7 @@ static int svm_range_validate_and_map(struct mm_struct *mm, r =amdgpu_hmm_range_get_pages (&prange->notifier, mm,NULL , prange->start <<PAGE_SHIFT , prange->npages, &hmm_range, - false, true); + false, true,NULL );if (r) {pr_debug ("failed %d to get svm range pages\n" , r); goto unreserve_out;
@@ -2716,7 +2716,7 @@ void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm) r =amdgpu_hmm_range_get_pages (&prange->notifier, mm,NULL , prange->start <<PAGE_SHIFT , prange->npages, &hmm_range, - false, true); + false, true,NULL );if (!r) {amdgpu_hmm_range_get_pages_done (hmm_range); prange->validated_once = true;
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/11 drm/amdkfd: set owner ref to svm range prefault
In reply to Alex Sierrasvm_range_prefault is called right before migrations to VRAM, to make sure pages are resident in system memory before the migration. With partial migrations, this reference is used by hmm range get pages to avoid migrating pages that are already in the same VRAM domain.
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 5 +++-- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 -1
@@ -519,7 +519,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc, prange->start, prange->last, best_loc);/* FIXME: workaround for page locking bug with invalid pages */ -svm_range_prefault (prange, mm); +svm_range_prefault (prange, mm,SVM_ADEV_PGMAP_OWNER (adev)); start = prange->start <<PAGE_SHIFT ; end = (prange->last +1 ) <<PAGE_SHIFT ;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +3 -2
@@ -2705,7 +2705,8 @@ svm_range_best_prefetch_location(struct svm_range *prange)/* FIXME: This is a workaround for page locking bug when some pages are * invalid during migration to VRAM */ -void svm_range_prefault (struct svm_range *prange,struct mm_struct *mm) +void svm_range_prefault (struct svm_range *prange,struct mm_struct *mm, +void *owner) {struct hmm_range *hmm_range;int r;
@@ -2716,7 +2717,7 @@ void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm) r =amdgpu_hmm_range_get_pages (&prange->notifier, mm,NULL , prange->start <<PAGE_SHIFT , prange->npages, &hmm_range, - false, true,NULL ); + false, true, owner);if (!r) {amdgpu_hmm_range_get_pages_done (hmm_range); prange->validated_once = true;
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +2 -1
@@ -176,7 +176,8 @@ void schedule_deferred_list_work(struct svm_range_list *svms);void svm_range_dma_unmap (struct device *dev,dma_addr_t *dma_addr,unsigned long offset,unsigned long npages);void svm_range_free_dma_mappings (struct svm_range *prange); -void svm_range_prefault (struct svm_range *prange,struct mm_struct *mm); +void svm_range_prefault (struct svm_range *prange,struct mm_struct *mm, +void *owner);struct kfd_process_device *svm_range_get_pdd_by_adev (struct svm_range *prange,struct amdgpu_device *adev);
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 4/11 drm/amdgpu: get owner ref in validate and map
In reply to Alex SierraGet the proper owner reference for amdgpu_hmm_range_get_pages function. This is useful for partial migrations. To avoid migrating back to system memory, VRAM pages, that are accessible by all devices in the same memory domain.
Ex. multiple devices in the same hive.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +24 -1
@@ -1337,6 +1337,17 @@ static void svm_range_unreserve_bos(struct svm_validate_context *ctx)ttm_eu_backoff_reservation (&ctx->ticket, &ctx->validate_list); } +static void *kfd_svm_page_owner (struct kfd_process *p,int32_t gpuidx) +{ +struct kfd_process_device *pdd; +struct amdgpu_device *adev; + + pdd =kfd_process_device_from_gpuidx (p, gpuidx); + adev = (struct amdgpu_device *)pdd->dev->kgd; + +return SVM_ADEV_PGMAP_OWNER (adev); +} +/* * Validation+GPU mapping with concurrent invalidation (MMU notifiers) *
@@ -1367,6 +1378,9 @@ static int svm_range_validate_and_map(struct mm_struct *mm, {struct svm_validate_context ctx;struct hmm_range *hmm_range; +struct kfd_process *p; +void *owner; +int32_t idx;int r =0 ; ctx.process = container_of(prange->svms, struct kfd_process, svms);
@@ -1413,10 +1427,19 @@ static int svm_range_validate_and_map(struct mm_struct *mm,svm_range_reserve_bos (&ctx);if (!prange->actual_loc) { + p = container_of(prange->svms, struct kfd_process, svms); + owner =kfd_svm_page_owner (p,find_first_bit (ctx.bitmap, +MAX_GPU_INSTANCE )); +for_each_set_bit (idx, ctx.bitmap,MAX_GPU_INSTANCE ) { +if (kfd_svm_page_owner (p, idx) != owner) { + owner =NULL ; +break ; + } + } r =amdgpu_hmm_range_get_pages (&prange->notifier, mm,NULL , prange->start <<PAGE_SHIFT , prange->npages, &hmm_range, - false, true,NULL ); + false, true, owner);if (r) {pr_debug ("failed %d to get svm range pages\n" , r); goto unreserve_out;
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 5/11 drm/amdkfd: use hmm range fault to get both domain pfns
In reply to Alex SierraNow that prange could have mixed domains (VRAM or SYSRAM), actual_loc nor svm_bo can not be used to check its current domain and eventually get its pfns to map them in GPU. Instead, pfns from both domains, are now obtained from hmm_range_fault through amdgpu_hmm_range_get_pages call. This is done everytime a GPU map occur.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 68 +++++++++++----------------- 1 file changed, 27 insertions(+), 41 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +27 -41
@@ -1426,42 +1426,38 @@ static int svm_range_validate_and_map(struct mm_struct *mm,svm_range_reserve_bos (&ctx); -if (!prange->actual_loc) { - p = container_of(prange->svms, struct kfd_process, svms); - owner =kfd_svm_page_owner (p,find_first_bit (ctx.bitmap, -MAX_GPU_INSTANCE )); -for_each_set_bit (idx, ctx.bitmap,MAX_GPU_INSTANCE ) { -if (kfd_svm_page_owner (p, idx) != owner) { - owner =NULL ; -break ; - } - } - r =amdgpu_hmm_range_get_pages (&prange->notifier, mm,NULL , - prange->start <<PAGE_SHIFT , - prange->npages, &hmm_range, - false, true, owner); -if (r) { -pr_debug ("failed %d to get svm range pages\n" , r); - goto unreserve_out; - } - - r =svm_range_dma_map (prange, ctx.bitmap, - hmm_range->hmm_pfns); -if (r) { -pr_debug ("failed %d to dma map range\n" , r); - goto unreserve_out; + p = container_of(prange->svms, struct kfd_process, svms); + owner =kfd_svm_page_owner (p,find_first_bit (ctx.bitmap, +MAX_GPU_INSTANCE )); +for_each_set_bit (idx, ctx.bitmap,MAX_GPU_INSTANCE ) { +if (kfd_svm_page_owner (p, idx) != owner) { + owner =NULL ; +break ; } + } + r =amdgpu_hmm_range_get_pages (&prange->notifier, mm,NULL , + prange->start <<PAGE_SHIFT , + prange->npages, &hmm_range, + false, true, owner); +if (r) { +pr_debug ("failed %d to get svm range pages\n" , r); + goto unreserve_out; + } - prange->validated_once = true; + r =svm_range_dma_map (prange, ctx.bitmap, + hmm_range->hmm_pfns); +if (r) { +pr_debug ("failed %d to dma map range\n" , r); + goto unreserve_out; } + prange->validated_once = true; +svm_range_lock (prange); -if (!prange->actual_loc) { -if (amdgpu_hmm_range_get_pages_done (hmm_range)) { -pr_debug ("hmm update the range, need validate again\n" ); - r = -EAGAIN ; - goto unlock_out; - } +if (amdgpu_hmm_range_get_pages_done (hmm_range)) { +pr_debug ("hmm update the range, need validate again\n" ); + r = -EAGAIN ; + goto unlock_out; }if (!list_empty (&prange->child_list)) {pr_debug ("range split by unmap in parallel, validate again\n" );
@@ -2785,16 +2781,6 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange, best_loc == prange->actual_loc)return 0 ; -/* -* Prefetch to GPU without host access flag, set actual_loc to gpu, then -* validate on gpu and map to gpus will be handled afterwards. -*/ -if (best_loc && !prange->actual_loc && - !(prange->flags &KFD_IOCTL_SVM_FLAG_HOST_ACCESS )) { - prange->actual_loc = best_loc; -return 0 ; - } -if (!best_loc) { r =svm_migrate_vram_to_ram (prange, mm); *migrated = !r;
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 6/11 drm/amdkfd: classify and map mixed svm range pages in GPU
In reply to Alex Sierra[Why]
svm ranges can have mixed pages from device or system memory. A good example is, after a prange has been allocated in VRAM and a copy-on-write is triggered by a fork. This invalidates some pages inside the prange. Endding up in mixed pages.
[How]
By classifying each page inside a prange, based on its type. Device or system memory, during dma mapping call. If page corresponds to VRAM domain, a flag is set to its dma_addr entry for each GPU. Then, at the GPU page table mapping. All group of contiguous pages within the same type are mapped with their proper pte flags.
v2:
Instead of using ttm_res to calculate vram pfns in the svm_range. It is now done by setting the vram real physical address into drm_addr array. This makes more flexible VRAM management, plus removes the need to have a BO reference in the svm_range.
v3:
Remove mapping member from svm_range
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 87 +++++++++++++++++----------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 +- 2 files changed, 55 insertions(+), 34 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +54 -33
@@ -119,28 +119,40 @@ static void svm_range_remove_notifier(struct svm_range *prange) }static int -svm_range_dma_map_dev (struct device *dev,dma_addr_t **dma_addr, -unsigned long *hmm_pfns,uint64_t npages) +svm_range_dma_map_dev (struct amdgpu_device *adev,struct svm_range *prange, +unsigned long *hmm_pfns,uint32_t gpuidx) {enum dma_data_direction dir =DMA_BIDIRECTIONAL ; -dma_addr_t *addr = *dma_addr; +dma_addr_t *addr = prange->dma_addr[gpuidx]; +struct device *dev = adev->dev;struct page *page;int i, r;if (!addr) { - addr =kvmalloc_array (npages,sizeof (*addr), + addr =kvmalloc_array (prange->npages,sizeof (*addr),GFP_KERNEL | __GFP_ZERO);if (!addr)return -ENOMEM ; - *dma_addr = addr; + prange->dma_addr[gpuidx] = addr; } -for (i =0 ; i < npages; i++) { +for (i =0 ; i < prange->npages; i++) {if (WARN_ONCE (addr[i] && !dma_mapping_error (dev, addr[i]),"leaking dma mapping\n" ))dma_unmap_page (dev, addr[i],PAGE_SIZE , dir); page =hmm_pfn_to_page (hmm_pfns[i]); +if (is_zone_device_page (page)) { +struct amdgpu_device *bo_adev = +amdgpu_ttm_adev (prange->svm_bo->bo->tbo.bdev); + + addr[i] = (hmm_pfns[i] <<PAGE_SHIFT ) + + bo_adev->vm_manager.vram_base_offset - + bo_adev->kfd.dev->pgmap.range.start; + addr[i] |=SVM_RANGE_VRAM_DOMAIN ; +pr_debug ("vram address detected: 0x%llx\n" , addr[i]); +continue ; + } addr[i] =dma_map_page (dev, page,0 ,PAGE_SIZE , dir); r =dma_mapping_error (dev, addr[i]);if (r) {
@@ -175,8 +187,7 @@ svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap, } adev = (struct amdgpu_device *)pdd->dev->kgd; - r =svm_range_dma_map_dev (adev->dev, &prange->dma_addr[gpuidx], - hmm_pfns, prange->npages); + r =svm_range_dma_map_dev (adev, prange, hmm_pfns, gpuidx);if (r)break ; }
@@ -1020,21 +1031,22 @@ svm_range_split_by_granularity(struct kfd_process *p, struct mm_struct *mm, }static uint64_t -svm_range_get_pte_flags (struct amdgpu_device *adev,struct svm_range *prange) +svm_range_get_pte_flags (struct amdgpu_device *adev,struct svm_range *prange, +int domain) {struct amdgpu_device *bo_adev;uint32_t flags = prange->flags;uint32_t mapping_flags =0 ;uint64_t pte_flags; -bool snoop = !prange->ttm_res; +bool snoop = (domain !=SVM_RANGE_VRAM_DOMAIN );bool coherent = flags &KFD_IOCTL_SVM_FLAG_COHERENT ; -if (prange->svm_bo && prange->ttm_res) +if (domain ==SVM_RANGE_VRAM_DOMAIN ) bo_adev =amdgpu_ttm_adev (prange->svm_bo->bo->tbo.bdev);switch (adev->asic_type) {case CHIP_ARCTURUS : -if (prange->svm_bo && prange->ttm_res) { +if (domain ==SVM_RANGE_VRAM_DOMAIN ) {if (bo_adev == adev) { mapping_flags |= coherent ?AMDGPU_VM_MTYPE_CC :AMDGPU_VM_MTYPE_RW ;
@@ -1050,7 +1062,7 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) }break ;case CHIP_ALDEBARAN : -if (prange->svm_bo && prange->ttm_res) { +if (domain ==SVM_RANGE_VRAM_DOMAIN ) {if (bo_adev == adev) { mapping_flags |= coherent ?AMDGPU_VM_MTYPE_CC :AMDGPU_VM_MTYPE_RW ;
@@ -1080,14 +1092,14 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) mapping_flags |=AMDGPU_VM_PAGE_EXECUTABLE ; pte_flags =AMDGPU_PTE_VALID ; - pte_flags |= prange->ttm_res ?0 :AMDGPU_PTE_SYSTEM ; + pte_flags |= (domain ==SVM_RANGE_VRAM_DOMAIN ) ?0 :AMDGPU_PTE_SYSTEM ; pte_flags |= snoop ?AMDGPU_PTE_SNOOPED :0 ; pte_flags |=amdgpu_gem_va_map_flags (adev, mapping_flags);pr_debug ("svms 0x%p [0x%lx 0x%lx] vram %d PTE 0x%llx mapping 0x%x\n" , prange->svms, prange->start, prange->last, - prange->ttm_res ?1 :0 , pte_flags, mapping_flags); + (domain ==SVM_RANGE_VRAM_DOMAIN ) ?1 :0 , pte_flags, mapping_flags);return pte_flags; }
@@ -1158,31 +1170,41 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,struct amdgpu_bo_va bo_va;bool table_freed = false;uint64_t pte_flags; +unsigned long last_start; +int last_domain;int r =0 ; +int64_t i;pr_debug ("svms 0x%p [0x%lx 0x%lx]\n" , prange->svms, prange->start, prange->last); -if (prange->svm_bo && prange->ttm_res) { +if (prange->svm_bo && prange->ttm_res) bo_va.is_xgmi =amdgpu_xgmi_same_hive (adev, bo_adev); - prange->mapping.bo_va = &bo_va; - } - prange->mapping.start = prange->start; - prange->mapping.last = prange->last; - prange->mapping.offset = prange->ttm_res ? prange->offset :0 ; - pte_flags =svm_range_get_pte_flags (adev, prange); + last_start = prange->start; +for (i =0 ; i < prange->npages; i++) { + last_domain = dma_addr[i] &SVM_RANGE_VRAM_DOMAIN ; + dma_addr[i] &= ~SVM_RANGE_VRAM_DOMAIN ; +if ((prange->start + i) < prange->last && + last_domain == (dma_addr[i +1 ] &SVM_RANGE_VRAM_DOMAIN )) +continue ; - r =amdgpu_vm_bo_update_mapping (adev, bo_adev, vm, false, false,NULL , - prange->mapping.start, - prange->mapping.last, pte_flags, - prange->mapping.offset, - prange->ttm_res, - dma_addr, &vm->last_update, - &table_freed); -if (r) { -pr_debug ("failed %d to map to gpu 0x%lx\n" , r, prange->start); - goto out; +pr_debug ("Mapping range [0x%lx 0x%llx] on domain: %s\n" , + last_start, prange->start + i, last_domain ?"GPU" :"CPU" ); + pte_flags =svm_range_get_pte_flags (adev, prange, last_domain); + r =amdgpu_vm_bo_update_mapping (adev, bo_adev, vm, false, false,NULL , + last_start, + prange->start + i, pte_flags, + last_start - prange->start, +NULL , + dma_addr, + &vm->last_update, + &table_freed); +if (r) { +pr_debug ("failed %d to map to gpu 0x%lx\n" , r, prange->start); + goto out; + } + last_start = prange->start + i +1 ; } r =amdgpu_vm_update_pdes (adev, vm, false);
@@ -1203,7 +1225,6 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, p->pasid, TLB_FLUSH_LEGACY); } out: - prange->mapping.bo_va =NULL ;return r; }
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +1 -1
@@ -35,6 +35,7 @@#include "amdgpu.h" #include "kfd_priv.h" +#define SVM_RANGE_VRAM_DOMAIN (1UL << 0)#define SVM_ADEV_PGMAP_OWNER (adev)\ ((adev)->hive ? (void *)(adev)->hive : (void *)(adev))
@@ -113,7 +114,6 @@ struct svm_range {struct list_head update_list;struct list_head remove_list;struct list_head insert_list; -struct amdgpu_bo_va_mapping mapping;uint64_t npages;dma_addr_t *dma_addr[MAX_GPU_INSTANCE ];struct ttm_resource *ttm_res;
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 7/11 drm/amdkfd: skip invalid pages during migrations
In reply to Alex SierraInvalid pages can be the result of pages that have been migrated already due to copy-on-write procedure or pages that were never migrated to VRAM in first place. This is not an issue anymore, as pranges now support mixed memory domains (CPU/GPU).
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 38 +++++++++++------------- 1 file changed, 18 insertions(+), 20 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +18 -20
@@ -420,7 +420,6 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,size_t size;void *buf;int r = -ENOMEM ; -int retry =0 ;memset (&migrate,0 ,sizeof (migrate)); migrate.vma = vma;
@@ -439,7 +438,6 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, migrate.dst = migrate.src + npages; scratch = (dma_addr_t *)(migrate.dst + npages); -retry: r =migrate_vma_setup (&migrate);if (r) { pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",
@@ -447,17 +445,9 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, goto out_free; }if (migrate.cpages != npages) { -pr_debug ("collect 0x%lx/0x%llx pages, retry\n" , migrate.cpages, +pr_debug ("Partial migration. 0x%lx/0x%llx pages can be migrated\n" , + migrate.cpages, npages); -migrate_vma_finalize (&migrate); -if (retry++ >=3 ) { - r = -ENOMEM ; -pr_debug ("failed %d migrate svms 0x%p [0x%lx 0x%lx]\n" , - r, prange->svms, prange->start, prange->last); - goto out_free; - } - - goto retry; }if (migrate.cpages) {
@@ -554,9 +544,8 @@ static void svm_migrate_page_free(struct page *page)static int svm_migrate_copy_to_ram (struct amdgpu_device *adev,struct svm_range *prange,struct migrate_vma *migrate,struct dma_fence **mfence, -dma_addr_t *scratch) +dma_addr_t *scratch,uint64_t npages) { -uint64_t npages = migrate->cpages;struct device *dev = adev->dev;uint64_t *src;dma_addr_t *dst;
@@ -573,15 +562,23 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, src = (uint64_t *)(scratch + npages); dst = scratch; -for (i =0 , j =0 ; i < npages; i++, j++, addr +=PAGE_SIZE ) { +for (i =0 , j =0 ; i < npages; i++, addr +=PAGE_SIZE ) {struct page *spage; spage =migrate_pfn_to_page (migrate->src[i]); -if (!spage) { -pr_debug ("failed get spage svms 0x%p [0x%lx 0x%lx]\n" , +if (!spage || !is_zone_device_page (spage)) { +pr_debug ("invalid page. Could be in CPU already svms 0x%p [0x%lx 0x%lx]\n" , prange->svms, prange->start, prange->last); - r = -ENOMEM ; - goto out_oom; +if (j) { + r =svm_migrate_copy_memory_gart (adev, dst + i - j, + src + i - j, j, +FROM_VRAM_TO_RAM , + mfence); +if (r) + goto out_oom; + j =0 ; + } +continue ; } src[i] =svm_migrate_addr (adev, spage);if (i >0 && src[i] != src[i -1 ] +PAGE_SIZE ) {
@@ -614,6 +611,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, migrate->dst[i] =migrate_pfn (page_to_pfn (dpage)); migrate->dst[i] |=MIGRATE_PFN_LOCKED ; + j++; } r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
@@ -672,7 +670,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,if (migrate.cpages) { r =svm_migrate_copy_to_ram (adev, prange, &migrate, &mfence, - scratch); + scratch, npages);migrate_vma_pages (&migrate);svm_migrate_copy_done (adev, mfence);migrate_vma_finalize (&migrate);
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 8/11 drm/amdkfd: skip migration for pages already in VRAM
In reply to Alex SierraMigration skipped for pages that are already in VRAM domain. These could be the result of previous partial migrations to SYS RAM, and prefetch back to VRAM.
Ex. Coherent pages in VRAM that were not written/invalidated after a copy-on-write.
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +7 -9
@@ -329,15 +329,13 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange,for (i = j =0 ; i < npages; i++) {struct page *spage; - dst[i] = vram_addr + (j <<PAGE_SHIFT ); - migrate->dst[i] =svm_migrate_addr_to_pfn (adev, dst[i]); -svm_migrate_get_vram_page (prange, migrate->dst[i]); - - migrate->dst[i] =migrate_pfn (migrate->dst[i]); - migrate->dst[i] |=MIGRATE_PFN_LOCKED ; - -if (migrate->src[i] &MIGRATE_PFN_VALID ) { - spage =migrate_pfn_to_page (migrate->src[i]); + spage =migrate_pfn_to_page (migrate->src[i]); +if (spage && !is_zone_device_page (spage)) { + dst[i] = vram_addr + (j <<PAGE_SHIFT ); + migrate->dst[i] =svm_migrate_addr_to_pfn (adev, dst[i]); +svm_migrate_get_vram_page (prange, migrate->dst[i]); + migrate->dst[i] =migrate_pfn (migrate->dst[i]); + migrate->dst[i] |=MIGRATE_PFN_LOCKED ; src[i] =dma_map_page (dev, spage,0 ,PAGE_SIZE ,DMA_TO_DEVICE ); r =dma_mapping_error (dev, src[i]);
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 9/11 drm/amdkfd: add invalid pages debug at vram migration
In reply to Alex SierraThis is for debug purposes only.
It conditionally generates partial migrations to test mixed CPU/GPU memory domain pages in a prange easily.
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +14 -0
@@ -401,6 +401,20 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange, } } +#ifdef DEBUG_FORCE_MIXED_DOMAINS +for (i =0 , j =0 ; i < npages; i +=4 , j++) { +if (j &1 ) +continue ; +svm_migrate_put_vram_page (adev, dst[i]); + migrate->dst[i] =0 ; +svm_migrate_put_vram_page (adev, dst[i +1 ]); + migrate->dst[i +1 ] =0 ; +svm_migrate_put_vram_page (adev, dst[i +2 ]); + migrate->dst[i +2 ] =0 ; +svm_migrate_put_vram_page (adev, dst[i +3 ]); + migrate->dst[i +3 ] =0 ; + } +#endif out:return r; }
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 10/11 drm/amdkfd: Maintain svm_bo reference in page->zone_device_data
In reply to Alex SierraEach zone-device page holds a reference to the SVM BO that manages its backing storage. This is necessary to correctly hold on to the BO in case zone_device pages are shared with a child-process.
--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 10 ++++++++-- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 10 +--------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 9 +++++++++ 3 files changed, 18 insertions(+), 11 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +8 -2
@@ -245,7 +245,8 @@ svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)struct page *page; page =pfn_to_page (pfn); - page->zone_device_data = prange; +svm_range_bo_ref (prange->svm_bo); + page->zone_device_data = prange->svm_bo;get_page (page);lock_page (page); }
@@ -550,7 +551,12 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,static void svm_migrate_page_free (struct page *page) { -/* Keep this function to avoid warning */ +struct svm_range_bo *svm_bo = page->zone_device_data; + +if (svm_bo) { +pr_debug ("svm_bo ref left: %d\n" ,kref_read (&svm_bo->kref)); +svm_range_bo_unref (svm_bo); + } } static int
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +1 -9
@@ -312,14 +312,6 @@ static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)return true; } -static struct svm_range_bo *svm_range_bo_ref (struct svm_range_bo *svm_bo) -{ -if (svm_bo) -kref_get (&svm_bo->kref); - -return svm_bo; -} -static void svm_range_bo_release (struct kref *kref) {struct svm_range_bo *svm_bo;
@@ -358,7 +350,7 @@ static void svm_range_bo_release(struct kref *kref)kfree (svm_bo); } -static void svm_range_bo_unref (struct svm_range_bo *svm_bo) +void svm_range_bo_unref (struct svm_range_bo *svm_bo) {if (!svm_bo)return ;
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +9 -0
@@ -150,6 +150,14 @@ static inline void svm_range_unlock(struct svm_range *prange)mutex_unlock (&prange->lock); } +static inline struct svm_range_bo *svm_range_bo_ref (struct svm_range_bo *svm_bo) +{ +if (svm_bo) +kref_get (&svm_bo->kref); + +return svm_bo; +} +int svm_range_list_init (struct kfd_process *p);void svm_range_list_fini (struct kfd_process *p); int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
@@ -186,6 +194,7 @@ svm_range_get_pdd_by_adev(struct svm_range *prange, struct amdgpu_device *adev);*/ #define KFD_IS_SVM_API_SUPPORTED (dev) ((dev)->pgmap.type != 0) +void svm_range_bo_unref (struct svm_range_bo *svm_bo);#else struct kfd_process ;
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 11/11 drm/amdkfd: inc counter on child ranges with xnack off
In reply to Alex SierraDuring GPU page table invalidation with xnack off, new ranges split may occur concurrently in the same prange. Creating a new child per split. Each child should also increment its invalid counter, to assure GPU page table updates in these ranges.
--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +7 -1
@@ -1622,6 +1622,7 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm, unsigned long start, unsigned long last) {struct svm_range_list *svms = prange->svms; +struct svm_range *pchild;struct kfd_process *p;int r =0 ;
@@ -1633,6 +1634,12 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,if (!p->xnack_enabled) {int evicted_ranges; +list_for_each_entry (pchild, &prange->child_list, child_list) { +mutex_lock_nested (&pchild->lock,1 ); +atomic_inc (&pchild->invalid); +mutex_unlock (&pchild->lock); + } +atomic_inc (&prange->invalid); evicted_ranges =atomic_inc_return (&svms->evicted_ranges);if (evicted_ranges !=1 )
@@ -1650,7 +1657,6 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,schedule_delayed_work (&svms->restore_work,msecs_to_jiffies (AMDGPU_SVM_RANGE_RESTORE_DELAY_MS )); }else { -struct svm_range *pchild;unsigned long s, l; pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
Signature · 6 lines
2.32.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Felix Kuehling
maintainerRe: [PATCH 11/11] drm/amdkfd: inc counter on child ranges with xnack off
Am 2021-06-29 um 2:02 p.m. schrieb Alex Sierra:
Quoted 24 lines
During GPU page table invalidation with xnack off, new ranges split may occur concurrently in the same prange. Creating a new child per split. Each child should also increment its invalid counter, to assure GPU page table updates in these ranges.
Signed-off-by Alex Sierra <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdkfd/kfd_svm.c +1
-0@@ -1622,6 +1622,7 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm, unsigned long start, unsigned long last) {struct svm_range_list *svms = prange->svms; +struct svm_range *pchild;struct kfd_process *p;int r =0 ;
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_evict(struct svm_range *prange, struct mm_struct *mm, · line 1638
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +2
-0@@ -1633,6 +1634,12 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,if (!p->xnack_enabled) {int evicted_ranges; +list_for_each_entry (pchild, &prange->child_list, child_list) { +mutex_lock_nested (&pchild->lock,1 );
You should check the address range of the child range and only increment pchild->invalid if it intersects with start..last.
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_evict(struct svm_range *prange, struct mm_struct *mm, · line 1643
Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +4
-0+atomic_inc (&pchild->invalid); +mutex_unlock (&pchild->lock); + } +atomic_inc (&prange->invalid);
Also add an address range check for the parent range.
Please make this the first patch in the series, or review/submit it separately from the series. The series is about the reorganization of VRAM management. This patch is an unrelated bug-fix.
Regards,
Felix
drivers/gpu/drm/amd/amdkfd/kfd_svm.c svm_range_evict(struct svm_range *prange, struct mm_struct *mm, · line 1662
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +0
-1evicted_ranges =atomic_inc_return (&svms->evicted_ranges);if (evicted_ranges !=1 )@@ -1650,7 +1657,6 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,schedule_delayed_work (&svms->restore_work,msecs_to_jiffies (AMDGPU_SVM_RANGE_RESTORE_DELAY_MS )); }else { -struct svm_range *pchild;unsigned long s, l; pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Felix Kuehling
maintainerAm 2021-06-29 um 2:01 p.m. schrieb Alex Sierra:
Quoted 2 lines
pgmap owner member at the svm migrate init could be referenced to either adev or hive, depending on device topology.
Please update the commit description before submitting the change:
GPUs in the same XGMI hive have direct access to all members' VRAM. When mapping memory to a GPU, we don't need hmm_range_fault to fault device-private pages in the same hive back to the host. Identifying the page owner as the hive, rather than the individual GPU, accomplishes this.
Quoted 44 lines
Signed-off-by Alex Sierra <[email protected]>Reviewed-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-)drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3
-3@@ -427,7 +427,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, migrate.start = start; migrate.end = end; migrate.flags =MIGRATE_VMA_SELECT_SYSTEM ; - migrate.pgmap_owner = adev; + migrate.pgmap_owner =SVM_ADEV_PGMAP_OWNER (adev); size =2 *sizeof (*migrate.src) +sizeof (uint64_t ) +sizeof (dma_addr_t); size *= npages;@@ -649,7 +649,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, migrate.start = start; migrate.end = end; migrate.flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE ; - migrate.pgmap_owner = adev; + migrate.pgmap_owner =SVM_ADEV_PGMAP_OWNER (adev); size =2 *sizeof (*migrate.src) +sizeof (uint64_t ) +sizeof (dma_addr_t); size *= npages;@@ -921,7 +921,7 @@ int svm_migrate_init(struct amdgpu_device *adev) pgmap->range.start = res->start; pgmap->range.end = res->end; pgmap->ops = &svm_migrate_pgmap_ops; - pgmap->owner = adev; + pgmap->owner =SVM_ADEV_PGMAP_OWNER (adev); pgmap->flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE ; r =devm_memremap_pages (adev->dev, pgmap);if (IS_ERR (r)) {drivers/gpu/drm/amd/amdkfd/kfd_svm.h +0
-0
drivers/gpu/drm/amd/amdkfd/kfd_svm.h line 43
Quoted 9 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +3
-0@@ -35,6 +35,9 @@#include "amdgpu.h" #include "kfd_priv.h" +#define SVM_ADEV_PGMAP_OWNER (adev)\ + ((adev)->hive ? (void *)(adev)->hive : (void *)(adev)) +struct svm_range_bo {struct amdgpu_bo *bo;struct kref kref;
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Other than the updated patch description for patch 1, the series (patches 1-10) is
Patch 11 was already reviewed and revised separately.
Thanks,
Felix
Am 2021-06-30 um 12:00 a.m. schrieb Felix Kuehling:
Quoted 57 lines
Am 2021-06-29 um 2:01 p.m. schrieb Alex Sierra:
Quoted 2 lines
pgmap owner member at the svm migrate init could be referenced to either adev or hive, depending on device topology.
Please update the commit description before submitting the change:
GPUs in the same XGMI hive have direct access to all members' VRAM. When mapping memory to a GPU, we don't need
hmm_range_faultto fault device-private pages in the same hive back to the host. Identifying the page owner as the hive, rather than the individual GPU, accomplishes this.Quoted 46 lines
Signed-off-by Alex Sierra <[email protected]>Reviewed-by Felix Kuehling <[email protected]>--- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 6 +++--- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-)drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +3
-3@@ -427,7 +427,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, migrate.start = start; migrate.end = end; migrate.flags =MIGRATE_VMA_SELECT_SYSTEM ; - migrate.pgmap_owner = adev; + migrate.pgmap_owner =SVM_ADEV_PGMAP_OWNER (adev); size =2 *sizeof (*migrate.src) +sizeof (uint64_t ) +sizeof (dma_addr_t); size *= npages;@@ -649,7 +649,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, migrate.start = start; migrate.end = end; migrate.flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE ; - migrate.pgmap_owner = adev; + migrate.pgmap_owner =SVM_ADEV_PGMAP_OWNER (adev); size =2 *sizeof (*migrate.src) +sizeof (uint64_t ) +sizeof (dma_addr_t); size *= npages;@@ -921,7 +921,7 @@ int svm_migrate_init(struct amdgpu_device *adev) pgmap->range.start = res->start; pgmap->range.end = res->end; pgmap->ops = &svm_migrate_pgmap_ops; - pgmap->owner = adev; + pgmap->owner =SVM_ADEV_PGMAP_OWNER (adev); pgmap->flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE ; r =devm_memremap_pages (adev->dev, pgmap);if (IS_ERR (r)) {drivers/gpu/drm/amd/amdkfd/kfd_svm.h +0
-0@@ -35,6 +35,9 @@#include "amdgpu.h" #include "kfd_priv.h"
Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.h line 43
Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +3
-0+#define SVM_ADEV_PGMAP_OWNER (adev)\ + ((adev)->hive ? (void *)(adev)->hive : (void *)(adev)) +struct svm_range_bo {struct amdgpu_bo *bo;struct kref kref;_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx