drm/amdgpu: cleanup initializing gtt_size
Subsystems (2)
PATCH 1/3 drm/amdgpu: cleanup initializing gtt_size
From: Christian König <[email protected]>
Stop spreading the code over all GMC generations.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 20 ++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 10 +--------- drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 10 +--------- drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 10 +--------- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 10 +--------- 6 files changed, 25 insertions(+), 36 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
@@ -561,6 +561,7 @@ struct amdgpu_gart {const struct amdgpu_gart_funcs *gart_funcs; }; +void amdgpu_gart_set_defaults (struct amdgpu_device *adev);int amdgpu_gart_table_ram_alloc (struct amdgpu_device *adev);void amdgpu_gart_table_ram_free (struct amdgpu_device *adev);int amdgpu_gart_table_vram_alloc (struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +20 -0
@@ -52,6 +52,26 @@/* * Common GART table functions. */ + +/** +* amdgpu_gart_set_defaults - set the default gtt_size +* +* @adev: amdgpu_device pointer +* +* Set the default gtt_size based on parameters and available VRAM. +*/ +void amdgpu_gart_set_defaults (struct amdgpu_device *adev) +{ +/* unless the user had overridden it, set the gart +* size equal to the 1024 or vram, whichever is larger. +*/ +if (amdgpu_gart_size ==-1 ) + adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), + adev->mc.mc_vram_size); +else + adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; +} +/** * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table *
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +1 -9
@@ -321,15 +321,7 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev) adev->mc.real_vram_size =RREG32 (mmCONFIG_MEMSIZE) *1024ULL *1024ULL ; adev->mc.visible_vram_size = adev->mc.aper_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - +amdgpu_gart_set_defaults (adev);gmc_v6_0_vram_gtt_location (adev, &adev->mc);return 0 ;
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1 -9
@@ -373,15 +373,7 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)if (adev->mc.visible_vram_size > adev->mc.real_vram_size) adev->mc.visible_vram_size = adev->mc.real_vram_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - +amdgpu_gart_set_defaults (adev);gmc_v7_0_vram_gtt_location (adev, &adev->mc);return 0 ;
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1 -9
@@ -535,15 +535,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)if (adev->mc.visible_vram_size > adev->mc.real_vram_size) adev->mc.visible_vram_size = adev->mc.real_vram_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - +amdgpu_gart_set_defaults (adev);gmc_v8_0_vram_gtt_location (adev, &adev->mc);return 0 ;
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -9
@@ -495,15 +495,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)if (adev->mc.visible_vram_size > adev->mc.real_vram_size) adev->mc.visible_vram_size = adev->mc.real_vram_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - +amdgpu_gart_set_defaults (adev);gmc_v9_0_vram_gtt_location (adev, &adev->mc);return 0 ;
Signature · 6 lines
2.7.4 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 1498484397-1308-1-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
PATCH 2/3 drm/amdgpu: fix amdgpu_debugfs_gem_bo_info
From: Christian König <[email protected]>
Otherwise we trigger a bunch of WARN_ONs when this is called.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +7 -3
@@ -784,6 +784,7 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)unsigned domain;const char *placement;unsigned pin_count; +uint64_t offset; domain =amdgpu_mem_type_to_domain (bo->tbo.mem.mem_type);switch (domain) {
@@ -798,9 +799,12 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data) placement =" CPU" ;break ; } -seq_printf (m,"\t0x%08x: %12ld byte %s @ 0x%010Lx" , - id,amdgpu_bo_size (bo), placement, -amdgpu_bo_gpu_offset (bo)); +seq_printf (m,"\t0x%08x: %12ld byte %s" , + id,amdgpu_bo_size (bo), placement); + + offset =ACCESS_ONCE (bo->tbo.mem.start); +if (offset !=AMDGPU_BO_INVALID_OFFSET ) +seq_printf (m," @ 0x%010Lx" , offset); pin_count =ACCESS_ONCE (bo->pin_count);if (pin_count)
Signature · 6 lines
2.7.4 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 1498484397-1308-2-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Deucher, Alexander
Reviewed-by Alex DeucherRE: [PATCH 2/3] drm/amdgpu: fix amdgpu_debugfs_gem_bo_info
Quoted 8 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian König
Sent: Monday, June 26, 2017 9:40 AM
To: [email protected]
Cc: [email protected]; Kuehling, Felix
Subject: [PATCH 2/3] drm/amdgpu: fixamdgpu_debugfs_gem_bo_info
Quoted 5 lines
From: Christian König <[email protected]>
Otherwise we trigger a bunch of
WARN_ONswhen this is called.Signed-off-by Christian König <[email protected]>
Quoted 39 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1
-0@@ -784,6 +784,7 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)unsigned domain;const char *placement;unsigned pin_count; +uint64_t offset; domain = amdgpu_mem_type_to_domain(bo-Quoted 1 line
tbo.mem.mem_type);
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +6
-3switch (domain) {@@ -798,9 +799,12 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data) placement =" CPU" ;break ; } -seq_printf (m,"\t0x%08x: %12ld byte %s @ 0x%010Lx" , - id,amdgpu_bo_size (bo), placement, -amdgpu_bo_gpu_offset (bo)); +seq_printf (m,"\t0x%08x: %12ld byte %s" , + id,amdgpu_bo_size (bo), placement); + + offset =ACCESS_ONCE (bo->tbo.mem.start); +if (offset !=AMDGPU_BO_INVALID_OFFSET ) +seq_printf (m," @ 0x%010Lx" , offset); pin_count =ACCESS_ONCE (bo->pin_count);if (
pin_count)
--
2.7.4
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/3 drm/amdgpu: add gtt_sys_limit
In reply to a message not in the archiveFrom: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 8 ++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 6 +++--- 7 files changed, 26 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
@@ -76,6 +76,7 @@extern int amdgpu_modeset;extern int amdgpu_vram_limit;extern int amdgpu_gart_size; +extern unsigned amdgpu_gart_sys_limit;extern int amdgpu_moverate;extern int amdgpu_benchmarking;extern int amdgpu_testing;
@@ -602,6 +603,7 @@ struct amdgpu_mc {u64 mc_vram_size;u64 visible_vram_size;u64 gtt_size; +u64 gtt_sys_limit;u64 gtt_start;u64 gtt_end;u64 vram_start;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev) } } +if (amdgpu_gart_sys_limit <32 ) { +dev_warn (adev->dev,"gart sys limit (%d) too small\n" , + amdgpu_gart_sys_limit); + amdgpu_gart_sys_limit =32 ; + } +amdgpu_check_vm_size (adev);amdgpu_check_block_size (adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
@@ -75,6 +75,7 @@int amdgpu_vram_limit =0 ;int amdgpu_gart_size =-1 ;/* auto */ +unsigned amdgpu_gart_sys_limit =256 ;int amdgpu_moverate =-1 ;/* auto */ int amdgpu_benchmarking =0 ;int amdgpu_testing =0 ;
@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);MODULE_PARM_DESC (gartsize,"Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)" );module_param_named (gartsize, amdgpu_gart_size, int,0600 ); +MODULE_PARM_DESC (gartlimit,"GART limit for the system domain in megabytes (default 256)" ); +module_param_named (gartlimit, amdgpu_gart_sys_limit, int,0600 ); +MODULE_PARM_DESC (moverate,"Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)" );module_param_named (moverate, amdgpu_moverate, int,0600 );
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6 -2
@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev) adev->mc.mc_vram_size); else adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; + + adev->mc.gtt_sys_limit =min ((uint64_t )amdgpu_gart_sys_limit <<20 , + adev->mc.gtt_size); }/**
@@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)if (r)return r;/* Compute table size */ - adev->gart.num_cpu_pages = adev->mc.gtt_size /PAGE_SIZE ; - adev->gart.num_gpu_pages = adev->mc.gtt_size /AMDGPU_GPU_PAGE_SIZE ; + adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit /PAGE_SIZE ; + adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit / +AMDGPU_GPU_PAGE_SIZE ;DRM_INFO ("GART: num cpu pages %u, num gpu pages %u\n" , adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1 -1
@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,/* Maximum bo size is the unpinned gtt size since we use the gtt to * handle vram to system pool migrations. */ - max_size = adev->mc.gtt_size - adev->gart_pin_size; + max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;if (size > max_size) {DRM_DEBUG ("Allocation size %ldMb bigger than %ldMb limit\n" , size >>20 , max_size >>20 );
drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4 -2
@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {static int amdgpu_gtt_mgr_init (struct ttm_mem_type_manager *man,unsigned long p_size) { +struct amdgpu_device *adev =amdgpu_ttm_adev (man->bdev);struct amdgpu_gtt_mgr *mgr; mgr =kzalloc (sizeof (*mgr),GFP_KERNEL );if (!mgr)return -ENOMEM ; -drm_mm_init (&mgr->mm,0 , p_size); +drm_mm_init (&mgr->mm,0 , adev->mc.gtt_sys_limit >>PAGE_SHIFT );spin_lock_init (&mgr->lock); mgr->available = p_size; man->priv = mgr;
@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,const struct ttm_place *place, struct ttm_mem_reg *mem) { + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);struct amdgpu_gtt_mgr *mgr = man->priv;struct drm_mm_node *node = mem->mm_node;enum drm_mm_insert_mode mode;
@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,if (place && place->lpfn) lpfn = place->lpfn;else - lpfn = man->size; + lpfn = adev->gart.num_cpu_pages; mode =DRM_MM_INSERT_BEST ;if (place && place->flags &TTM_PL_FLAG_TOPDOWN )
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -3
@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,* allocating address space for the BO. */ abo->placements[i].lpfn = - adev->mc.gtt_size >>PAGE_SHIFT ; + adev->gart.num_cpu_pages; } }break ;
@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);
@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);
Signature · 6 lines
2.7.4 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 1498484397-1308-3-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
I'm wondering what makes this possible. Let me quote the last discussion we had about GART:
Quoted 10 lines · Felix Kuehling
Quoted 8 lines
Even with GART address space being allocated on demand, it still seems to be limiting the maximum available system memory that can be allocated from TTM. We have a test that allocates a bunch of 128MB buffers. On a 32GB system memory system with a 4GB GPU it can only get 31 buffers, so a bit under 4GB. Looks like BOs remain bound to GART after being initialized or migrated to GTT. For KFD that limits the amount of usable system memory, for
amdgpu_cs, I think it limits the amount of system memory that can be used in a single command submission.
Quoted 8 lines
Quoted 7 lines · Christian König
Quoted 1 line
Are these two effects intentional?
Yes, thought about dropping this as well but testing showed that it is still necessary.
The total amount of memory bound to the GPU must be limited by the GART size or otherwise the swapping code won't work any more. E.g. suspend/resume fails immediately if I remove that.
Has that changed? I don't remember seeing any changes to that effect.
Regards,
Felix
Quoted 175 lines · Christian König
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 8 ++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 6 +++--- 7 files changed, 26 insertions(+), 8 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -76,6 +76,7 @@extern int amdgpu_modeset;extern int amdgpu_vram_limit;extern int amdgpu_gart_size; +extern unsigned amdgpu_gart_sys_limit;extern int amdgpu_moverate;extern int amdgpu_benchmarking;extern int amdgpu_testing;@@ -602,6 +603,7 @@ struct amdgpu_mc {u64 mc_vram_size;u64 visible_vram_size;u64 gtt_size; +u64 gtt_sys_limit;u64 gtt_start;u64 gtt_end;u64 vram_start;drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6
-0@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev) } } +if (amdgpu_gart_sys_limit <32 ) { +dev_warn (adev->dev,"gart sys limit (%d) too small\n" , + amdgpu_gart_sys_limit); + amdgpu_gart_sys_limit =32 ; + } +amdgpu_check_vm_size (adev);amdgpu_check_block_size (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4
-0@@ -75,6 +75,7 @@int amdgpu_vram_limit =0 ;int amdgpu_gart_size =-1 ;/* auto */ +unsigned amdgpu_gart_sys_limit =256 ;int amdgpu_moverate =-1 ;/* auto */ int amdgpu_benchmarking =0 ;int amdgpu_testing =0 ;@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);MODULE_PARM_DESC (gartsize,"Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)" );module_param_named (gartsize, amdgpu_gart_size, int,0600 ); +MODULE_PARM_DESC (gartlimit,"GART limit for the system domain in megabytes (default 256)" ); +module_param_named (gartlimit, amdgpu_gart_sys_limit, int,0600 ); +MODULE_PARM_DESC (moverate,"Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)" );module_param_named (moverate, amdgpu_moverate, int,0600 );drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6
-2@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev) adev->mc.mc_vram_size); else adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; + + adev->mc.gtt_sys_limit =min ((uint64_t )amdgpu_gart_sys_limit <<20 , + adev->mc.gtt_size); }/** @@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)if (r)return r;/* Compute table size */ - adev->gart.num_cpu_pages = adev->mc.gtt_size /PAGE_SIZE ; - adev->gart.num_gpu_pages = adev->mc.gtt_size /AMDGPU_GPU_PAGE_SIZE ; + adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit /PAGE_SIZE ; + adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit / +AMDGPU_GPU_PAGE_SIZE ;DRM_INFO ("GART: num cpu pages %u, num gpu pages %u\n" , adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1
-1@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,/* Maximum bo size is the unpinned gtt size since we use the gtt to * handle vram to system pool migrations. */ - max_size = adev->mc.gtt_size - adev->gart_pin_size; + max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;if (size > max_size) {DRM_DEBUG ("Allocation size %ldMb bigger than %ldMb limit\n" , size >>20 , max_size >>20 );drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4
-2@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {static int amdgpu_gtt_mgr_init (struct ttm_mem_type_manager *man,unsigned long p_size) { +struct amdgpu_device *adev =amdgpu_ttm_adev (man->bdev);struct amdgpu_gtt_mgr *mgr; mgr =kzalloc (sizeof (*mgr),GFP_KERNEL );if (!mgr)return -ENOMEM ; -drm_mm_init (&mgr->mm,0 , p_size); +drm_mm_init (&mgr->mm,0 , adev->mc.gtt_sys_limit >>PAGE_SHIFT );spin_lock_init (&mgr->lock); mgr->available = p_size; man->priv = mgr;@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,const struct ttm_place *place, struct ttm_mem_reg *mem) { + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);struct amdgpu_gtt_mgr *mgr = man->priv;struct drm_mm_node *node = mem->mm_node;enum drm_mm_insert_mode mode;@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,if (place && place->lpfn) lpfn = place->lpfn;else - lpfn = man->size; + lpfn = adev->gart.num_cpu_pages; mode =DRM_MM_INSERT_BEST ;if (place && place->flags &TTM_PL_FLAG_TOPDOWN )drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2
-2@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,* allocating address space for the BO. */ abo->placements[i].lpfn = - adev->mc.gtt_size >>PAGE_SHIFT ; + adev->gart.num_cpu_pages; } }break ;@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, · line 437
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1
-1@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): c2d01766-450a-9f03-6d3f-532154a4fcd3-5C7GfCeVMHo@public.gmane.org
Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Am 26.06.2017 um 22:12 schrieb Felix Kuehling:
Quoted 12 lines
I'm wondering what makes this possible. Let me quote the last discussion we had about GART:
Quoted 9 lines · Felix Kuehling
Quoted 8 lines
Even with GART address space being allocated on demand, it still seems to be limiting the maximum available system memory that can be allocated from TTM. We have a test that allocates a bunch of 128MB buffers. On a 32GB system memory system with a 4GB GPU it can only get 31 buffers, so a bit under 4GB. Looks like BOs remain bound to GART after being initialized or migrated to GTT. For KFD that limits the amount of usable system memory, for
amdgpu_cs, I think it limits the amount of system memory that can be used in a single command submission.
That's why I added a new limit instead of modifying the existing one. The gartsize parameter still works as it did before.
Quoted 9 lines
Quoted 8 lines · Christian König
Quoted 7 lines
Quoted 1 line
Are these two effects intentional?
Yes, thought about dropping this as well but testing showed that it is still necessary.
The total amount of memory bound to the GPU must be limited by the GART size or otherwise the swapping code won't work any more. E.g. suspend/resume fails immediately if I remove that.
Has that changed? I don't remember seeing any changes to that effect.
No, there are still a bunch of problems to solve. I should have put a WIP mark on that patch.
The maximum BO size is limited with this patch and will probably result in a bunch of S3 problems.
But as long as you don't care about those limitations it should work.
Christian.
Quoted 185 lines
Regards,
FelixQuoted 179 lines · Christian König
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 8 ++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 6 +++--- 7 files changed, 26 insertions(+), 8 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -76,6 +76,7 @@extern int amdgpu_modeset;extern int amdgpu_vram_limit;extern int amdgpu_gart_size; +extern unsigned amdgpu_gart_sys_limit;extern int amdgpu_moverate;extern int amdgpu_benchmarking;extern int amdgpu_testing;@@ -602,6 +603,7 @@ struct amdgpu_mc {u64 mc_vram_size;u64 visible_vram_size;u64 gtt_size; +u64 gtt_sys_limit;u64 gtt_start;u64 gtt_end;u64 vram_start;drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6
-0@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev) } } +if (amdgpu_gart_sys_limit <32 ) { +dev_warn (adev->dev,"gart sys limit (%d) too small\n" , + amdgpu_gart_sys_limit); + amdgpu_gart_sys_limit =32 ; + } +amdgpu_check_vm_size (adev);amdgpu_check_block_size (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4
-0@@ -75,6 +75,7 @@int amdgpu_vram_limit =0 ;int amdgpu_gart_size =-1 ;/* auto */ +unsigned amdgpu_gart_sys_limit =256 ;int amdgpu_moverate =-1 ;/* auto */ int amdgpu_benchmarking =0 ;int amdgpu_testing =0 ;@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);MODULE_PARM_DESC (gartsize,"Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)" );module_param_named (gartsize, amdgpu_gart_size, int,0600 ); +MODULE_PARM_DESC (gartlimit,"GART limit for the system domain in megabytes (default 256)" ); +module_param_named (gartlimit, amdgpu_gart_sys_limit, int,0600 ); +MODULE_PARM_DESC (moverate,"Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)" );module_param_named (moverate, amdgpu_moverate, int,0600 );drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6
-2@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev) adev->mc.mc_vram_size); else adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; + + adev->mc.gtt_sys_limit =min ((uint64_t )amdgpu_gart_sys_limit <<20 , + adev->mc.gtt_size); }/** @@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)if (r)return r;/* Compute table size */ - adev->gart.num_cpu_pages = adev->mc.gtt_size /PAGE_SIZE ; - adev->gart.num_gpu_pages = adev->mc.gtt_size /AMDGPU_GPU_PAGE_SIZE ; + adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit /PAGE_SIZE ; + adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit / +AMDGPU_GPU_PAGE_SIZE ;DRM_INFO ("GART: num cpu pages %u, num gpu pages %u\n" , adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1
-1@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,/* Maximum bo size is the unpinned gtt size since we use the gtt to * handle vram to system pool migrations. */ - max_size = adev->mc.gtt_size - adev->gart_pin_size; + max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;if (size > max_size) {DRM_DEBUG ("Allocation size %ldMb bigger than %ldMb limit\n" , size >>20 , max_size >>20 );drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4
-2@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {static int amdgpu_gtt_mgr_init (struct ttm_mem_type_manager *man,unsigned long p_size) { +struct amdgpu_device *adev =amdgpu_ttm_adev (man->bdev);struct amdgpu_gtt_mgr *mgr; mgr =kzalloc (sizeof (*mgr),GFP_KERNEL );if (!mgr)return -ENOMEM ; -drm_mm_init (&mgr->mm,0 , p_size); +drm_mm_init (&mgr->mm,0 , adev->mc.gtt_sys_limit >>PAGE_SHIFT );spin_lock_init (&mgr->lock); mgr->available = p_size; man->priv = mgr;@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,const struct ttm_place *place, struct ttm_mem_reg *mem) { + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);struct amdgpu_gtt_mgr *mgr = man->priv;struct drm_mm_node *node = mem->mm_node;enum drm_mm_insert_mode mode;@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,if (place && place->lpfn) lpfn = place->lpfn;else - lpfn = man->size; + lpfn = adev->gart.num_cpu_pages; mode =DRM_MM_INSERT_BEST ;if (place && place->flags &TTM_PL_FLAG_TOPDOWN )drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2
-3@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,* allocating address space for the BO. */ abo->placements[i].lpfn = - adev->mc.gtt_size >>PAGE_SHIFT ; + adev->gart.num_cpu_pages; } }break ;@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ;
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, · line 390
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1
-0+ placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
In reply to a message not in the archiveQuoted 7 lines · Christian König
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>
Hmm.
On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up 1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure. But in the grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth cutting GTT memory by a factor of 16?
John
Quoted 179 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 8 ++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 6 +++--- 7 files changed, 26 insertions(+), 8 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +2
-0@@ -76,6 +76,7 @@extern int amdgpu_modeset;extern int amdgpu_vram_limit;extern int amdgpu_gart_size; +extern unsigned amdgpu_gart_sys_limit;extern int amdgpu_moverate;extern int amdgpu_benchmarking;extern int amdgpu_testing;@@ -602,6 +603,7 @@ struct amdgpu_mc {u64 mc_vram_size;u64 visible_vram_size;u64 gtt_size; +u64 gtt_sys_limit;u64 gtt_start;u64 gtt_end;u64 vram_start;drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6
-0@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev) } } +if (amdgpu_gart_sys_limit <32 ) { +dev_warn (adev->dev,"gart sys limit (%d) too small\n" , + amdgpu_gart_sys_limit); + amdgpu_gart_sys_limit =32 ; + } +amdgpu_check_vm_size (adev);amdgpu_check_block_size (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4
-0@@ -75,6 +75,7 @@int amdgpu_vram_limit =0 ;int amdgpu_gart_size =-1 ;/* auto */ +unsigned amdgpu_gart_sys_limit =256 ;int amdgpu_moverate =-1 ;/* auto */ int amdgpu_benchmarking =0 ;int amdgpu_testing =0 ;@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);MODULE_PARM_DESC (gartsize,"Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)" );module_param_named (gartsize, amdgpu_gart_size, int,0600 ); +MODULE_PARM_DESC (gartlimit,"GART limit for the system domain in megabytes (default 256)" ); +module_param_named (gartlimit, amdgpu_gart_sys_limit, int,0600 ); +MODULE_PARM_DESC (moverate,"Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)" );module_param_named (moverate, amdgpu_moverate, int,0600 );drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6
-2@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev) adev->mc.mc_vram_size); else adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20; + + adev->mc.gtt_sys_limit =min ((uint64_t )amdgpu_gart_sys_limit <<20 , + adev->mc.gtt_size); }/** @@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)if (r)return r;/* Compute table size */ - adev->gart.num_cpu_pages = adev->mc.gtt_size /PAGE_SIZE ; - adev->gart.num_gpu_pages = adev->mc.gtt_size /AMDGPU_GPU_PAGE_SIZE ; + adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit /PAGE_SIZE ; + adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit / +AMDGPU_GPU_PAGE_SIZE ;DRM_INFO ("GART: num cpu pages %u, num gpu pages %u\n" , adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1
-1@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,/* Maximum bo size is the unpinned gtt size since we use the gtt to * handle vram to system pool migrations. */ - max_size = adev->mc.gtt_size - adev->gart_pin_size; + max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;if (size > max_size) {DRM_DEBUG ("Allocation size %ldMb bigger than %ldMb limit\n" , size >>20 , max_size >>20 );drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4
-2@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {static int amdgpu_gtt_mgr_init (struct ttm_mem_type_manager *man,unsigned long p_size) { +struct amdgpu_device *adev =amdgpu_ttm_adev (man->bdev);struct amdgpu_gtt_mgr *mgr; mgr =kzalloc (sizeof (*mgr),GFP_KERNEL );if (!mgr)return -ENOMEM ; -drm_mm_init (&mgr->mm,0 , p_size); +drm_mm_init (&mgr->mm,0 , adev->mc.gtt_sys_limit >>PAGE_SHIFT );spin_lock_init (&mgr->lock); mgr->available = p_size; man->priv = mgr;@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,const struct ttm_place *place, struct ttm_mem_reg *mem) { + struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);struct amdgpu_gtt_mgr *mgr = man->priv;struct drm_mm_node *node = mem->mm_node;enum drm_mm_insert_mode mode;@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,if (place && place->lpfn) lpfn = place->lpfn;else - lpfn = man->size; + lpfn = adev->gart.num_cpu_pages; mode =DRM_MM_INSERT_BEST ;if (place && place->flags &TTM_PL_FLAG_TOPDOWN )drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3
-3@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,* allocating address space for the BO. */ abo->placements[i].lpfn = - adev->mc.gtt_size >>PAGE_SHIFT ; + adev->gart.num_cpu_pages; } }break ;@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, placement.num_busy_placement =1 ; placement.busy_placement = &placements; placements.fpfn =0 ; - placements.lpfn = adev->mc.gtt_size >>PAGE_SHIFT ; + placements.lpfn = adev->gart.num_cpu_pages; placements.flags =TTM_PL_MASK_CACHING |TTM_PL_FLAG_TT ; r =ttm_bo_mem_space (bo, &placement, &tmp_mem, interruptible, no_wait_gpu);Signature · 1 line
2.7.4
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20170626233901.GA30922-6hIufAJW0g7Gr8qjsLp7YGXnswh1EIUO@public.gmane.org
Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Quoted 9 lines · John Brooks
Quoted 7 lines · Christian König
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>
Quoted 7 lines
Hmm.
On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up 1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure. But in the grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth cutting GTT memory by a factor of 16?
I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.
Signature · 6 lines
Earthling Michel Dänzer | http://www.amd.com Libre software enthusiast | Mesa and X developer _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): e3546936-bd31-d650-b5c8-807822213827-otUistvHUpPR7s880joybQ@public.gmane.org
Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Am 27.06.2017 um 04:57 schrieb Michel Dänzer:
Quoted 8 lines · John Brooks
Quoted 7 lines · Christian König
Quoted 6 lines
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Quoted 8 lines
Quoted 8 lines
Quoted 1 line
Signed-off-by Christian König <[email protected]>Hmm.
On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up 1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure. But in the grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth cutting GTT memory by a factor of 16?
The amount of GTT memory the application can use through the VM page tables stays the same.
Only the system VM is limited to 256MB and so saves us a whole bunch of space.
Quoted 3 lines
I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.
Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.
Christian.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 7c144cd0-a5e2-7a6c-7192-194fdee000c2-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Quoted 26 lines · Christian König
Am 27.06.2017 um 04:57 schrieb Michel Dänzer:
Quoted 12 lines · John Brooks
Quoted 11 lines · Christian König
Quoted 8 lines
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>Hmm.
Quoted 9 lines
Quoted 9 lines
On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up
1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the
size of the table to 512 KiB. A relatively large saving, to be sure. But in the
grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth
cutting GTT memory by a factor of 16?The amount of GTT memory the application can use through the VM page tables stays the same.
Quoted 8 lines
Only the system VM is limited to 256MB and so saves us a whole bunch of space.
Quoted 3 lines
I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.
Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.
OK, I should adapt this for the KFD branch. Currently we make our GART much bigger. On a system with lots of system memory, we can use up half the visible VRAM for the GART. With something like this we can get it back to something reasonable. But a 256MB limit on single allocation size is definitely too small.
Also, if this breaks S3, we have to make sure the hybrid branches don't pick it up accidentally.
Regards,
Felix
Quoted 2 lines
Christian.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 300c3193-1341-934b-4fdd-f484fdd18e89-5C7GfCeVMHo@public.gmane.org
RE: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Yeah, recently we indeed encountered an issue as below:
On workstation machine it has 32G system memory and thus the GTT table size takes up about 192M visible VRAM. Maybe if system has bigger memory, GTT table size will exceed 256M, so Limitation for max GTT table size is very necessary.
Thanks
Roger(Hongbo.He)
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Wednesday, June 28, 2017 3:25 AM
To: Christian König <[email protected]>; Michel Dänzer <[email protected]>; John Brooks <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Quoted 26 lines · Christian König
Am 27.06.2017 um 04:57 schrieb Michel Dänzer:
Quoted 12 lines · John Brooks
Quoted 11 lines · Christian König
Quoted 8 lines
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>Hmm.
Quoted 9 lines
Quoted 9 lines
On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up
1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure.
But in the
grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth
cutting GTT memory by a factor of 16?The amount of GTT memory the application can use through the VM page tables stays the same.
Quoted 8 lines
Only the system VM is limited to 256MB and so saves us a whole bunch of space.
Quoted 3 lines
I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.
Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.
OK, I should adapt this for the KFD branch. Currently we make our GART much bigger. On a system with lots of system memory, we can use up half the visible VRAM for the GART. With something like this we can get it back to something reasonable. But a 256MB limit on single allocation size is definitely too small.
Also, if this breaks S3, we have to make sure the hybrid branches don't pick it up accidentally.
Regards,
Felix
Quoted 2 lines
Christian.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
RE: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
In reply to Felix KuehlingSetting limitation for max GTT table size is necessary.
But my only concern is that is it enough for amdgpu_gart_sys_limit 256M. whether it will trigger GTT eviction from GTT to CPU domain easily? Because GTT size always equals to VRAM size before it scales with system memory size.
Thanks
Roger(Hongbo.He)
-----Original Message-----
From: He, Roger
Sent: Wednesday, June 28, 2017 9:58 AM
To: 'Felix Kuehling' <[email protected]>; Christian König <[email protected]>; Michel Dänzer <[email protected]>; John Brooks <[email protected]>
Cc: [email protected]
Subject: RE: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Yeah, recently we indeed encountered an issue as below:
On workstation machine it has 32G system memory and thus the GTT table size takes up about 192M visible VRAM. Maybe if system has bigger memory, GTT table size will exceed 256M, so Limitation for max GTT table size is very necessary.
Thanks
Roger(Hongbo.He)
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Wednesday, June 28, 2017 3:25 AM
To: Christian König <[email protected]>; Michel Dänzer <[email protected]>; John Brooks <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit
Quoted 26 lines · Christian König
Am 27.06.2017 um 04:57 schrieb Michel Dänzer:
Quoted 12 lines · John Brooks
Quoted 11 lines · Christian König
Quoted 8 lines
From: Christian König <[email protected]>
Limit the size of the GART table for the system domain.
This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.
Signed-off-by Christian König <[email protected]>Hmm.
Quoted 9 lines
Quoted 9 lines
On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up
1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure.
But in the
grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth
cutting GTT memory by a factor of 16?The amount of GTT memory the application can use through the VM page tables stays the same.
Quoted 8 lines
Only the system VM is limited to 256MB and so saves us a whole bunch of space.
Quoted 3 lines
I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.
Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.
OK, I should adapt this for the KFD branch. Currently we make our GART much bigger. On a system with lots of system memory, we can use up half the visible VRAM for the GART. With something like this we can get it back to something reasonable. But a 256MB limit on single allocation size is definitely too small.
Also, if this breaks S3, we have to make sure the hybrid branches don't pick it up accidentally.
Regards,
Felix
Quoted 2 lines
Christian.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Deucher, Alexander
Reviewed-by Alex DeucherQuoted 8 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian König
Sent: Monday, June 26, 2017 9:40 AM
To: [email protected]
Cc: [email protected]; Kuehling, Felix
Subject: [PATCH 1/3] drm/amdgpu: cleanup initializinggtt_size
Quoted 5 lines
From: Christian König <[email protected]>
Stop spreading the code over all GMC generations.
Signed-off-by Christian König <[email protected]>
Quoted 152 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 20 ++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 10 +--------- drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 10 +--------- drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 10 +--------- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 10 +--------- 6 files changed, 25 insertions(+), 36 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -561,6 +561,7 @@ struct amdgpu_gart {const struct amdgpu_gart_funcs *gart_funcs; }; +void amdgpu_gart_set_defaults (struct amdgpu_device *adev);int amdgpu_gart_table_ram_alloc (struct amdgpu_device *adev);void amdgpu_gart_table_ram_free (struct amdgpu_device *adev);int amdgpu_gart_table_vram_alloc (struct amdgpu_device *adev);drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +20
-0@@ -52,6 +52,26 @@/* * Common GART table functions. */ + +/** +* amdgpu_gart_set_defaults - set the default gtt_size +* +* @adev: amdgpu_device pointer +* +* Set the default gtt_size based on parameters and available VRAM. +*/ +void amdgpu_gart_set_defaults (struct amdgpu_device *adev) +{ +/* unless the user had overridden it, set the gart +* size equal to the 1024 or vram, whichever is larger. +*/ +if (amdgpu_gart_size ==-1 ) + adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), + adev->mc.mc_vram_size); +else + adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; +} +/** * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table *
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +1
-9@@ -321,15 +321,7 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev) adev->mc.real_vram_size =RREG32 (mmCONFIG_MEMSIZE) *1024ULL *1024ULL ; adev->mc.visible_vram_size = adev->mc.aper_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - + amdgpu_gart_set_defaults(adev);drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +0
-0gmc_v6_0_vram_gtt_location (adev, &adev->mc);return 0 ;drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1
-9@@ -373,15 +373,7 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)if (adev->mc.visible_vram_size > adev->mc.real_vram_size) adev->mc.visible_vram_size = adev->mc.real_vram_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - + amdgpu_gart_set_defaults(adev);gmc_v7_0_vram_gtt_location (adev, &adev->mc);drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +0
-0return 0 ;drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1
-9@@ -535,15 +535,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)if (adev->mc.visible_vram_size > adev->mc.real_vram_size) adev->mc.visible_vram_size = adev->mc.real_vram_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - + amdgpu_gart_set_defaults(adev);gmc_v8_0_vram_gtt_location (adev, &adev->mc);drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +0
-0return 0 ;drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1
-9@@ -495,15 +495,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)if (adev->mc.visible_vram_size > adev->mc.real_vram_size) adev->mc.visible_vram_size = adev->mc.real_vram_size; -/* unless the user had overridden it, set the gart -* size equal to the 1024 or vram, whichever is larger. -*/ -if (amdgpu_gart_size ==-1 ) - adev->mc.gtt_size =max ((AMDGPU_DEFAULT_GTT_SIZE_MB <<20 ), - adev->mc.mc_vram_size); -else - adev->mc.gtt_size = (uint64_t )amdgpu_gart_size <<20 ; - + amdgpu_gart_set_defaults(adev);gmc_v9_0_vram_gtt_location (adev, &adev->mc);
Quoted 8 lines
return 0;
--
2.7.4_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx