Skip to contentSkip to conversation

drm/amdgpu: cleanup initializing gtt_size

PATCH In review Christian König 6 participants 13 messages Jun 26, 2017 – Jun 28, 2017 8 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 drm/amdgpu: cleanup initializing gtt_size

From: Christian König <[email protected]>

Stop spreading the code over all GMC generations.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h      |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 20 ++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 10 +---------
 6 files changed, 25 insertions(+), 36 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 7624294..ab1dad2 100644
@@ -561,6 +561,7 @@ struct amdgpu_gart {
 	const struct amdgpu_gart_funcs *gart_funcs;
 };
 
+void amdgpu_gart_set_defaults(struct amdgpu_device *adev);
 int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
 void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
 int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +20 -0
index ccef3cf..8877015 100644
@@ -52,6 +52,26 @@
 /*
  * Common GART table functions.
  */
+
+/**
+ * amdgpu_gart_set_defaults - set the default gtt_size
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set the default gtt_size based on parameters and available VRAM.
+ */
+void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
+{
+	/* unless the user had overridden it, set the gart
+	 * size equal to the 1024 or vram, whichever is larger.
+	 */
+	if (amdgpu_gart_size == -1)
+		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
+					adev->mc.mc_vram_size);
+	else
+		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+}
+
 /**
  * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table
  *
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +1 -9
index 35655a1..5cc3f39 100644
@@ -321,15 +321,7 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v6_0_vram_gtt_location(adev, &adev->mc);
 
 	return 0;
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1 -9
index 83c21a1..15f2c0f 100644
@@ -373,15 +373,7 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v7_0_vram_gtt_location(adev, &adev->mc);
 
 	return 0;
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1 -9
index c92af24..213af65 100644
@@ -535,15 +535,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v8_0_vram_gtt_location(adev, &adev->mc);
 
 	return 0;
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -9
index 7c6188b..128989e 100644
@@ -495,15 +495,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v9_0_vram_gtt_location(adev, &adev->mc);
 
 	return 0;
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1498484397-1308-1-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#2 lore raw

PATCH 2/3 drm/amdgpu: fix amdgpu_debugfs_gem_bo_info

From: Christian König <[email protected]>

Otherwise we trigger a bunch of WARN_ONs when this is called.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +7 -3
index 621f739..96c4493 100644
@@ -784,6 +784,7 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 	unsigned domain;
 	const char *placement;
 	unsigned pin_count;
+	uint64_t offset;
 
 	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
 	switch (domain) {
@@ -798,9 +799,12 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 		placement = " CPU";
 		break;
 	}
-	seq_printf(m, "\t0x%08x: %12ld byte %s @ 0x%010Lx",
-		   id, amdgpu_bo_size(bo), placement,
-		   amdgpu_bo_gpu_offset(bo));
+	seq_printf(m, "\t0x%08x: %12ld byte %s",
+		   id, amdgpu_bo_size(bo), placement);
+
+	offset = ACCESS_ONCE(bo->tbo.mem.start);
+	if (offset != AMDGPU_BO_INVALID_OFFSET)
+		seq_printf(m, " @ 0x%010Lx", offset);
 
 	pin_count = ACCESS_ONCE(bo->pin_count);
 	if (pin_count)
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1498484397-1308-2-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

Reviewed-by Alex Deucher
#4 lore raw

RE: [PATCH 2/3] drm/amdgpu: fix amdgpu_debugfs_gem_bo_info

Quoted 8 lines

-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian König
Sent: Monday, June 26, 2017 9:40 AM
To: [email protected]
Cc: [email protected]; Kuehling, Felix
Subject: [PATCH 2/3] drm/amdgpu: fix amdgpu_debugfs_gem_bo_info

Quoted 5 lines

From: Christian König <[email protected]>

Otherwise we trigger a bunch of WARN_ONs when this is called.

Quoted 39 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1 -0
index 621f739..96c4493 100644
@@ -784,6 +784,7 @@ static int amdgpu_debugfs_gem_bo_info(int id, void
 *ptr, void *data)
 	unsigned domain;
 	const char *placement;
 	unsigned pin_count;
+	uint64_t offset;
 
 	domain = amdgpu_mem_type_to_domain(bo-
Quoted 1 line

tbo.mem.mem_type);

drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +6 -3
 	switch (domain) {
@@ -798,9 +799,12 @@ static int amdgpu_debugfs_gem_bo_info(int id, void
 *ptr, void *data)
 		placement = " CPU";
 		break;
 	}
-	seq_printf(m, "\t0x%08x: %12ld byte %s @ 0x%010Lx",
-		   id, amdgpu_bo_size(bo), placement,
-		   amdgpu_bo_gpu_offset(bo));
+	seq_printf(m, "\t0x%08x: %12ld byte %s",
+		   id, amdgpu_bo_size(bo), placement);
+
+	offset = ACCESS_ONCE(bo->tbo.mem.start);
+	if (offset != AMDGPU_BO_INVALID_OFFSET)
+		seq_printf(m, " @ 0x%010Lx", offset);
 
 	pin_count = ACCESS_ONCE(bo->pin_count);

if (pin_count)
--
2.7.4

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#5 lore raw

PATCH 3/3 drm/amdgpu: add gtt_sys_limit

In reply to a message not in the archive

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h         | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c  | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c     | 4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c    | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c     | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 6 +++---
 7 files changed, 26 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
index ab1dad2..a511029 100644
@@ -76,6 +76,7 @@
 extern int amdgpu_modeset;
 extern int amdgpu_vram_limit;
 extern int amdgpu_gart_size;
+extern unsigned amdgpu_gart_sys_limit;
 extern int amdgpu_moverate;
 extern int amdgpu_benchmarking;
 extern int amdgpu_testing;
@@ -602,6 +603,7 @@ struct amdgpu_mc {
 	u64			mc_vram_size;
 	u64			visible_vram_size;
 	u64			gtt_size;
+	u64			gtt_sys_limit;
 	u64			gtt_start;
 	u64			gtt_end;
 	u64			vram_start;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 44484bb..b6edb83 100644
@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 		}
 	}
 
+	if (amdgpu_gart_sys_limit < 32) {
+		dev_warn(adev->dev, "gart sys limit (%d) too small\n",
+				 amdgpu_gart_sys_limit);
+			amdgpu_gart_sys_limit = 32;
+	}
+
 	amdgpu_check_vm_size(adev);
 
 	amdgpu_check_block_size(adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
index 5a1d794..907ae5e 100644
@@ -75,6 +75,7 @@
 
 int amdgpu_vram_limit = 0;
 int amdgpu_gart_size = -1; /* auto */
+unsigned amdgpu_gart_sys_limit = 256;
 int amdgpu_moverate = -1; /* auto */
 int amdgpu_benchmarking = 0;
 int amdgpu_testing = 0;
@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)");
 module_param_named(gartsize, amdgpu_gart_size, int, 0600);
 
+MODULE_PARM_DESC(gartlimit, "GART limit for the system domain in megabytes (default 256)");
+module_param_named(gartlimit, amdgpu_gart_sys_limit, int, 0600);
+
 MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
 module_param_named(moverate, amdgpu_moverate, int, 0600);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6 -2
index 8877015..5c6a461 100644
@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
 					adev->mc.mc_vram_size);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+
+	adev->mc.gtt_sys_limit = min((uint64_t)amdgpu_gart_sys_limit << 20,
+				     adev->mc.gtt_size);
 }
 
 /**
@@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 	/* Compute table size */
-	adev->gart.num_cpu_pages = adev->mc.gtt_size / PAGE_SIZE;
-	adev->gart.num_gpu_pages = adev->mc.gtt_size / AMDGPU_GPU_PAGE_SIZE;
+	adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit / PAGE_SIZE;
+	adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit /
+		AMDGPU_GPU_PAGE_SIZE;
 	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
 		 adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1 -1
index 96c4493..6eac481 100644
@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 		/* Maximum bo size is the unpinned gtt size since we use the gtt to
 		 * handle vram to system pool migrations.
 		 */
-		max_size = adev->mc.gtt_size - adev->gart_pin_size;
+		max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;
 		if (size > max_size) {
 			DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n",
 				  size >> 20, max_size >> 20);
drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4 -2
index f7d22c4..7609229 100644
@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {
 static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 			       unsigned long p_size)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr;
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
 		return -ENOMEM;
 
-	drm_mm_init(&mgr->mm, 0, p_size);
+	drm_mm_init(&mgr->mm, 0, adev->mc.gtt_sys_limit >> PAGE_SHIFT);
 	spin_lock_init(&mgr->lock);
 	mgr->available = p_size;
 	man->priv = mgr;
@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 			 const struct ttm_place *place,
 			 struct ttm_mem_reg *mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr = man->priv;
 	struct drm_mm_node *node = mem->mm_node;
 	enum drm_mm_insert_mode mode;
@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 	if (place && place->lpfn)
 		lpfn = place->lpfn;
 	else
-		lpfn = man->size;
+		lpfn = adev->gart.num_cpu_pages;
 
 	mode = DRM_MM_INSERT_BEST;
 	if (place && place->flags & TTM_PL_FLAG_TOPDOWN)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -3
index e4860ac..1e0fcb6 100644
@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				 * allocating address space for the BO.
 				 */
 				abo->placements[i].lpfn =
-					adev->mc.gtt_size >> PAGE_SHIFT;
+					adev->gart.num_cpu_pages;
 			}
 		}
 		break;
@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1498484397-1308-3-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#7 lore raw

Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

I'm wondering what makes this possible. Let me quote the last discussion we had about GART:

Quoted 10 lines · Felix Kuehling
Quoted 8 lines

Even with GART address space being allocated on demand, it still seems to be limiting the maximum available system memory that can be allocated from TTM. We have a test that allocates a bunch of 128MB buffers. On a 32GB system memory system with a 4GB GPU it can only get 31 buffers, so a bit under 4GB. Looks like BOs remain bound to GART after being initialized or migrated to GTT. For KFD that limits the amount of usable system memory, for amdgpu_cs, I think it limits the amount of system memory that can be used in a single command submission.

Quoted 8 lines
Quoted 7 lines · Christian König
Quoted 1 line

Are these two effects intentional?

Yes, thought about dropping this as well but testing showed that it is still necessary.

The total amount of memory bound to the GPU must be limited by the GART size or otherwise the swapping code won't work any more. E.g. suspend/resume fails immediately if I remove that.

Has that changed? I don't remember seeing any changes to that effect.

Regards,
Felix

Quoted 175 lines · Christian König

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h         | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c  | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c     | 4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c    | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c     | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 6 +++---
 7 files changed, 26 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
index ab1dad2..a511029 100644
@@ -76,6 +76,7 @@
 extern int amdgpu_modeset;
 extern int amdgpu_vram_limit;
 extern int amdgpu_gart_size;
+extern unsigned amdgpu_gart_sys_limit;
 extern int amdgpu_moverate;
 extern int amdgpu_benchmarking;
 extern int amdgpu_testing;
@@ -602,6 +603,7 @@ struct amdgpu_mc {
 	u64			mc_vram_size;
 	u64			visible_vram_size;
 	u64			gtt_size;
+	u64			gtt_sys_limit;
 	u64			gtt_start;
 	u64			gtt_end;
 	u64			vram_start;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 44484bb..b6edb83 100644
@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 		}
 	}
 
+	if (amdgpu_gart_sys_limit < 32) {
+		dev_warn(adev->dev, "gart sys limit (%d) too small\n",
+				 amdgpu_gart_sys_limit);
+			amdgpu_gart_sys_limit = 32;
+	}
+
 	amdgpu_check_vm_size(adev);
 
 	amdgpu_check_block_size(adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
index 5a1d794..907ae5e 100644
@@ -75,6 +75,7 @@
 
 int amdgpu_vram_limit = 0;
 int amdgpu_gart_size = -1; /* auto */
+unsigned amdgpu_gart_sys_limit = 256;
 int amdgpu_moverate = -1; /* auto */
 int amdgpu_benchmarking = 0;
 int amdgpu_testing = 0;
@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)");
 module_param_named(gartsize, amdgpu_gart_size, int, 0600);
 
+MODULE_PARM_DESC(gartlimit, "GART limit for the system domain in megabytes (default 256)");
+module_param_named(gartlimit, amdgpu_gart_sys_limit, int, 0600);
+
 MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
 module_param_named(moverate, amdgpu_moverate, int, 0600);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6 -2
index 8877015..5c6a461 100644
@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
 					adev->mc.mc_vram_size);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+
+	adev->mc.gtt_sys_limit = min((uint64_t)amdgpu_gart_sys_limit << 20,
+				     adev->mc.gtt_size);
 }
 
 /**
@@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 	/* Compute table size */
-	adev->gart.num_cpu_pages = adev->mc.gtt_size / PAGE_SIZE;
-	adev->gart.num_gpu_pages = adev->mc.gtt_size / AMDGPU_GPU_PAGE_SIZE;
+	adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit / PAGE_SIZE;
+	adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit /
+		AMDGPU_GPU_PAGE_SIZE;
 	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
 		 adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1 -1
index 96c4493..6eac481 100644
@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 		/* Maximum bo size is the unpinned gtt size since we use the gtt to
 		 * handle vram to system pool migrations.
 		 */
-		max_size = adev->mc.gtt_size - adev->gart_pin_size;
+		max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;
 		if (size > max_size) {
 			DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n",
 				  size >> 20, max_size >> 20);
drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4 -2
index f7d22c4..7609229 100644
@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {
 static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 			       unsigned long p_size)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr;
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
 		return -ENOMEM;
 
-	drm_mm_init(&mgr->mm, 0, p_size);
+	drm_mm_init(&mgr->mm, 0, adev->mc.gtt_sys_limit >> PAGE_SHIFT);
 	spin_lock_init(&mgr->lock);
 	mgr->available = p_size;
 	man->priv = mgr;
@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 			 const struct ttm_place *place,
 			 struct ttm_mem_reg *mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr = man->priv;
 	struct drm_mm_node *node = mem->mm_node;
 	enum drm_mm_insert_mode mode;
@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 	if (place && place->lpfn)
 		lpfn = place->lpfn;
 	else
-		lpfn = man->size;
+		lpfn = adev->gart.num_cpu_pages;
 
 	mode = DRM_MM_INSERT_BEST;
 	if (place && place->flags & TTM_PL_FLAG_TOPDOWN)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -2
index e4860ac..1e0fcb6 100644
@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				 * allocating address space for the BO.
 				 */
 				abo->placements[i].lpfn =
-					adev->mc.gtt_size >> PAGE_SHIFT;
+					adev->gart.num_cpu_pages;
 			}
 		}
 		break;
@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, · line 437

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): c2d01766-450a-9f03-6d3f-532154a4fcd3-5C7GfCeVMHo@public.gmane.org

#9 lore raw

Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Am 26.06.2017 um 22:12 schrieb Felix Kuehling:

Quoted 12 lines

I'm wondering what makes this possible. Let me quote the last discussion we had about GART:

Quoted 9 lines · Felix Kuehling
Quoted 8 lines

Even with GART address space being allocated on demand, it still seems to be limiting the maximum available system memory that can be allocated from TTM. We have a test that allocates a bunch of 128MB buffers. On a 32GB system memory system with a 4GB GPU it can only get 31 buffers, so a bit under 4GB. Looks like BOs remain bound to GART after being initialized or migrated to GTT. For KFD that limits the amount of usable system memory, for amdgpu_cs, I think it limits the amount of system memory that can be used in a single command submission.

That's why I added a new limit instead of modifying the existing one. The gartsize parameter still works as it did before.

Quoted 9 lines
Quoted 8 lines · Christian König
Quoted 7 lines
Quoted 1 line

Are these two effects intentional?

Yes, thought about dropping this as well but testing showed that it is still necessary.

The total amount of memory bound to the GPU must be limited by the GART size or otherwise the swapping code won't work any more. E.g. suspend/resume fails immediately if I remove that.

Has that changed? I don't remember seeing any changes to that effect.

No, there are still a bunch of problems to solve. I should have put a WIP mark on that patch.

The maximum BO size is limited with this patch and will probably result in a bunch of S3 problems.

But as long as you don't care about those limitations it should work.

Christian.

Quoted 185 lines

Regards,
Felix

Quoted 179 lines · Christian König

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h         | 2 ++
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c  | 6 ++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c     | 4 ++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c    | 8 ++++++--
  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c     | 2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++--
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 6 +++---
  7 files changed, 26 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
index ab1dad2..a511029 100644
@@ -76,6 +76,7 @@
  extern int amdgpu_modeset;
  extern int amdgpu_vram_limit;
  extern int amdgpu_gart_size;
+extern unsigned amdgpu_gart_sys_limit;
  extern int amdgpu_moverate;
  extern int amdgpu_benchmarking;
  extern int amdgpu_testing;
@@ -602,6 +603,7 @@ struct amdgpu_mc {
  	u64			mc_vram_size;
  	u64			visible_vram_size;
  	u64			gtt_size;
+	u64			gtt_sys_limit;
  	u64			gtt_start;
  	u64			gtt_end;
  	u64			vram_start;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 44484bb..b6edb83 100644
@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
  		}
  	}
 
+	if (amdgpu_gart_sys_limit < 32) {
+		dev_warn(adev->dev, "gart sys limit (%d) too small\n",
+				 amdgpu_gart_sys_limit);
+			amdgpu_gart_sys_limit = 32;
+	}
+
  	amdgpu_check_vm_size(adev);
 
  	amdgpu_check_block_size(adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
index 5a1d794..907ae5e 100644
@@ -75,6 +75,7 @@
 
  int amdgpu_vram_limit = 0;
  int amdgpu_gart_size = -1; /* auto */
+unsigned amdgpu_gart_sys_limit = 256;
  int amdgpu_moverate = -1; /* auto */
  int amdgpu_benchmarking = 0;
  int amdgpu_testing = 0;
@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
  MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)");
  module_param_named(gartsize, amdgpu_gart_size, int, 0600);
 
+MODULE_PARM_DESC(gartlimit, "GART limit for the system domain in megabytes (default 256)");
+module_param_named(gartlimit, amdgpu_gart_sys_limit, int, 0600);
+
  MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
  module_param_named(moverate, amdgpu_moverate, int, 0600);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6 -2
index 8877015..5c6a461 100644
@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
  					adev->mc.mc_vram_size);
  	else
  		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+
+	adev->mc.gtt_sys_limit = min((uint64_t)amdgpu_gart_sys_limit << 20,
+				     adev->mc.gtt_size);
  }
 
  /**
@@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
  	if (r)
  		return r;
  	/* Compute table size */
-	adev->gart.num_cpu_pages = adev->mc.gtt_size / PAGE_SIZE;
-	adev->gart.num_gpu_pages = adev->mc.gtt_size / AMDGPU_GPU_PAGE_SIZE;
+	adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit / PAGE_SIZE;
+	adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit /
+		AMDGPU_GPU_PAGE_SIZE;
  	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
  		 adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1 -1
index 96c4493..6eac481 100644
@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
  		/* Maximum bo size is the unpinned gtt size since we use the gtt to
  		 * handle vram to system pool migrations.
  		 */
-		max_size = adev->mc.gtt_size - adev->gart_pin_size;
+		max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;
  		if (size > max_size) {
  			DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n",
  				  size >> 20, max_size >> 20);
drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4 -2
index f7d22c4..7609229 100644
@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {
  static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
  			       unsigned long p_size)
  {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
  	struct amdgpu_gtt_mgr *mgr;
 
  	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
  	if (!mgr)
  		return -ENOMEM;
 
-	drm_mm_init(&mgr->mm, 0, p_size);
+	drm_mm_init(&mgr->mm, 0, adev->mc.gtt_sys_limit >> PAGE_SHIFT);
  	spin_lock_init(&mgr->lock);
  	mgr->available = p_size;
  	man->priv = mgr;
@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
  			 const struct ttm_place *place,
  			 struct ttm_mem_reg *mem)
  {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
  	struct amdgpu_gtt_mgr *mgr = man->priv;
  	struct drm_mm_node *node = mem->mm_node;
  	enum drm_mm_insert_mode mode;
@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
  	if (place && place->lpfn)
  		lpfn = place->lpfn;
  	else
-		lpfn = man->size;
+		lpfn = adev->gart.num_cpu_pages;
 
  	mode = DRM_MM_INSERT_BEST;
  	if (place && place->flags & TTM_PL_FLAG_TOPDOWN)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -3
index e4860ac..1e0fcb6 100644
@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
  				 * allocating address space for the BO.
  				 */
  				abo->placements[i].lpfn =
-					adev->mc.gtt_size >> PAGE_SHIFT;
+					adev->gart.num_cpu_pages;
  			}
  		}
  		break;
@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
  	placement.num_busy_placement = 1;
  	placement.busy_placement = &placements;
  	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
  	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
  			     interruptible, no_wait_gpu);
@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
  	placement.num_busy_placement = 1;
  	placement.busy_placement = &placements;
  	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, · line 390

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -0
+	placements.lpfn = adev->gart.num_cpu_pages;
  	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
  	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
  			     interruptible, no_wait_gpu);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#10 lore raw

Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

In reply to a message not in the archive
Quoted 7 lines · Christian König

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

Hmm.

On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up 1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure. But in the grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth cutting GTT memory by a factor of 16?

John

Quoted 179 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h         | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c  | 6 ++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c     | 4 ++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c    | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c     | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c | 6 ++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c     | 6 +++---
 7 files changed, 26 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
index ab1dad2..a511029 100644
@@ -76,6 +76,7 @@
 extern int amdgpu_modeset;
 extern int amdgpu_vram_limit;
 extern int amdgpu_gart_size;
+extern unsigned amdgpu_gart_sys_limit;
 extern int amdgpu_moverate;
 extern int amdgpu_benchmarking;
 extern int amdgpu_testing;
@@ -602,6 +603,7 @@ struct amdgpu_mc {
 	u64			mc_vram_size;
 	u64			visible_vram_size;
 	u64			gtt_size;
+	u64			gtt_sys_limit;
 	u64			gtt_start;
 	u64			gtt_end;
 	u64			vram_start;
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index 44484bb..b6edb83 100644
@@ -1122,6 +1122,12 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 		}
 	}
 
+	if (amdgpu_gart_sys_limit < 32) {
+		dev_warn(adev->dev, "gart sys limit (%d) too small\n",
+				 amdgpu_gart_sys_limit);
+			amdgpu_gart_sys_limit = 32;
+	}
+
 	amdgpu_check_vm_size(adev);
 
 	amdgpu_check_block_size(adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
index 5a1d794..907ae5e 100644
@@ -75,6 +75,7 @@
 
 int amdgpu_vram_limit = 0;
 int amdgpu_gart_size = -1; /* auto */
+unsigned amdgpu_gart_sys_limit = 256;
 int amdgpu_moverate = -1; /* auto */
 int amdgpu_benchmarking = 0;
 int amdgpu_testing = 0;
@@ -124,6 +125,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)");
 module_param_named(gartsize, amdgpu_gart_size, int, 0600);
 
+MODULE_PARM_DESC(gartlimit, "GART limit for the system domain in megabytes (default 256)");
+module_param_named(gartlimit, amdgpu_gart_sys_limit, int, 0600);
+
 MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
 module_param_named(moverate, amdgpu_moverate, int, 0600);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +6 -2
index 8877015..5c6a461 100644
@@ -70,6 +70,9 @@ void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
 					adev->mc.mc_vram_size);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+
+	adev->mc.gtt_sys_limit = min((uint64_t)amdgpu_gart_sys_limit << 20,
+				     adev->mc.gtt_size);
 }
 
 /**
@@ -350,8 +353,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 	/* Compute table size */
-	adev->gart.num_cpu_pages = adev->mc.gtt_size / PAGE_SIZE;
-	adev->gart.num_gpu_pages = adev->mc.gtt_size / AMDGPU_GPU_PAGE_SIZE;
+	adev->gart.num_cpu_pages = adev->mc.gtt_sys_limit / PAGE_SIZE;
+	adev->gart.num_gpu_pages = adev->mc.gtt_sys_limit /
+		AMDGPU_GPU_PAGE_SIZE;
 	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
 		 adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
 
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +1 -1
index 96c4493..6eac481 100644
@@ -62,7 +62,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 		/* Maximum bo size is the unpinned gtt size since we use the gtt to
 		 * handle vram to system pool migrations.
 		 */
-		max_size = adev->mc.gtt_size - adev->gart_pin_size;
+		max_size = adev->mc.gtt_sys_limit - adev->gart_pin_size;
 		if (size > max_size) {
 			DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n",
 				  size >> 20, max_size >> 20);
drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +4 -2
index f7d22c4..7609229 100644
@@ -42,13 +42,14 @@ struct amdgpu_gtt_mgr {
 static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 			       unsigned long p_size)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr;
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
 		return -ENOMEM;
 
-	drm_mm_init(&mgr->mm, 0, p_size);
+	drm_mm_init(&mgr->mm, 0, adev->mc.gtt_sys_limit >> PAGE_SHIFT);
 	spin_lock_init(&mgr->lock);
 	mgr->available = p_size;
 	man->priv = mgr;
@@ -95,6 +96,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 			 const struct ttm_place *place,
 			 struct ttm_mem_reg *mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr = man->priv;
 	struct drm_mm_node *node = mem->mm_node;
 	enum drm_mm_insert_mode mode;
@@ -112,7 +114,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 	if (place && place->lpfn)
 		lpfn = place->lpfn;
 	else
-		lpfn = man->size;
+		lpfn = adev->gart.num_cpu_pages;
 
 	mode = DRM_MM_INSERT_BEST;
 	if (place && place->flags & TTM_PL_FLAG_TOPDOWN)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +3 -3
index e4860ac..1e0fcb6 100644
@@ -221,7 +221,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				 * allocating address space for the BO.
 				 */
 				abo->placements[i].lpfn =
-					adev->mc.gtt_size >> PAGE_SHIFT;
+					adev->gart.num_cpu_pages;
 			}
 		}
 		break;
@@ -384,7 +384,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
@@ -431,7 +431,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = adev->gart.num_cpu_pages;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
Signature · 1 line
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 20170626233901.GA30922-6hIufAJW0g7Gr8qjsLp7YGXnswh1EIUO@public.gmane.org

#12 lore raw

Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Quoted 9 lines · John Brooks
Quoted 7 lines · Christian König

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

Quoted 7 lines

Hmm.

On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up 1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure. But in the grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth cutting GTT memory by a factor of 16?

I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.

Signature · 6 lines
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): e3546936-bd31-d650-b5c8-807822213827-otUistvHUpPR7s880joybQ@public.gmane.org

#14 lore raw

Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Am 27.06.2017 um 04:57 schrieb Michel Dänzer:

Quoted 8 lines · John Brooks
Quoted 7 lines · Christian König
Quoted 6 lines

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

Quoted 8 lines
Quoted 8 lines
Quoted 1 line

Hmm.

On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up 1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure. But in the grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth cutting GTT memory by a factor of 16?

The amount of GTT memory the application can use through the VM page tables stays the same.

Only the system VM is limited to 256MB and so saves us a whole bunch of space.

Quoted 3 lines

I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.

Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.

Christian.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 7c144cd0-a5e2-7a6c-7192-194fdee000c2-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#16 lore raw

Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Quoted 26 lines · Christian König

Am 27.06.2017 um 04:57 schrieb Michel Dänzer:

Quoted 12 lines · John Brooks
Quoted 11 lines · Christian König
Quoted 8 lines

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

Hmm.

Quoted 9 lines
Quoted 9 lines

On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up
1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the
size of the table to 512 KiB. A relatively large saving, to be sure. But in the
grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth
cutting GTT memory by a factor of 16?

The amount of GTT memory the application can use through the VM page tables stays the same.

Quoted 8 lines

Only the system VM is limited to 256MB and so saves us a whole bunch of space.

Quoted 3 lines

I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.

Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.

OK, I should adapt this for the KFD branch. Currently we make our GART much bigger. On a system with lots of system memory, we can use up half the visible VRAM for the GART. With something like this we can get it back to something reasonable. But a 256MB limit on single allocation size is definitely too small.

Also, if this breaks S3, we have to make sure the hybrid branches don't pick it up accidentally.

Regards,
Felix

Quoted 2 lines

Christian.

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 300c3193-1341-934b-4fdd-f484fdd18e89-5C7GfCeVMHo@public.gmane.org

#18 lore raw

RE: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Yeah, recently we indeed encountered an issue as below:
On workstation machine it has 32G system memory and thus the GTT table size takes up about 192M visible VRAM. Maybe if system has bigger memory, GTT table size will exceed 256M, so Limitation for max GTT table size is very necessary.

Thanks
Roger(Hongbo.He)
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Wednesday, June 28, 2017 3:25 AM
To: Christian König <[email protected]>; Michel Dänzer <[email protected]>; John Brooks <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Quoted 26 lines · Christian König

Am 27.06.2017 um 04:57 schrieb Michel Dänzer:

Quoted 12 lines · John Brooks
Quoted 11 lines · Christian König
Quoted 8 lines

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

Hmm.

Quoted 9 lines
Quoted 9 lines

On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up
1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure.
But in the
grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth
cutting GTT memory by a factor of 16?

The amount of GTT memory the application can use through the VM page tables stays the same.

Quoted 8 lines

Only the system VM is limited to 256MB and so saves us a whole bunch of space.

Quoted 3 lines

I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.

Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.

OK, I should adapt this for the KFD branch. Currently we make our GART much bigger. On a system with lots of system memory, we can use up half the visible VRAM for the GART. With something like this we can get it back to something reasonable. But a 256MB limit on single allocation size is definitely too small.

Also, if this breaks S3, we have to make sure the hybrid branches don't pick it up accidentally.

Regards,
Felix

Quoted 2 lines

Christian.

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#19 lore raw

RE: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

In reply to Felix Kuehling

Setting limitation for max GTT table size is necessary.
But my only concern is that is it enough for amdgpu_gart_sys_limit 256M. whether it will trigger GTT eviction from GTT to CPU domain easily? Because GTT size always equals to VRAM size before it scales with system memory size.

Thanks
Roger(Hongbo.He)
-----Original Message-----
From: He, Roger
Sent: Wednesday, June 28, 2017 9:58 AM
To: 'Felix Kuehling' <[email protected]>; Christian König <[email protected]>; Michel Dänzer <[email protected]>; John Brooks <[email protected]>
Cc: [email protected]
Subject: RE: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Yeah, recently we indeed encountered an issue as below:
On workstation machine it has 32G system memory and thus the GTT table size takes up about 192M visible VRAM. Maybe if system has bigger memory, GTT table size will exceed 256M, so Limitation for max GTT table size is very necessary.

Thanks
Roger(Hongbo.He)
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Wednesday, June 28, 2017 3:25 AM
To: Christian König <[email protected]>; Michel Dänzer <[email protected]>; John Brooks <[email protected]>
Cc: [email protected]
Subject: Re: [PATCH 3/3] drm/amdgpu: add gtt_sys_limit

Quoted 26 lines · Christian König

Am 27.06.2017 um 04:57 schrieb Michel Dänzer:

Quoted 12 lines · John Brooks
Quoted 11 lines · Christian König
Quoted 8 lines

From: Christian König <[email protected]>

Limit the size of the GART table for the system domain.

This saves us a bunch of visible VRAM, but also limitates the maximum BO size we can swap out.

Hmm.

Quoted 9 lines
Quoted 9 lines

On my system, GTT is 4096MiB (1048576 pages). For this, the table takes up
1048576*8 bytes = 8MiB. Reducing GTT to 256MiB (65536 pages) would reduce the size of the table to 512 KiB. A relatively large saving, to be sure.
But in the
grander scheme of things, is saving 7.5MiB (3% of visible VRAM @ 256M) worth
cutting GTT memory by a factor of 16?

The amount of GTT memory the application can use through the VM page tables stays the same.

Quoted 8 lines

Only the system VM is limited to 256MB and so saves us a whole bunch of space.

Quoted 3 lines

I'm afraid not, especially since it would limit the maximum BO size to < 256MB, if I understand correctly. Pretty sure that would cause failures with real world apps.

Yes, exactly that's the major problem here. I should have put a WIP mark on the patch.

OK, I should adapt this for the KFD branch. Currently we make our GART much bigger. On a system with lots of system memory, we can use up half the visible VRAM for the GART. With something like this we can get it back to something reasonable. But a 256MB limit on single allocation size is definitely too small.

Also, if this breaks S3, we have to make sure the hybrid branches don't pick it up accidentally.

Regards,
Felix

Quoted 2 lines

Christian.

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Reviewed-by Alex Deucher
#20 lore raw
In reply to a message not in the archive
Quoted 8 lines

-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian König
Sent: Monday, June 26, 2017 9:40 AM
To: [email protected]
Cc: [email protected]; Kuehling, Felix
Subject: [PATCH 1/3] drm/amdgpu: cleanup initializing gtt_size

Quoted 5 lines

From: Christian König <[email protected]>

Stop spreading the code over all GMC generations.

Quoted 152 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h      |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 20
++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c    | 10 +---------
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 10 +---------
 6 files changed, 25 insertions(+), 36 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 7624294..ab1dad2 100644
@@ -561,6 +561,7 @@ struct amdgpu_gart {
 	const struct amdgpu_gart_funcs *gart_funcs;
 };
 
+void amdgpu_gart_set_defaults(struct amdgpu_device *adev);
 int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
 void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
 int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +20 -0
index ccef3cf..8877015 100644
@@ -52,6 +52,26 @@
 /*
  * Common GART table functions.
  */
+
+/**
+ * amdgpu_gart_set_defaults - set the default gtt_size
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set the default gtt_size based on parameters and available VRAM.
+ */
+void amdgpu_gart_set_defaults(struct amdgpu_device *adev)
+{
+	/* unless the user had overridden it, set the gart
+	 * size equal to the 1024 or vram, whichever is larger.
+	 */
+	if (amdgpu_gart_size == -1)
+		adev->mc.gtt_size =
 max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
+					adev->mc.mc_vram_size);
+	else
+		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+}
+
 /**
  * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table

*

drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +1 -9
index 35655a1..5cc3f39 100644
@@ -321,15 +321,7 @@ static int gmc_v6_0_mc_init(struct amdgpu_device
 *adev)
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) *
 1024ULL * 1024ULL;
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size =
 max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +0 -0
 	gmc_v6_0_vram_gtt_location(adev, &adev->mc);
 
 	return 0;
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1 -9
index 83c21a1..15f2c0f 100644
@@ -373,15 +373,7 @@ static int gmc_v7_0_mc_init(struct amdgpu_device
 *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size =
 max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v7_0_vram_gtt_location(adev, &adev->mc);
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +0 -0
 	return 0;
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1 -9
index c92af24..213af65 100644
@@ -535,15 +535,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device
 *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size =
 max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v8_0_vram_gtt_location(adev, &adev->mc);
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +0 -0
 	return 0;
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -9
index 7c6188b..128989e 100644
@@ -495,15 +495,7 @@ static int gmc_v9_0_mc_init(struct amdgpu_device
 *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size =
 max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
-
+	amdgpu_gart_set_defaults(adev);
 	gmc_v9_0_vram_gtt_location(adev, &adev->mc);
Quoted 8 lines

return 0;
--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx