Skip to contentSkip to conversation

drm/amdgpu: Export setup_vm_pt_regs() logic for gfxhub 2.0

PATCH In review Zhao, Yong 3 participants 9 messages Sep 25, 2019 – Sep 27, 2019 6 messages not in the archive

Subsystems (3)
lore
Share & tools
lore

Zhao, Yong

author
#0 lore raw

PATCH 1/3 drm/amdgpu: Export setup_vm_pt_regs() logic for gfxhub 2.0

The KFD code will call this function later.

Change-Id I88a53368cdee719b2c75393e5cdbd8290584548e
---
 drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 20 ++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h |  2 ++
 2 files changed, 14 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +12 -8
index a9238735d361..b601c6740ef5 100644
@@ -46,21 +46,25 @@ u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev)
 	return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24;
 }
 
-static void gfxhub_v2_0_init_gart_pt_regs(struct amdgpu_device *adev)
+void gfxhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+				uint64_t page_table_base)
 {
-	uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo);
+	/* two registers distance between mmGCVM_CONTEXT0_* to mmGCVM_CONTEXT1_* */
+	int offset = mmGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+			- mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
 
+	WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+				offset * vmid, lower_32_bits(page_table_base));
 
-	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
-		     lower_32_bits(value));
-
-	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
-		     upper_32_bits(value));
+	WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+				offset * vmid, upper_32_bits(page_table_base));
 }
 
 static void gfxhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 {
-	gfxhub_v2_0_init_gart_pt_regs(adev);
+	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	gfxhub_v2_0_setup_vm_pt_regs(adev, 0, pt_base);
 
 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
 		     (u32)(adev->gmc.gart_start >> 12));
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h +2 -0
index 06807940748b..392b8cd94fc0 100644
@@ -31,5 +31,7 @@ void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
 					  bool value);
 void gfxhub_v2_0_init(struct amdgpu_device *adev);
 u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev);
+void gfxhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+				uint64_t page_table_base);
 
 #endif
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#2 lore raw

PATCH 2/3 drm/amdkfd: Use setup_vm_pt_regs function from base driver in KFD

This was done on GFX9 previously, now do it for GFX10.

Change-Id I4442e60534c59bc9526a673559f018ba8058deac
---
 .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c    | 23 +++----------------
 1 file changed, 3 insertions(+), 20 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +3 -20
index fe5b702c75ce..64568ed32793 100644
@@ -42,6 +42,7 @@
 #include "v10_structs.h"
 #include "nv.h"
 #include "nvd.h"
+#include "gfxhub_v2_0.h"
 
 enum hqd_dequeue_request_type {
 	NO_ACTION = 0,
@@ -251,11 +252,6 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
 			ATC_VMID0_PASID_MAPPING__VALID_MASK;
 
 	pr_debug("pasid 0x%x vmid %d, reg value %x\n", pasid, vmid, pasid_mapping);
-	/*
-	 * need to do this twice, once for gfx and once for mmhub
-	 * for ATC add 16 to VMID for mmhub, for IH different registers.
-	 * ATC_VMID0..15 registers are separate from ATC_VMID16..31.
-	 */
 
 	pr_debug("ATHUB, reg %x\n", SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid);
 	WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
@@ -910,7 +906,6 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 		uint64_t page_table_base)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	uint64_t base = page_table_base | AMDGPU_PTE_VALID;
 
 	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
 		pr_err("trying to set page table base for wrong VMID %u\n",
@@ -918,18 +913,6 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 		return;
 	}
 
-	/* TODO: take advantage of per-process address space size. For
-	 * now, all processes share the same address space size, like
-	 * on GFX8 and older.
-	 */
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
-
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
-			lower_32_bits(adev->vm_manager.max_pfn - 1));
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
-			upper_32_bits(adev->vm_manager.max_pfn - 1));
-
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+	/* SDMA is on gfxhub as well on Navi1* series */
+	gfxhub_v2_0_setup_vm_pt_regs(adev, vmid, page_table_base);
 }
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

Reviewed-by Felix Kuehling
#4 lore raw

Re: [PATCH 2/3] drm/amdkfd: Use setup_vm_pt_regs function from base driver in KFD

Quoted 4 lines · Yong

This was done on GFX9 previously, now do it for GFX10.

Change-Id I4442e60534c59bc9526a673559f018ba8058deac
Quoted 47 lines
---
  .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c    | 23 +++----------------
  1 file changed, 3 insertions(+), 20 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +1 -13
index fe5b702c75ce..64568ed32793 100644
@@ -42,6 +42,7 @@
  #include "v10_structs.h"
  #include "nv.h"
  #include "nvd.h"
+#include "gfxhub_v2_0.h"
 
  enum hqd_dequeue_request_type {
  	NO_ACTION = 0,
@@ -251,11 +252,6 @@ static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
  			ATC_VMID0_PASID_MAPPING__VALID_MASK;
 
  	pr_debug("pasid 0x%x vmid %d, reg value %x\n", pasid, vmid, pasid_mapping);
-	/*
-	 * need to do this twice, once for gfx and once for mmhub
-	 * for ATC add 16 to VMID for mmhub, for IH different registers.
-	 * ATC_VMID0..15 registers are separate from ATC_VMID16..31.
-	 */
 
  	pr_debug("ATHUB, reg %x\n", SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid);
  	WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
@@ -910,7 +906,6 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
  		uint64_t page_table_base)
  {
  	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	uint64_t base = page_table_base | AMDGPU_PTE_VALID;
 
  	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
  		pr_err("trying to set page table base for wrong VMID %u\n",
@@ -918,18 +913,6 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
  		return;
  	}
 
-	/* TODO: take advantage of per-process address space size. For
-	 * now, all processes share the same address space size, like
-	 * on GFX8 and older.
-	 */
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
-

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, · line 918

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +2 -7
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
-			lower_32_bits(adev->vm_manager.max_pfn - 1));
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
-			upper_32_bits(adev->vm_manager.max_pfn - 1));
-
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
-	WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+	/* SDMA is on gfxhub as well on Navi1* series */
+	gfxhub_v2_0_setup_vm_pt_regs(adev, vmid, page_table_base);
  }

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

author
#5 lore raw

PATCH 3/3 drm/amdkfd: Remove the control stack workaround for GFX10

In reply to a message not in the archive

The GFX10 does not have this hardware bug any more, so remove it.

Change-Id I446c9685549a09ac8846a42ee22d86cfb93fd98c
---
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  | 37 ++-----------------
 1 file changed, 4 insertions(+), 33 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +4 -33
index 9cd3eb2d90bd..4a236b2c2354 100644
@@ -69,35 +69,13 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
 static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
 		struct queue_properties *q)
 {
-	int retval;
-	struct kfd_mem_obj *mqd_mem_obj = NULL;
+	struct kfd_mem_obj *mqd_mem_obj;
 
-	/* From V9,  for CWSR, the control stack is located on the next page
-	 * boundary after the mqd, we will use the gtt allocation function
-	 * instead of sub-allocation function.
-	 */
-	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
-		mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
-		if (!mqd_mem_obj)
-			return NULL;
-		retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
-			ALIGN(q->ctl_stack_size, PAGE_SIZE) +
-				ALIGN(sizeof(struct v10_compute_mqd), PAGE_SIZE),
-			&(mqd_mem_obj->gtt_mem),
-			&(mqd_mem_obj->gpu_addr),
-			(void *)&(mqd_mem_obj->cpu_ptr), true);
-	} else {
-		retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
-				&mqd_mem_obj);
-	}
-
-	if (retval) {
-		kfree(mqd_mem_obj);
+	if (kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
+			&mqd_mem_obj))
 		return NULL;
-	}
 
 	return mqd_mem_obj;
-
 }
 
 static void init_mqd(struct mqd_manager *mm, void **mqd,
@@ -250,14 +228,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
 static void free_mqd(struct mqd_manager *mm, void *mqd,
 			struct kfd_mem_obj *mqd_mem_obj)
 {
-	struct kfd_dev *kfd = mm->dev;
-
-	if (mqd_mem_obj->gtt_mem) {
-		amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
-		kfree(mqd_mem_obj);
-	} else {
-		kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
-	}
+	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
 }
 
 static bool is_occupied(struct mqd_manager *mm, void *mqd,
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

#7 lore raw

Re: [PATCH 3/3] drm/amdkfd: Remove the control stack workaround for GFX10

Quoted 1 line · Yong

The GFX10 does not have this hardware bug any more, so remove it.

I wouldn't call this a bug and a workaround. More like a change in the HW or FW behaviour and a corresponding driver change. I.e. in GFXv8 the control stack was in the user mode CWSR allocation. In GFXv9 it moved into a kernel mode buffer next to the MQD. So in GFXv10 the control stack moved back into the user mode CWSR buffer?

Regards,
  Felix

Quoted 56 lines
Change-Id I446c9685549a09ac8846a42ee22d86cfb93fd98c
---
  .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  | 37 ++-----------------
  1 file changed, 4 insertions(+), 33 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +3 -27
index 9cd3eb2d90bd..4a236b2c2354 100644
@@ -69,35 +69,13 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
  static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
  		struct queue_properties *q)
  {
-	int retval;
-	struct kfd_mem_obj *mqd_mem_obj = NULL;
+	struct kfd_mem_obj *mqd_mem_obj;
 
-	/* From V9,  for CWSR, the control stack is located on the next page
-	 * boundary after the mqd, we will use the gtt allocation function
-	 * instead of sub-allocation function.
-	 */
-	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
-		mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
-		if (!mqd_mem_obj)
-			return NULL;
-		retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
-			ALIGN(q->ctl_stack_size, PAGE_SIZE) +
-				ALIGN(sizeof(struct v10_compute_mqd), PAGE_SIZE),
-			&(mqd_mem_obj->gtt_mem),
-			&(mqd_mem_obj->gpu_addr),
-			(void *)&(mqd_mem_obj->cpu_ptr), true);
-	} else {
-		retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
-				&mqd_mem_obj);
-	}
-
-	if (retval) {
-		kfree(mqd_mem_obj);
+	if (kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
+			&mqd_mem_obj))
  		return NULL;
-	}
 
  	return mqd_mem_obj;
-
  }
 
  static void init_mqd(struct mqd_manager *mm, void **mqd,
@@ -250,14 +228,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
  static void free_mqd(struct mqd_manager *mm, void *mqd,
  			struct kfd_mem_obj *mqd_mem_obj)
  {
-	struct kfd_dev *kfd = mm->dev;
-

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c static int destroy_mqd(struct mqd_manager *mm, void *mqd, · line 234

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +1 -6
-	if (mqd_mem_obj->gtt_mem) {
-		amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
-		kfree(mqd_mem_obj);
-	} else {
-		kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
-	}
+	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
  }
 
  static bool is_occupied(struct mqd_manager *mm, void *mqd,

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 3d63ddfd-2773-b89d-c0dc-75ac1397b1fc-5C7GfCeVMHo@public.gmane.org

author
#9 lore raw

Re: [PATCH 3/3] drm/amdkfd: Remove the control stack workaround for GFX10

Yes. I confirmed with CP guys and they said the behavior on GFX10 is the same as GFX8 now. I remember that the workaround on GFX9 was to help with a HW bug, but not too sure.

Regards,

Yong

Quoted 75 lines · Felix
Quoted 1 line · Yong

The GFX10 does not have this hardware bug any more, so remove it.

I wouldn't call this a bug and a workaround. More like a change in the HW or FW behaviour and a corresponding driver change. I.e. in GFXv8 the control stack was in the user mode CWSR allocation. In GFXv9 it moved into a kernel mode buffer next to the MQD. So in GFXv10 the control stack moved back into the user mode CWSR buffer?

Regards,
  Felix

Quoted 64 lines
Change-Id I446c9685549a09ac8846a42ee22d86cfb93fd98c
---
   .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  | 37 ++-----------------
   1 file changed, 4 insertions(+), 33 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +4 -33
index 9cd3eb2d90bd..4a236b2c2354 100644
@@ -69,35 +69,13 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
   static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
   		struct queue_properties *q)
   {
-	int retval;
-	struct kfd_mem_obj *mqd_mem_obj = NULL;
+	struct kfd_mem_obj *mqd_mem_obj;
 
-	/* From V9,  for CWSR, the control stack is located on the next page
-	 * boundary after the mqd, we will use the gtt allocation function
-	 * instead of sub-allocation function.
-	 */
-	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
-		mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
-		if (!mqd_mem_obj)
-			return NULL;
-		retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
-			ALIGN(q->ctl_stack_size, PAGE_SIZE) +
-				ALIGN(sizeof(struct v10_compute_mqd), PAGE_SIZE),
-			&(mqd_mem_obj->gtt_mem),
-			&(mqd_mem_obj->gpu_addr),
-			(void *)&(mqd_mem_obj->cpu_ptr), true);
-	} else {
-		retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
-				&mqd_mem_obj);
-	}
-
-	if (retval) {
-		kfree(mqd_mem_obj);
+	if (kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
+			&mqd_mem_obj))
   		return NULL;
-	}
 
   	return mqd_mem_obj;
-
   }
 
   static void init_mqd(struct mqd_manager *mm, void **mqd,
@@ -250,14 +228,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
   static void free_mqd(struct mqd_manager *mm, void *mqd,
   			struct kfd_mem_obj *mqd_mem_obj)
   {
-	struct kfd_dev *kfd = mm->dev;
-
-	if (mqd_mem_obj->gtt_mem) {
-		amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
-		kfree(mqd_mem_obj);
-	} else {
-		kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
-	}
+	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
   }
 
Quoted 1 line

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c static int destroy_mqd(struct mqd_manager *mm, void *mqd, · line 234

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +0 -0
   static bool is_occupied(struct mqd_manager *mm, void *mqd,

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 09afdf34-0c72-5306-103a-1290734fa1a2-5C7GfCeVMHo@public.gmane.org

author
#11 lore raw

Re: [PATCH 3/3] drm/amdkfd: Remove the control stack workaround for GFX10

Hi Felix,

I reworded this patch in the next series. Please review the first two patches in this series.

Regards,
Yong
________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Zhao, Yong <[email protected]>
Sent: Wednesday, September 25, 2019 2:34 PM
To: Kuehling, Felix <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH 3/3] drm/amdkfd: Remove the control stack workaround for GFX10

Yes. I confirmed with CP guys and they said the behavior on GFX10 is the same as GFX8 now. I remember that the workaround on GFX9 was to help with a HW bug, but not too sure.

Regards,

Yong

Quoted 75 lines · Felix
Quoted 1 line · Yong

The GFX10 does not have this hardware bug any more, so remove it.

I wouldn't call this a bug and a workaround. More like a change in the HW or FW behaviour and a corresponding driver change. I.e. in GFXv8 the control stack was in the user mode CWSR allocation. In GFXv9 it moved into a kernel mode buffer next to the MQD. So in GFXv10 the control stack moved back into the user mode CWSR buffer?

Regards,
Felix

Quoted 64 lines
Change-Id I446c9685549a09ac8846a42ee22d86cfb93fd98c
---
   .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  | 37 ++-----------------
   1 file changed, 4 insertions(+), 33 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +4 -33
index 9cd3eb2d90bd..4a236b2c2354 100644
@@ -69,35 +69,13 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
   static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
              struct queue_properties *q)
   {
-    int retval;
-    struct kfd_mem_obj *mqd_mem_obj = NULL;
+    struct kfd_mem_obj *mqd_mem_obj;
 
-    /* From V9,  for CWSR, the control stack is located on the next page
-     * boundary after the mqd, we will use the gtt allocation function
-     * instead of sub-allocation function.
-     */
-    if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
-            mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
-            if (!mqd_mem_obj)
-                    return NULL;
-            retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
-                    ALIGN(q->ctl_stack_size, PAGE_SIZE) +
-                            ALIGN(sizeof(struct v10_compute_mqd), PAGE_SIZE),
-                    &(mqd_mem_obj->gtt_mem),
-                    &(mqd_mem_obj->gpu_addr),
-                    (void *)&(mqd_mem_obj->cpu_ptr), true);
-    } else {
-            retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
-                            &mqd_mem_obj);
-    }
-
-    if (retval) {
-            kfree(mqd_mem_obj);
+    if (kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
+                    &mqd_mem_obj))
              return NULL;
-    }
 
      return mqd_mem_obj;
-
   }
 
   static void init_mqd(struct mqd_manager *mm, void **mqd,
@@ -250,14 +228,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
   static void free_mqd(struct mqd_manager *mm, void *mqd,
                      struct kfd_mem_obj *mqd_mem_obj)
   {
-    struct kfd_dev *kfd = mm->dev;
-
-    if (mqd_mem_obj->gtt_mem) {
-            amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
-            kfree(mqd_mem_obj);
-    } else {
-            kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
-    }
+    kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
   }
 
Quoted 1 line

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c static int destroy_mqd(struct mqd_manager *mm, void *mqd, · line 234

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +0 -0
   static bool is_occupied(struct mqd_manager *mm, void *mqd,

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Reviewed-by Felix Kuehling
#12 lore raw
In reply to a message not in the archive

For GFXv9 you made an equivalent change for both GFXHub and MMHub ("drm/amdgpu: Expose *_setup_vm_pt_regs for kfd to use"). Your GFXv9 commit was also reviewed by Alex and Christian. You should get at least one of them to Ack or Review this change.

For GFXv10 you're only changing the GFXHub. I suspect that's because KFD doesn't care about MMHub on GFXv10. That's fine with me.

You can add

Thanks,
  Felix

Quoted 52 lines · Yong

The KFD code will call this function later.

Change-Id I88a53368cdee719b2c75393e5cdbd8290584548e
---
  drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 20 ++++++++++++--------
  drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h |  2 ++
  2 files changed, 14 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +12 -8
index a9238735d361..b601c6740ef5 100644
@@ -46,21 +46,25 @@ u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev)
  	return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24;
  }
 
-static void gfxhub_v2_0_init_gart_pt_regs(struct amdgpu_device *adev)
+void gfxhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+				uint64_t page_table_base)
  {
-	uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo);
+	/* two registers distance between mmGCVM_CONTEXT0_* to mmGCVM_CONTEXT1_* */
+	int offset = mmGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+			- mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
 
+	WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+				offset * vmid, lower_32_bits(page_table_base));
 
-	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
-		     lower_32_bits(value));
-
-	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
-		     upper_32_bits(value));
+	WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+				offset * vmid, upper_32_bits(page_table_base));
  }
 
  static void gfxhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
  {
-	gfxhub_v2_0_init_gart_pt_regs(adev);
+	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	gfxhub_v2_0_setup_vm_pt_regs(adev, 0, pt_base);
 
  	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  		     (u32)(adev->gmc.gart_start >> 12));
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h +0 -0
index 06807940748b..392b8cd94fc0 100644

drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, · line 37

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h +2 -0
@@ -31,5 +31,7 @@ void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
  					  bool value);
  void gfxhub_v2_0_init(struct amdgpu_device *adev);
  u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev);
+void gfxhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+				uint64_t page_table_base);
 
  #endif

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 5d25ca56-6604-c7dc-a669-f0feabd5dc5f-5C7GfCeVMHo@public.gmane.org

Reviewed-by Reviewed-by Felix Kuehling
#14 lore raw

________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Kuehling, Felix <[email protected]>
Sent: Thursday, September 26, 2019 6:26 PM
To: Zhao, Yong <[email protected]>; [email protected] <[email protected]>
Subject: Re: [PATCH 1/3] drm/amdgpu: Export setup_vm_pt_regs() logic for gfxhub 2.0

For GFXv9 you made an equivalent change for both GFXHub and MMHub ("drm/amdgpu: Expose *_setup_vm_pt_regs for kfd to use"). Your GFXv9 commit was also reviewed by Alex and Christian. You should get at least one of them to Ack or Review this change.

For GFXv10 you're only changing the GFXHub. I suspect that's because KFD doesn't care about MMHub on GFXv10. That's fine with me.

You can add

Thanks,
Felix

Quoted 52 lines · Yong

The KFD code will call this function later.

Change-Id I88a53368cdee719b2c75393e5cdbd8290584548e
---
  drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c | 20 ++++++++++++--------
  drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h |  2 ++
  2 files changed, 14 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c +12 -8
index a9238735d361..b601c6740ef5 100644
@@ -46,21 +46,25 @@ u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev)
       return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24;
  }
 
-static void gfxhub_v2_0_init_gart_pt_regs(struct amdgpu_device *adev)
+void gfxhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+                             uint64_t page_table_base)
  {
-     uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo);
+     /* two registers distance between mmGCVM_CONTEXT0_* to mmGCVM_CONTEXT1_* */
+     int offset = mmGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+                     - mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
 
+     WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+                             offset * vmid, lower_32_bits(page_table_base));
 
-     WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
-                  lower_32_bits(value));
-
-     WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
-                  upper_32_bits(value));
+     WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+                             offset * vmid, upper_32_bits(page_table_base));
  }
 
  static void gfxhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
  {
-     gfxhub_v2_0_init_gart_pt_regs(adev);
+     uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+     gfxhub_v2_0_setup_vm_pt_regs(adev, 0, pt_base);
 
       WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
                    (u32)(adev->gmc.gart_start >> 12));
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h +0 -0
index 06807940748b..392b8cd94fc0 100644

drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, · line 37

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h +2 -0
@@ -31,5 +31,7 @@ void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
                                         bool value);
  void gfxhub_v2_0_init(struct amdgpu_device *adev);
  u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev);
+void gfxhub_v2_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+                             uint64_t page_table_base);
 
  #endif

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx