Skip to contentSkip to conversation

drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

PATCH In review Emily Deng 4 participants 6 messages Aug 14, 2018 – Aug 15, 2018 4 messages not in the archive

Subsystems (2)
lore
Share & tools
lore

Emily Deng

author
#0 lore raw

PATCH v2 1/1 drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

v2:
Add firmware version checking.

SWDEV-161497

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 63 ++++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 +
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  7 ++++
 3 files changed, 72 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +63 -0
index 21adb1b6..436030c 100644
@@ -233,6 +233,69 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 	pr_err("failed to write reg:%x\n", reg);
 }
 
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid)
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+	struct drm_amdgpu_info_firmware fw_info;
+
+	if (ring->me == 1) {
+		fw_info.ver = adev->gfx.mec_fw_version;
+		fw_info.feature = adev->gfx.mec_feature_version;
+	} else {
+		fw_info.ver = adev->gfx.mec2_fw_version;
+		fw_info.feature = adev->gfx.mec2_feature_version;
+	}
+
+	if (fw_info.ver <  0x00000190 || fw_info.feature < 42)
+		return -EPERM;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return 0;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+	return r;
+}
+
 /**
  * amdgpu_virt_request_full_gpu() - request full gpu access
  * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
index 880ac11..8908ff9 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device *adev);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid);
 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +7 -0
index 6999042..812f71e 100644
@@ -331,6 +331,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
 	/* Use register 17 for GART */
 	const unsigned eng = 17;
 	unsigned i, j;
+	int r;
 
 	spin_lock(&adev->gmc.invalidate_lock);
 
@@ -338,6 +339,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
 		struct amdgpu_vmhub *hub = &adev->vmhub[i];
 		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			r = amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
+			if (!r)
+				continue;
+		}
+
 		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
 		/* Busy wait for ACK.*/
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1534244211-4717-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org

#2 lore raw

Am 14.08.2018 um 12:56 schrieb Emily Deng:

Quoted 20 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

v2:
Add firmware version checking.

SWDEV-161497

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 63 ++++++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 +
  drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  7 ++++
  3 files changed, 72 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..436030c 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
@@ -233,6 +233,69 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
  	pr_err("failed to write reg:%x\n", reg);
  }
 
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid)

Drop that function here. It is Vega specific and doesn't belong into common code.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8 -0
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+	struct drm_amdgpu_info_firmware fw_info;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 255

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	if (ring->me == 1) {
+		fw_info.ver = adev->gfx.mec_fw_version;
+		fw_info.feature = adev->gfx.mec_feature_version;
+	} else {
+		fw_info.ver = adev->gfx.mec2_fw_version;
+		fw_info.feature = adev->gfx.mec2_feature_version;
+	}
+
+	if (fw_info.ver <  0x00000190 || fw_info.feature < 42)
+		return -EPERM;

Please move that check into gfx_v9_0_ring_emit_reg_write_reg_wait().

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 257

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);

That check is superfluous.

Quoted 70 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +41 -0
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return 0;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+	return r;
+}
+
  /**
   * amdgpu_virt_request_full_gpu() - request full gpu access
   * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
index 880ac11..8908ff9 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device *adev);
  void amdgpu_virt_init_setting(struct amdgpu_device *adev);
  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid);
  int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
  int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
  int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
index 6999042..812f71e 100644
@@ -331,6 +331,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
  	/* Use register 17 for GART */
  	const unsigned eng = 17;
  	unsigned i, j;
+	int r;
 
  	spin_lock(&adev->gmc.invalidate_lock);
 

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 347

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +6 -0
@@ -338,6 +339,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
  		struct amdgpu_vmhub *hub = &adev->vmhub[i];
  		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			r = amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
+			if (!r)
+				continue;
+		}
+

Drop that SRIOV check here. We don't want specialized code path for SRIOV.

Christian.

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 350

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
  		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
  		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): a865d2c4-2373-1101-2dd1-ca3a8f0792eb-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#4 lore raw

Thanks for your review, will send a patch to review again as your suggestion. I think it will be better to use the amdgpu_sriov_vf(adev). As need to let somebody to test all your bare metal test cases, such as vega10, and other asics, as I don't have those environments, and only could test sriov. If passed those bare metal test cases and fixed the bugs if occurs, then could remove the amdgpu_sriov_vf(adev). Do you think so?

Best wishes
Emily Deng

Quoted 152 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 8:46 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH v2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 12:56 schrieb Emily Deng:

Quoted 11 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

v2:
Add firmware version checking.

SWDEV-161497

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 63

++++++++++++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device

Quoted 10 lines
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 +
  drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  7 ++++
  3 files changed, 72 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..436030c 100644
@@ -233,6 +233,69 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
  	pr_err("failed to write reg:%x\n", reg);
  }
 
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 237

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
+		unsigned eng, u32 req, uint32_t vmid)

Drop that function here. It is Vega specific and doesn't belong into common code.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8 -0
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+	struct drm_amdgpu_info_firmware fw_info;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 255

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	if (ring->me == 1) {
+		fw_info.ver = adev->gfx.mec_fw_version;
+		fw_info.feature = adev->gfx.mec_feature_version;
+	} else {
+		fw_info.ver = adev->gfx.mec2_fw_version;
+		fw_info.feature = adev->gfx.mec2_feature_version;
+	}
+
+	if (fw_info.ver <  0x00000190 || fw_info.feature < 42)
+		return -EPERM;

Please move that check into gfx_v9_0_ring_emit_reg_write_reg_wait().

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 257

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);

That check is superfluous.

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device

Quoted 21 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return 0;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+	return r;
+}
+
  /**
   * amdgpu_virt_request_full_gpu() - request full gpu access
   * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..8908ff9 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device

*adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device · line 292

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
  void amdgpu_virt_init_setting(struct amdgpu_device *adev);
  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
 uint32_t v);
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
  int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
  int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);

int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git
a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 6999042..812f71e 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -331,6 +331,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
  	/* Use register 17 for GART */
  	const unsigned eng = 17;
  	unsigned i, j;
+	int r;
 
  	spin_lock(&adev->gmc.invalidate_lock);
 
@@ -338,6 +339,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
  		struct amdgpu_vmhub *hub = &adev->vmhub[i];
  		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			r = amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 12 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 347

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +4 -0
+			if (!r)
+				continue;
+		}
+

Drop that SRIOV check here. We don't want specialized code path for SRIOV.

Christian.

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
  		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
  		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB11250F5E45EBEBDD519056BB8F3F0-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#6 lore raw
Quoted 1 line

As need to let somebody to test all your bare metal test cases, such as vega10, and other asics, as I don't have those environments, and only could test sriov.

That is not much of a problem. We have plenty of people on the mailing list (including me) who can test this on both Vega10 and Raven.

Just remove the SRIOV check and send the patch out with a request for testing it on bare metal.

Christian.

Am 15.08.2018 um 04:28 schrieb Deng, Emily:

Quoted 4 lines

Thanks for your review, will send a patch to review again as your suggestion. I think it will be better to use the amdgpu_sriov_vf(adev). As need to let somebody to test all your bare metal test cases, such as vega10, and other asics, as I don't have those environments, and only could test sriov. If passed those bare metal test cases and fixed the bugs if occurs, then could remove the amdgpu_sriov_vf(adev). Do you think so?

Quoted 162 lines

Best wishes
Emily Deng

Quoted 160 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 8:46 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH v2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 12:56 schrieb Emily Deng:

Quoted 11 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

v2:
Add firmware version checking.

SWDEV-161497

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 63

++++++++++++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device

Quoted 10 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 +
   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  7 ++++
   3 files changed, 72 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..436030c 100644
@@ -233,6 +233,69 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
   	pr_err("failed to write reg:%x\n", reg);
   }
 
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 237

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
+		unsigned eng, u32 req, uint32_t vmid)

Drop that function here. It is Vega specific and doesn't belong into common code.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8 -0
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+	struct drm_amdgpu_info_firmware fw_info;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) · line 255

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	if (ring->me == 1) {
+		fw_info.ver = adev->gfx.mec_fw_version;
+		fw_info.feature = adev->gfx.mec_feature_version;
+	} else {
+		fw_info.ver = adev->gfx.mec2_fw_version;
+		fw_info.feature = adev->gfx.mec2_feature_version;
+	}
+
+	if (fw_info.ver <  0x00000190 || fw_info.feature < 42)
+		return -EPERM;

Please move that check into gfx_v9_0_ring_emit_reg_write_reg_wait().

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 257

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);

That check is superfluous.

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device

Quoted 21 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return 0;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+	return r;
+}
+
   /**
    * amdgpu_virt_request_full_gpu() - request full gpu access
    * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..8908ff9 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device

*adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device · line 292

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
 uint32_t v);
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
   int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git

a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 6999042..812f71e 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -331,6 +331,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
   	/* Use register 17 for GART */
   	const unsigned eng = 17;
   	unsigned i, j;
+	int r;
 
   	spin_lock(&adev->gmc.invalidate_lock);
 
@@ -338,6 +339,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
   		struct amdgpu_vmhub *hub = &adev->vmhub[i];
   		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			r = amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 347

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +4 -0
+			if (!r)
+				continue;
+		}
+

Drop that SRIOV check here. We don't want specialized code path for SRIOV.

Christian.

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
   		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
   		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 2e0a1f3e-4e8a-b9ae-bfe6-46b4b74144ce-5C7GfCeVMHo@public.gmane.org

#8 lore raw
Quoted 7 lines

-----Original Message-----
From: Koenig, Christian
Sent: Wednesday, August 15, 2018 3:08 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH v2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 7 lines
Quoted 1 line

As need to let somebody to test all your bare metal test cases, such as vega10,

and other asics, as I don't have those environments, and only could test sriov. That is not much of a problem. We have plenty of people on the mailing list (including me) who can test this on both Vega10 and Raven.

Just remove the SRIOV check and send the patch out with a request for testing it on bare metal.

Ok, I am testing it on SRIOV, and then sent it to review.

Quoted 131 lines

Christian.

Am 15.08.2018 um 04:28 schrieb Deng, Emily:

Quoted 1 line

Thanks for your review, will send a patch to review again as your suggestion. I

think it will be better to use the amdgpu_sriov_vf(adev).

Quoted 1 line

As need to let somebody to test all your bare metal test cases, such as vega10,

and other asics, as I don't have those environments, and only could test sriov.

Quoted 1 line

If passed those bare metal test cases and fixed the bugs if occurs, then could

remove the amdgpu_sriov_vf(adev). Do you think so?

Quoted 27 lines

Best wishes
Emily Deng

Quoted 24 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 8:46 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH v2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 12:56 schrieb Emily Deng:

Quoted 11 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

v2:
Add firmware version checking.

SWDEV-161497

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 63

++++++++++++++++++++++++++++++++

Quoted 4 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 +
   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  7 ++++
   3 files changed, 72 insertions(+)
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..436030c 100644
@@ -233,6 +233,69 @@ void amdgpu_virt_kiq_wreg(struct

amdgpu_device

Quoted 79 lines
Quoted 79 lines

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
   	pr_err("failed to write reg:%x\n", reg);
   }
 
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device
+*adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 238

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
+		unsigned eng, u32 req, uint32_t vmid)

Drop that function here. It is Vega specific and doesn't belong into common code.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8 -0
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+	struct drm_amdgpu_info_firmware fw_info;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 256

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	if (ring->me == 1) {
+		fw_info.ver = adev->gfx.mec_fw_version;
+		fw_info.feature = adev->gfx.mec_feature_version;
+	} else {
+		fw_info.ver = adev->gfx.mec2_fw_version;
+		fw_info.feature = adev->gfx.mec2_feature_version;
+	}
+
+	if (fw_info.ver <  0x00000190 || fw_info.feature < 42)
+		return -EPERM;

Please move that check into gfx_v9_0_ring_emit_reg_write_reg_wait().

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 258

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);

That check is superfluous.

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 262

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 287

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return 0;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+	return r;
+}
+
   /**
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
    * amdgpu_virt_request_full_gpu() - request full gpu access
    * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..8908ff9 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct

amdgpu_device

Quoted 46 lines
Quoted 3 lines
Quoted 3 lines

*adev);

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t

reg);

Quoted 42 lines
Quoted 42 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct · line 293

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t
 reg, uint32_t v);
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device
+*adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
   int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git

a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 6999042..812f71e 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -331,6 +331,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
   	/* Use register 17 for GART */
   	const unsigned eng = 17;
   	unsigned i, j;
+	int r;
 
   	spin_lock(&adev->gmc.invalidate_lock);
 
@@ -338,6 +339,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
   		struct amdgpu_vmhub *hub = &adev->vmhub[i];
   		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			r = amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 347

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +4 -0
+			if (!r)
+				continue;
+		}
+

Drop that SRIOV check here. We don't want specialized code path for SRIOV.

Christian.

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
   		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
   		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#9 lore raw
In reply to a message not in the archive

Remove the sriov check, The patch can work on both vega10 and Raven under bare metal with tip firmware.

Best Regards

Rex

________________________________
From: amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> on behalf of Christian König <[email protected]>
Sent: Wednesday, August 15, 2018 3:07 PM
To: Deng, Emily; [email protected]
Subject: Re: [PATCH v2] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 1 line

As need to let somebody to test all your bare metal test cases, such as vega10, and other asics, as I don't have those environments, and only could test sriov.

That is not much of a problem. We have plenty of people on the mailing list (including me) who can test this on both Vega10 and Raven.

Just remove the SRIOV check and send the patch out with a request for testing it on bare metal.

Christian.

Am 15.08.2018 um 04:28 schrieb Deng, Emily:

Quoted 4 lines

Thanks for your review, will send a patch to review again as your suggestion. I think it will be better to use the amdgpu_sriov_vf(adev). As need to let somebody to test all your bare metal test cases, such as vega10, and other asics, as I don't have those environments, and only could test sriov. If passed those bare metal test cases and fixed the bugs if occurs, then could remove the amdgpu_sriov_vf(adev). Do you think so?

Quoted 162 lines

Best wishes
Emily Deng

Quoted 160 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 8:46 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH v2] drm/amdgpu/sriov: For sriov runtime, use kiq to do
invalidate tlb

Am 14.08.2018 um 12:56 schrieb Emily Deng:

Quoted 11 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

v2:
Add firmware version checking.

SWDEV-161497

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 63

++++++++++++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device

Quoted 10 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 +
   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  7 ++++
   3 files changed, 72 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..436030c 100644
@@ -233,6 +233,69 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 236

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
     pr_err("failed to write reg:%x\n", reg);
   }
 
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 237

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
+           unsigned eng, u32 req, uint32_t vmid)

Drop that function here. It is Vega specific and doesn't belong into common code.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8 -0
+{
+   signed long r, cnt = 0;
+   unsigned long flags;
+   uint32_t seq;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+   struct drm_amdgpu_info_firmware fw_info;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 255

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+   if (ring->me == 1) {
+           fw_info.ver = adev->gfx.mec_fw_version;
+           fw_info.feature = adev->gfx.mec_feature_version;
+   } else {
+           fw_info.ver = adev->gfx.mec2_fw_version;
+           fw_info.feature = adev->gfx.mec2_feature_version;
+   }
+
+   if (fw_info.ver <  0x00000190 || fw_info.feature < 42)
+           return -EPERM;

Please move that check into gfx_v9_0_ring_emit_reg_write_reg_wait().

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 257

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+
+   BUG_ON(!ring->funcs->emit_reg_write_reg_wait);

That check is superfluous.

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 261

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+
+   spin_lock_irqsave(&kiq->ring_lock, flags);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+                                       hub->vm_inv_eng0_ack + eng,
+                                       req, 1 << vmid);
+   amdgpu_fence_emit_polling(ring, &seq);
+   amdgpu_ring_commit(ring);
+   spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+   r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+   /* don't wait anymore for gpu reset case because this way may
+    * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+    * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+    * never return if we keep waiting in virt_kiq_rreg, which cause
+    * gpu_recover() hang there.
+    *
+    * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device · line 286

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+    * */
+   if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+           goto failed_kiq;
+
+   if (in_interrupt())
+           might_sleep();
+
+   while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+           msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+           r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device

Quoted 21 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+   }
+
+   if (cnt > MAX_KIQ_REG_TRY)
+           goto failed_kiq;
+
+   return 0;
+
+failed_kiq:
+   pr_err("failed to invalidate tlb with kiq\n");
+   return r;
+}
+
   /**
    * amdgpu_virt_request_full_gpu() - request full gpu access
    * @amdgpu:      amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..8908ff9 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device

*adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device · line 292

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
 uint32_t v);
+signed long  amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,

struct amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+           unsigned eng, u32 req, uint32_t vmid);
   int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git

a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 6999042..812f71e 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -331,6 +331,7 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
     /* Use register 17 for GART */
     const unsigned eng = 17;
     unsigned i, j;
+   int r;
 
     spin_lock(&adev->gmc.invalidate_lock);
 
@@ -338,6 +339,12 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
             struct amdgpu_vmhub *hub = &adev->vmhub[i];
             u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+           if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+                   r = amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 347

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +4 -0
+                   if (!r)
+                           continue;
+           }
+

Drop that SRIOV check here. We don't want specialized code path for SRIOV.

Christian.

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
             WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
             /* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx Info Page - freedesktop.org<https://lists.freedesktop.org/mailman/listinfo/amd-gfx>
lists.freedesktop.org
Subscribing to amd-gfx: Subscribe to amd-gfx by filling out the following form. Use of all freedesktop.org lists is subject to our Code of Conduct.

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx