Skip to contentSkip to conversation

drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

PATCH Superseded Emily Deng 3 participants 9 messages Aug 14, 2018 8 messages not in the archive

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Emily Deng

author
#0 lore raw

PATCH 1/1 drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

SWDEV-161497

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50 ++++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
 3 files changed, 57 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +50 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 	pr_err("failed to write reg:%x\n", reg);
 }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid)
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+}
+
 /**
  * amdgpu_virt_request_full_gpu() - request full gpu access
  * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device *adev);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid);
 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +5 -0
index ed467de..6a886d9 100644
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
 		struct amdgpu_vmhub *hub = &adev->vmhub[i];
 		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
+			continue;
+		}
+
 		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
 		/* Busy wait for ACK.*/
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1534232783-30577-1-git-send-email-Emily.Deng-5C7GfCeVMHo@public.gmane.org

#2 lore raw

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

Instead just lower the timeout and suppress the warning when SRIOV is active.

Christian.

Quoted 90 lines

SWDEV-161497

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50 ++++++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
  drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
  3 files changed, 57 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +50 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
  	pr_err("failed to write reg:%x\n", reg);
  }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid)
+{
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n");
+}
+
  /**
   * amdgpu_virt_request_full_gpu() - request full gpu access
   * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device *adev);
  void amdgpu_virt_init_setting(struct amdgpu_device *adev);
  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct amdgpu_vmhub *hub,
+		unsigned eng, u32 req, uint32_t vmid);
  int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
  int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
  int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
index ed467de..6a886d9 100644
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
  		struct amdgpu_vmhub *hub = &adev->vmhub[i];

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 349

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +5 -0
  		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp, vmid);
+			continue;
+		}
+
  		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
  		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 7e81733b-de41-6369-fd8c-5f977ced1ba2-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#4 lore raw
Quoted 7 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 6 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and wait ack, and won't be interrupted by world switch.

Quoted 1 line

Instead just lower the timeout and suppress the warning when SRIOV is active.

With the kiq to do tlb flush, no warning issue expected.

Best wishes
Emily Deng

Quoted 107 lines

Christian.

Quoted 6 lines

SWDEV-161497

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device

Quoted 10 lines
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
  drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
  3 files changed, 57 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
  	pr_err("failed to write reg:%x\n", reg);
  }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device

Quoted 19 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
  /**
   * amdgpu_virt_request_full_gpu() - request full gpu access
   * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device

*adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device · line 292

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
  void amdgpu_virt_init_setting(struct amdgpu_device *adev);
  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
 uint32_t v);
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
  int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
  int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);

int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git
a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
  		struct amdgpu_vmhub *hub = &adev->vmhub[i];
  		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 349

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
  		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
  		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB112564B2829F24E6106D099C8F380-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#6 lore raw

Am 14.08.2018 um 10:13 schrieb Deng, Emily:

Quoted 7 lines
Quoted 7 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 7 lines
Quoted 5 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and wait ack, and won't be interrupted by world switch.

That is effectively duplicating the TLB handling.

Quoted 2 lines
Quoted 1 line

Instead just lower the timeout and suppress the warning when SRIOV is active.

With the kiq to do tlb flush, no warning issue expected.

Either fix that issue thoughtfully or let us live with the warning message.

But please no more SRIOV workaround we can't test on bare metal and break things with older firmware.

Regards,
Christian.

Quoted 114 lines

Best wishes
Emily Deng

Quoted 111 lines

Christian.

Quoted 5 lines

SWDEV-161497

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device

Quoted 10 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
   3 files changed, 57 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
   	pr_err("failed to write reg:%x\n", reg);
   }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device

Quoted 19 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
   /**
    * amdgpu_virt_request_full_gpu() - request full gpu access
    * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct amdgpu_device

*adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct amdgpu_device · line 292

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
 uint32_t v);
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev, struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
   int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git

a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
   		struct amdgpu_vmhub *hub = &adev->vmhub[i];
   		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
   		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
Quoted 5 lines
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, · line 349

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
   		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 5adb285f-4f29-f5cd-6b61-455d380b41c0-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#8 lore raw
Quoted 25 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 4:20 PM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:13 schrieb Deng, Emily:

Quoted 7 lines
Quoted 7 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 7 lines
Quoted 5 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and

wait ack, and won't be interrupted by world switch.

Quoted 8 lines

That is effectively duplicating the TLB handling.

Quoted 1 line
Quoted 1 line

Instead just lower the timeout and suppress the warning when SRIOV is

active.

Quoted 1 line

With the kiq to do tlb flush, no warning issue expected.

Either fix that issue thoughtfully or let us live with the warning message.

Sorry, I don't know your mean, with one command, it will have no any warning. It will wait the ack in the same command.

Quoted 2 lines

But please no more SRIOV workaround we can't test on bare metal and break things with older firmware.

It seems a workaround here, but it truly is not a workaround. I think it is doing the right thing. As the new firmware support the one command, why we need use two commands to do the invalidate and wait ack, and we don't know when it will sent the ack back?

Quoted 92 lines

Regards,
Christian.

Quoted 15 lines

Best wishes
Emily Deng

Quoted 12 lines

Christian.

Quoted 5 lines

SWDEV-161497

---
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

Quoted 4 lines
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
   3 files changed, 57 insertions(+)
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct

amdgpu_device

Quoted 57 lines
Quoted 57 lines

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
   	pr_err("failed to write reg:%x\n", reg);
   }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 249

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 274

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
   /**
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
    * amdgpu_virt_request_full_gpu() - request full gpu access
    * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct

amdgpu_device

Quoted 36 lines
Quoted 3 lines
Quoted 3 lines

*adev);

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
   void amdgpu_virt_init_setting(struct amdgpu_device *adev);
   uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t

reg);

Quoted 32 lines
Quoted 22 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct · line 293

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
   void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t
 reg, uint32_t v);
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
   int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
   int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git

a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
   		struct amdgpu_vmhub *hub = &adev->vmhub[i];
   		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 349

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
   		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
   		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB11250D672958E386D94E1E7C8F380-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#10 lore raw

Am 14.08.2018 um 10:26 schrieb Deng, Emily:

Quoted 32 lines
Quoted 23 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 4:20 PM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:13 schrieb Deng, Emily:

Quoted 7 lines
Quoted 7 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 7 lines
Quoted 5 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and

Quoted 9 lines

wait ack, and won't be interrupted by world switch.

That is effectively duplicating the TLB handling.

Quoted 1 line
Quoted 1 line

Instead just lower the timeout and suppress the warning when SRIOV is

active.

Quoted 1 line

With the kiq to do tlb flush, no warning issue expected.

Either fix that issue thoughtfully or let us live with the warning message.

Quoted 7 lines

Sorry, I don't know your mean, with one command, it will have no any warning. It will wait the ack in the same command.

Quoted 2 lines

But please no more SRIOV workaround we can't test on bare metal and break things with older firmware.

It seems a workaround here, but it truly is not a workaround. I think it is doing the right thing. As the new firmware support the one command, why we need use two commands to do the invalidate and wait ack, and we don't know when it will sent the ack back?

The problem is that you depend on new firmware here.

See firmware and driver are distributed separately and we got into quite a bunch of problems because the ring_emit_reg_write_reg_wait() isn't correctly implemented.

See gfx_v9_0_ring_emit_reg_write_reg_wait as well:

Quoted 6 lines

static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask) {
        int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);

Quoted 7 lines

        if (amdgpu_sriov_vf(ring->adev)) gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, ref, mask, 0x20);
        else
                amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, ref, mask); }

Before we add any more broken workarounds like that please fix the existing code to check for the for the firmware version and NOT if SRIOV is present or not.

Regards,
Christian.

Quoted 121 lines
Quoted 121 lines

Regards,
Christian.

Quoted 14 lines

Best wishes
Emily Deng

Quoted 12 lines

Christian.

Quoted 5 lines

SWDEV-161497

---
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

Quoted 4 lines
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
    drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
    3 files changed, 57 insertions(+)
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct

amdgpu_device

Quoted 57 lines
Quoted 57 lines

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
    	pr_err("failed to write reg:%x\n", reg);
    }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 249

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 274

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
    /**
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
     * amdgpu_virt_request_full_gpu() - request full gpu access
     * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct

amdgpu_device

Quoted 3 lines
Quoted 3 lines

*adev);

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
    void amdgpu_virt_init_setting(struct amdgpu_device *adev);
    uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t

reg);

Quoted 27 lines
Quoted 27 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct · line 293

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
    void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t
 reg, uint32_t v);
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
    int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
    int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
    int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff --git

a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
    		struct amdgpu_vmhub *hub = &adev->vmhub[i];
    		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
    		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 349

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
    		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): ec897ca0-d821-2c7a-2460-39eedb4b0894-5C7GfCeVMHo@public.gmane.org

#12 lore raw
Quoted 49 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Tuesday, August 14, 2018 4:32 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:26 schrieb Deng, Emily:

Quoted 31 lines
Quoted 31 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 4:20 PM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:13 schrieb Deng, Emily:

Quoted 7 lines
Quoted 7 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 8 lines
Quoted 5 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and

wait ack, and won't be interrupted by world switch.

That is effectively duplicating the TLB handling.

Quoted 2 lines
Quoted 2 lines

Instead just lower the timeout and suppress the warning when SRIOV is

active.

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

With the kiq to do tlb flush, no warning issue expected.

Either fix that issue thoughtfully or let us live with the warning message.

Sorry, I don't know your mean, with one command, it will have no any warning. It will wait the ack in the same command.

Quoted 2 lines

But please no more SRIOV workaround we can't test on bare metal and break things with older firmware.

It seems a workaround here, but it truly is not a workaround. I think it is doing

the right thing.

Quoted 8 lines
Quoted 2 lines

As the new firmware support the one command, why we need use two commands to do the invalidate and wait ack, and we don't know when it will

sent the ack back?

The problem is that you depend on new firmware here.
See firmware and driver are distributed separately and we got into quite a bunch of problems because the ring_emit_reg_write_reg_wait() isn't correctly implemented.

So it uses sriov to distinguish this, as sriov always use the tip firmware. Do you think need to add the firmware version check here? And also add to the below code?
Best wishes
Emily Deng

Quoted 155 lines

See gfx_v9_0_ring_emit_reg_write_reg_wait as well:

Quoted 8 lines

static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
                                                  uint32_t reg0, uint32_t reg1,
                                                  uint32_t ref, uint32_t mask) {
        int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);

Quoted 8 lines

        if (amdgpu_sriov_vf(ring->adev)) gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, ref, mask, 0x20); else amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
                                                           ref, mask); }

Before we add any more broken workarounds like that please fix the existing code to check for the for the firmware version and NOT if SRIOV is present or not.

Regards,
Christian.

Quoted 94 lines
Quoted 94 lines

Regards,
Christian.

Quoted 14 lines

Best wishes
Emily Deng

Quoted 12 lines

Christian.

Quoted 5 lines

SWDEV-161497

---
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

Quoted 4 lines
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
    drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
    3 files changed, 57 insertions(+)
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct

amdgpu_device

Quoted 57 lines
Quoted 57 lines

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
    	pr_err("failed to write reg:%x\n", reg);
    }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 249

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 274

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
    /**
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
     * amdgpu_virt_request_full_gpu() - request full gpu access
     * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct

amdgpu_device

Quoted 4 lines
Quoted 4 lines

*adev);

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
    void amdgpu_virt_init_setting(struct amdgpu_device *adev);
    uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev,

uint32_t

Quoted 8 lines
Quoted 8 lines

reg);

Quoted 7 lines
Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h

Quoted 4 lines

void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t
reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
    int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool

init);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool

init);

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 7 lines

int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff
--git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
    		struct amdgpu_vmhub *hub = &adev->vmhub[i];
    		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
    		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 349

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
    		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB11250B1E30DBF68E649B351E8F380-rpdhrqHFk07v2MZdTKcfDgdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#14 lore raw

Am 14.08.2018 um 10:35 schrieb Deng, Emily:

Quoted 51 lines
Quoted 51 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Tuesday, August 14, 2018 4:32 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:26 schrieb Deng, Emily:

Quoted 31 lines
Quoted 31 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 4:20 PM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:13 schrieb Deng, Emily:

Quoted 7 lines
Quoted 7 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 8 lines
Quoted 5 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and

wait ack, and won't be interrupted by world switch.

That is effectively duplicating the TLB handling.

Quoted 2 lines
Quoted 2 lines

Instead just lower the timeout and suppress the warning when SRIOV is

active.

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

With the kiq to do tlb flush, no warning issue expected.

Either fix that issue thoughtfully or let us live with the warning message.

Sorry, I don't know your mean, with one command, it will have no any warning. It will wait the ack in the same command.

Quoted 2 lines

But please no more SRIOV workaround we can't test on bare metal and break things with older firmware.

It seems a workaround here, but it truly is not a workaround. I think it is doing

the right thing.

Quoted 2 lines

As the new firmware support the one command, why we need use two commands to do the invalidate and wait ack, and we don't know when it will

Quoted 7 lines
Quoted 6 lines

sent the ack back?

The problem is that you depend on new firmware here.
See firmware and driver are distributed separately and we got into quite a bunch of problems because the ring_emit_reg_write_reg_wait() isn't correctly implemented.

So it uses sriov to distinguish this, as sriov always use the tip firmware.

That assumption is not correct. You SRIOV users should use the tip firmware, but there is no guarantee that they are actually doing it.

Quoted 1 line

Do you think need to add the firmware version check here?

Yes, please do so.

Quoted 1 line

And also add to the below code?

No, probably best approach would be to use the same code path for both bare metal and SRIOV.

Regards,
Christian.

Quoted 160 lines

Best wishes
Emily Deng

Quoted 157 lines

See gfx_v9_0_ring_emit_reg_write_reg_wait as well:

Quoted 8 lines

static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
                                                  uint32_t reg0,
uint32_t reg1,
                                                  uint32_t ref,
uint32_t mask) {
        int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);

Quoted 8 lines

        if (amdgpu_sriov_vf(ring->adev)) gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, ref, mask, 0x20); else amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0,
reg1,
                                                           ref, mask);
}

Before we add any more broken workarounds like that please fix the existing code to check for the for the firmware version and NOT if SRIOV is present or not.

Regards,
Christian.

Quoted 94 lines
Quoted 94 lines

Regards,
Christian.

Quoted 14 lines

Best wishes
Emily Deng

Quoted 12 lines

Christian.

Quoted 5 lines

SWDEV-161497

---
     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

Quoted 4 lines
     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
     drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
     3 files changed, 57 insertions(+)
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct

amdgpu_device

Quoted 57 lines
Quoted 57 lines

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
     	pr_err("failed to write reg:%x\n", reg);
     }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 249

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +12 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +

eng,

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 274

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
     /**
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
      * amdgpu_virt_request_full_gpu() - request full gpu access
      * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct

amdgpu_device

Quoted 4 lines
Quoted 4 lines

*adev);

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
     void amdgpu_virt_init_setting(struct amdgpu_device *adev);
     uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev,

uint32_t

Quoted 8 lines
Quoted 8 lines

reg);

Quoted 7 lines
Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h

Quoted 4 lines

void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t
reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
     int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool

init);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool

init);

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 7 lines

int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff
--git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -0
     		struct amdgpu_vmhub *hub = &adev->vmhub[i];
     		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) && amdgpu_sriov_runtime(adev)) {
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng, tmp,

vmid);

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
     		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 349

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
     		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): aa872eb7-6586-4c87-8454-6c1993e826c4-5C7GfCeVMHo@public.gmane.org

#16 lore raw
Quoted 68 lines

-----Original Message-----
From: Koenig, Christian
Sent: Tuesday, August 14, 2018 4:38 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:35 schrieb Deng, Emily:

Quoted 60 lines
Quoted 60 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Christian König
Sent: Tuesday, August 14, 2018 4:32 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:26 schrieb Deng, Emily:

Quoted 33 lines
Quoted 33 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 4:20 PM
To: Deng, Emily <[email protected]>; Koenig, Christian <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Am 14.08.2018 um 10:13 schrieb Deng, Emily:

Quoted 8 lines
Quoted 8 lines

-----Original Message-----
From: Christian König <[email protected]>
Sent: Tuesday, August 14, 2018 3:54 PM
To: Deng, Emily <[email protected]>; [email protected]
Subject: Re: [PATCH] drm/amdgpu/sriov: For sriov runtime, use kiq to do invalidate tlb

Quoted 8 lines
Quoted 5 lines

Am 14.08.2018 um 09:46 schrieb Emily Deng:

Quoted 2 lines

To avoid the tlb flush not interrupted by world switch, use kiq and one command to do tlb invalidate.

Well NAK, this just duplicates the TLB handling and moves it outside of the GMC code.

No, it not duplicates the TLB handling, it only send one command, and not duplicate. With kiq, it only use one command to do the invalidate tlb and

wait ack, and won't be interrupted by world switch.

That is effectively duplicating the TLB handling.

Quoted 2 lines
Quoted 2 lines

Instead just lower the timeout and suppress the warning when SRIOV is

active.

Quoted 1 line

With the kiq to do tlb flush, no warning issue expected.

Quoted 8 lines
Quoted 2 lines

Either fix that issue thoughtfully or let us live with the warning message.

Sorry, I don't know your mean, with one command, it will have no any warning. It will wait the ack in the same command.

Quoted 2 lines

But please no more SRIOV workaround we can't test on bare metal and break things with older firmware.

It seems a workaround here, but it truly is not a workaround. I think it is doing

the right thing.

Quoted 3 lines

As the new firmware support the one command, why we need use two commands to do the invalidate and wait ack, and we don't know when it will

sent the ack back?

The problem is that you depend on new firmware here.
See firmware and driver are distributed separately and we got into quite a bunch of problems because the ring_emit_reg_write_reg_wait() isn't correctly implemented.

Quoted 8 lines
Quoted 1 line

So it uses sriov to distinguish this, as sriov always use the tip firmware.

That assumption is not correct. You SRIOV users should use the tip firmware, but there is no guarantee that they are actually doing it.

Quoted 1 line

Do you think need to add the firmware version check here?

Yes, please do so.

Ok, I will send a v2 patch

Quoted 4 lines
Quoted 1 line

And also add to the below code?

No, probably best approach would be to use the same code path for both bare metal and SRIOV.

But the bare metal doesn't have world switch, the SRIOV has, and don't know when it will be switched back to this VF. So I think for SRIOV, it will be better to use one command. And it have another issue that I think you already know what I am mentioning.

Quoted 151 lines

Regards,
Christian.

Quoted 73 lines

Best wishes
Emily Deng

Quoted 56 lines

See gfx_v9_0_ring_emit_reg_write_reg_wait as well:

Quoted 8 lines

static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
                                                  uint32_t reg0,
uint32_t reg1,
                                                  uint32_t ref,
uint32_t mask) {
        int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);

Quoted 9 lines

        if (amdgpu_sriov_vf(ring->adev)) gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0,
reg1,
                                      ref, mask, 0x20); else amdgpu_ring_emit_reg_write_reg_wait_helper(ring,
reg0, reg1,
                                                           ref,
mask); }

Before we add any more broken workarounds like that please fix the existing code to check for the for the firmware version and NOT if SRIOV is present or not.

Regards,
Christian.

Quoted 31 lines
Quoted 31 lines

Regards,
Christian.

Quoted 14 lines

Best wishes
Emily Deng

Quoted 12 lines

Christian.

Quoted 5 lines

SWDEV-161497

---
     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 50

++++++++++++++++++++++++++++++++

Quoted 4 lines
     drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  2 ++
     drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    |  5 ++++
     3 files changed, 57 insertions(+)
Quoted 6 lines
Quoted 6 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
index 21adb1b6..aa6ddcc 100644
@@ -233,6 +233,56 @@ void amdgpu_virt_kiq_wreg(struct

amdgpu_device

Quoted 7 lines
Quoted 7 lines

*adev, uint32_t reg, uint32_t v)

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 237

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
     	pr_err("failed to write reg:%x\n", reg);
     }
 
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

Quoted 12 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+		unsigned eng, u32 req, uint32_t vmid) {
+	signed long r, cnt = 0;
+	unsigned long flags;
+	uint32_t seq;
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 249

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +8 -0
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_reg_write_reg_wait);
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub-

vm_inv_eng0_req

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
++
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

eng,

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 257

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +7 -0
+					    hub->vm_inv_eng0_ack + eng,
+					    req, 1 << vmid);
+	amdgpu_fence_emit_polling(ring, &seq);
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 261

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+
+	/* don't wait anymore for gpu reset case because this way may
+	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
+	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue()

will

Quoted 60 lines
Quoted 60 lines
Quoted 40 lines
Quoted 40 lines
Quoted 26 lines
Quoted 26 lines
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
+	 * never return if we keep waiting in virt_kiq_rreg, which cause
+	 * gpu_recover() hang there.
+	 *
+	 * also don't wait anymore for IRQ context

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c void amdgpu_virt_kiq_wreg(struct · line 275

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	 * */
+	if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
+		goto failed_kiq;
+
+	if (in_interrupt())
+		might_sleep();
+
+	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
+		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
+		r = amdgpu_fence_wait_polling(ring, seq,

MAX_KIQ_REG_WAIT);

Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10 -0
+	}
+
+	if (cnt > MAX_KIQ_REG_TRY)
+		goto failed_kiq;
+
+	return;
+
+failed_kiq:
+	pr_err("failed to invalidate tlb with kiq\n"); }
+
     /**
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h void amdgpu_free_static_csa(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +0 -0
      * amdgpu_virt_request_full_gpu() - request full gpu access
      * @amdgpu:	amdgpu device.
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 880ac11..a2e3c78 100644
@@ -288,6 +288,8 @@ void amdgpu_free_static_csa(struct

amdgpu_device

Quoted 4 lines
Quoted 4 lines

*adev);

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
     void amdgpu_virt_init_setting(struct amdgpu_device *adev);
     uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev,

uint32_t

reg);

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h

Quoted 4 lines

void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev,
uint32_t reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -0
+void amdgpu_virt_kiq_invalidate_tlb(struct amdgpu_device *adev,
+struct

amdgpu_vmhub *hub,

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
+		unsigned eng, u32 req, uint32_t vmid);
     int amdgpu_virt_request_full_gpu(struct amdgpu_device

*adev, bool

init);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

int amdgpu_virt_release_full_gpu(struct amdgpu_device
*adev, bool

init);

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct

Quoted 7 lines

int amdgpu_virt_reset_gpu(struct amdgpu_device *adev); diff
--git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index ed467de..6a886d9 100644

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0 -0
@@ -339,6 +339,11 @@ static void gmc_v9_0_flush_gpu_tlb(struct

amdgpu_device *adev,

Quoted 22 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 342

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
     		struct amdgpu_vmhub *hub = &adev->vmhub[i];
     		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
 
+		if (amdgpu_sriov_vf(adev) &&

amdgpu_sriov_runtime(adev)) {

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 343

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
+			amdgpu_virt_kiq_invalidate_tlb(adev, hub, eng,

tmp,

Quoted 15 lines
Quoted 15 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 7 lines

vmid);

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_flush_gpu_tlb(struct · line 349

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +3 -0
+			continue;
+		}
+
     		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
 
     		/* Busy wait for ACK.*/

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx