Skip to contentSkip to conversation

Add support AMD GPU virtualization soultion

PATCH In review Xiangliang Yu 4 participants 38 messages Jan 11, 2017 – Jan 12, 2017 24 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v3 cover Add support AMD GPU virtualization soultion

This patch series will enable AMD GPU virtualization and wires it into amdgpu modules.

Changes in V3:
1. rename field name;
2. remove extra fence_get()/fence_put();
3. fixed coding style;
4. update patch comment;

Changes in V2:
1. define new high level interface for non-asic specific;
2. add asic specific file to support specific asic;
3. not include KIQ patch as it has been merged into kernel;
4. not include CSA patch as monk will submit it;

Xiangliang Yu (11):
drm/amdgpu/ring: add two interfaces to support r/w registers with kiq drm/amdgpu/gfx8: implement emit_rreg/wreg function drm/amdgpu/virt: add runtime flag
drm/amdgpu/virt: use kiq to access registers drm/amdgpu/virt: add high level interfaces for virt drm/amdgpu/virt: implement VI virt operation interfaces drm/amdgpu: request/release full gpu access if device is vf drm/amdgpu/vi: add support virtualization
drm/amdgpu/virt: enable virtual display
drm/amdgpu/virt: disable dc
drm/amdgpu: do not reset gpu for virtualization

 drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592 +++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
 drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
 11 files changed, 926 insertions(+), 4 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1484140698-10548-1-git-send-email-Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org

#2 lore raw

PATCH v3 1/11 drm/amdgpu/ring: add two interfaces to support r/w registers with kiq

During virtual runtime, need to send command to kiq ring to read/write GPU registers. Add two interface to support the two actions.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h      | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h | 2 ++
 2 files changed, 4 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -0
index 13c30ab..13d87d7 100644
@@ -1646,6 +1646,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
 #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
 #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
+#define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
+#define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +2 -0
index 92bc89b..c813cbe 100644
@@ -135,6 +135,8 @@ struct amdgpu_ring_funcs {
 	void (*end_use)(struct amdgpu_ring *ring);
 	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
 	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
+	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 };
 
 struct amdgpu_ring {
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#3 lore raw

PATCH v3 2/11 drm/amdgpu/gfx8: implement emit_rreg/wreg function

In reply to a message not in the archive

Implement emit_rreg/wreg function for kiq ring.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |  3 ++-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2 -1
index 0d821d9..003bffb 100644
@@ -30,7 +30,8 @@
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t caps;
+	uint32_t		caps;
+	uint32_t		reg_val_offs;
 };
 
 #define amdgpu_sriov_enabled(adev) \
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +37 -0
index b487b97..a18b392 100644
@@ -1373,6 +1373,12 @@ static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
 {
 	int r = 0;
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
+		if (r)
+			return r;
+	}
+
 	ring->adev = NULL;
 	ring->ring_obj = NULL;
 	ring->use_doorbell = true;
@@ -1399,6 +1405,9 @@ static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
 static void gfx_v8_0_kiq_free_ring(struct amdgpu_ring *ring,
 				   struct amdgpu_irq_src *irq)
 {
+	if (amdgpu_sriov_vf(ring->adev))
+		amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
+
 	amdgpu_ring_fini(ring);
 	irq->data = NULL;
 }
@@ -6681,6 +6690,32 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 	amdgpu_ring_write(ring, 0);
 }
 
+static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
+	amdgpu_ring_write(ring, 0 |	/* src: register*/
+				(5 << 8) |	/* dst: memory */
+				(1 << 20));	/* write confirm */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
+				adev->virt.reg_val_offs * 4));
+	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
+				adev->virt.reg_val_offs * 4));
+}
+
+static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
+				  uint32_t val)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, val);
+}
+
 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
 						 enum amdgpu_interrupt_state state)
 {
@@ -6995,6 +7030,8 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
 	.test_ib = gfx_v8_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_rreg = gfx_v8_0_ring_emit_rreg,
+	.emit_wreg = gfx_v8_0_ring_emit_wreg,
 };
 
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#4 lore raw

PATCH v3 3/11 drm/amdgpu/virt: add runtime flag

In reply to a message not in the archive

Add new flag to define gpu runtime that is out of full gpu access.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 4 ++++
 1 file changed, 4 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +4 -0
index 003bffb..3d38a9f 100644
@@ -28,6 +28,7 @@
 #define AMDGPU_SRIOV_CAPS_ENABLE_IOV   (1 << 1) /* sr-iov is enabled on this GPU */
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
+#define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
@@ -43,6 +44,9 @@ struct amdgpu_virt {
 #define amdgpu_sriov_bios(adev) \
 ((adev)->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
 
+#define amdgpu_sriov_runtime(adev) \
+((adev)->virt.caps & AMDGPU_SRIOV_CAPS_RUNTIME)
+
 #define amdgpu_passthrough(adev) \
 ((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE)
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#5 lore raw

PATCH v3 4/11 drm/amdgpu/virt: use kiq to access registers

In reply to a message not in the archive

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
 drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
 drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
 5 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -0
index f82919d..9a2fd3e 100644
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 {
 	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 {
 	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +82 -0

82 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
+	struct mutex		lock;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -59,4 +60,8 @@ static inline bool is_virtual_machine(void)
 #endif
 }
 
+void amdgpu_virt_init_setting(struct amdgpu_device *adev);
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+
 #endif
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
index 7350a8f..dc0d4fa 100644
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
 		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
 	adev->rev_id = vi_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
 	switch (adev->asic_type) {
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1484140698-10548-5-git-send-email-Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org

#7 lore raw

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 32 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
  drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
  drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
  5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
  	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
  	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
  	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
  # add asic specific block
  amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  {
  	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
  	if ((reg * 4) < adev->rmmio_size && !always_indirect)
  		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
  	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
  {
  	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 117 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
  	if ((reg * 4) < adev->rmmio_size && !always_indirect)
  		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
  	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +82 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev)
+{
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
+{
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
+{
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
  struct amdgpu_virt {
  	uint32_t		caps;
  	uint32_t		reg_val_offs;
+	struct mutex		lock;
  };
 
  #define amdgpu_sriov_enabled(adev) \
@@ -59,4 +60,8 @@ static inline bool is_virtual_machine(void)
  #endif
  }
 
+void amdgpu_virt_init_setting(struct amdgpu_device *adev);
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+
  #endif
drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
  		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
  		smc_enabled = true;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
  	adev->rev_id = vi_get_rev_id(adev);
  	adev->external_rev_id = 0xFF;
  	switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): c6c7dbf9-03db-2528-78cd-b1390dcab10d-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#9 lore raw

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
  drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
  drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
  5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
  	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
  	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
  	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
  # add asic specific block
  amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \ 
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  {
  	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
  	if ((reg * 4) < adev->rmmio_size && !always_indirect)
  		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
  	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
  {
  	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 131 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
  	if ((reg * 4) < adev->rmmio_size && !always_indirect)
  		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
  	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT 
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t 
+reg) {
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
+uint32_t v) {
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
  struct amdgpu_virt {
  	uint32_t		caps;
  	uint32_t		reg_val_offs;
+	struct mutex		lock;
  };
 
  #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline 

bool is_virtual_machine(void)
#endif
}

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void 
+amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
+uint32_t v);
+
  #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
  		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
  		smc_enabled = true;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
  	adev->rev_id = vi_get_rev_id(adev);
  	adev->external_rev_id = 0xFF;
  	switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB11104C83B83FB2CEA6B26A2884660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#11 lore raw

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 206 lines

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
   5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
   # add asic specific block
   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
   {
   	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
   	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
   {
   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 134 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
   	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v) {
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
   struct amdgpu_virt {
   	uint32_t		caps;
   	uint32_t		reg_val_offs;
+	struct mutex		lock;
   };
 
   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static inline

bool is_virtual_machine(void)

   #endif
   }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
+amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v);
+
   #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
   		smc_enabled = true;
 
Quoted 11 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
   	adev->rev_id = vi_get_rev_id(adev);
   	adev->external_rev_id = 0xFF;
   	switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): d4e006ef-a0a2-bbc3-3582-d259d1e6a75b-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#13 lore raw

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Yeah, make sense

I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 212 lines

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
   drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
   drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
   drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
   5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
   	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
   	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
   	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
   # add asic specific block
   amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o 

\ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
   {
   	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
   		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
   	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
   {
   	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 134 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
   	if ((reg * 4) < adev->rmmio_size && !always_indirect)
   		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
   	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO 
+EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
+USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
+uint32_t v) {
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
   struct amdgpu_virt {
   	uint32_t		caps;
   	uint32_t		reg_val_offs;
+	struct mutex		lock;
   };
 
   #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static 

inline bool is_virtual_machine(void)

   #endif
   }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void 
+amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
+uint32_t v);
+
   #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
   		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
   		smc_enabled = true;
 
Quoted 11 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
   	adev->rev_id = vi_get_rev_id(adev);
   	adev->external_rev_id = 0xFF;
   	switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB11107C560860494991E6D6F884660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#15 lore raw

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

do you think that's okay ?

Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:

Quoted 248 lines

Yeah, make sense

I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 212 lines

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
    5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
    	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
    	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
    	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
    # add asic specific block
    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o

\ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
    {
    	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
    		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
    	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
    {
    	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 134 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
    		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
    	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v) {
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
    struct amdgpu_virt {
    	uint32_t		caps;
    	uint32_t		reg_val_offs;
+	struct mutex		lock;
    };
 
    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static

inline bool is_virtual_machine(void)

    #endif
    }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void
+amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v);
+
    #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
    		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
    		smc_enabled = true;
 
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
    	adev->rev_id = vi_get_rev_id(adev);
    	adev->external_rev_id = 0xFF;
    	switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): f79c252c-369b-235e-2160-231b4ac32235-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#17 lore raw

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

We should BUG and not access register at all if found during RUNTIME && IRQ_context By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

do you think that's okay ?

Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:

Quoted 249 lines

Yeah, make sense

I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 212 lines

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
    5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
    	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
    	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
    	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
    # add asic specific block
    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o 

kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
    {
    	uint32_t ret;
 
+	if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+		return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
    		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
    	else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
    {
    	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (amdgpu_sriov_runtime(adev))
+		return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 134 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
    	if ((reg * 4) < adev->rmmio_size && !always_indirect)
    		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
    	else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to 
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO 
+EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 
+USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+	mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+	signed long r;
+	uint32_t val;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_rreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_rreg(ring, reg);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+
+	val = adev->wb.wb[adev->virt.reg_val_offs];
+
+	return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
+uint32_t v) {
+	signed long r;
+	struct fence *f;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+	struct amdgpu_ring *ring = &kiq->ring;
+
+	BUG_ON(!ring->funcs->emit_wreg);
+
+	mutex_lock(&adev->virt.lock);
+	amdgpu_ring_alloc(ring, 32);
+	amdgpu_ring_emit_hdp_flush(ring);
+	amdgpu_ring_emit_wreg(ring, reg, v);
+	amdgpu_ring_emit_hdp_invalidate(ring);
+	amdgpu_fence_emit(ring, &f);
+	amdgpu_ring_commit(ring);
+	mutex_unlock(&adev->virt.lock);
+
+	r = fence_wait(f, false);
+	if (r)
+		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+	fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
    struct amdgpu_virt {
    	uint32_t		caps;
    	uint32_t		reg_val_offs;
+	struct mutex		lock;
    };
 
    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static 

inline bool is_virtual_machine(void)

    #endif
    }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t 
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); 
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, 
+uint32_t v);
+
    #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
    		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
    		smc_enabled = true;
 
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_init_setting(adev);
+
    	adev->rev_id = vi_get_rev_id(adev);
    	adev->external_rev_id = 0xFF;
    	switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB11106D68693C869D3E7D431884660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#19 lore raw

答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Xiangliang

please BUG() when register access occured in RUNTIME and IRQ context, e.g.:

if (amdgpu_sriov_runtime(adev)) {

}
return amdgpu_virt_kiq_wreg(adev, reg, v);

with above addressed, Reviewed-by: Monk Liu <[email protected]>

________________________________
发件人: amd-gfx <[email protected]> 代表 Liu, Monk <[email protected]> 发送时间: 2017年1月11日 22:54:52
收件人: Christian König; Yu, Xiangliang; [email protected] 主题: RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

We should BUG and not access register at all if found during RUNTIME && IRQ_context By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

do you think that's okay ?

Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:

Quoted 249 lines

Yeah, make sense

I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 212 lines

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
    5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
     atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
     amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
     amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
    # add asic specific block
    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o

kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
    {
     uint32_t ret;
 
+   if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+           return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
     if ((reg * 4) < adev->rmmio_size && !always_indirect)
             ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
     else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
    {
     trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+   if (amdgpu_sriov_runtime(adev))
+           return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 134 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
     if ((reg * 4) < adev->rmmio_size && !always_indirect)
             writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
     else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+   mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+   signed long r;
+   uint32_t val;
+   struct fence *f;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+
+   BUG_ON(!ring->funcs->emit_rreg);
+
+   mutex_lock(&adev->virt.lock);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_hdp_flush(ring);
+   amdgpu_ring_emit_rreg(ring, reg);
+   amdgpu_ring_emit_hdp_invalidate(ring);
+   amdgpu_fence_emit(ring, &f);
+   amdgpu_ring_commit(ring);
+   mutex_unlock(&adev->virt.lock);
+
+   r = fence_wait(f, false);
+   if (r)
+           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+   fence_put(f);
+
+   val = adev->wb.wb[adev->virt.reg_val_offs];
+
+   return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v) {
+   signed long r;
+   struct fence *f;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+
+   BUG_ON(!ring->funcs->emit_wreg);
+
+   mutex_lock(&adev->virt.lock);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_hdp_flush(ring);
+   amdgpu_ring_emit_wreg(ring, reg, v);
+   amdgpu_ring_emit_hdp_invalidate(ring);
+   amdgpu_fence_emit(ring, &f);
+   amdgpu_ring_commit(ring);
+   mutex_unlock(&adev->virt.lock);
+
+   r = fence_wait(f, false);
+   if (r)
+           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+   fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
    struct amdgpu_virt {
     uint32_t                caps;
     uint32_t                reg_val_offs;
+   struct mutex            lock;
    };
 
    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static

inline bool is_virtual_machine(void)

    #endif
    }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v);
+
    #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
             (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
             smc_enabled = true;
 
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+   if (amdgpu_sriov_vf(adev))
+           amdgpu_virt_init_setting(adev);
+
     adev->rev_id = vi_get_rev_id(adev);
     adev->external_rev_id = 0xFF;
     switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB11108F4CBFB28D71407DE56184790-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#21 lore raw

答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Xiangliang

please BUG() when register access occured in RUNTIME and IRQ context, e.g.:

for register read:
if (amdgpu_sriov_runtime(adev)) {
if (in_interrupt())

        BUG();
   else
       return amdgpu_virt_kiq_rreg(adev, reg, v);
}

and also for register write, with above addressed, Reviewed-by: Monk Liu <[email protected]>

________________________________
发件人: Liu, Monk
发送时间: 2017年1月12日 11:19:40
收件人: Christian König; Yu, Xiangliang; [email protected] 主题: 答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Xiangliang

please BUG() when register access occured in RUNTIME and IRQ context, e.g.:

if (amdgpu_sriov_runtime(adev)) {

}
return amdgpu_virt_kiq_wreg(adev, reg, v);

with above addressed, Reviewed-by: Monk Liu <[email protected]>

________________________________
发件人: amd-gfx <[email protected]> 代表 Liu, Monk <[email protected]> 发送时间: 2017年1月11日 22:54:52
收件人: Christian König; Yu, Xiangliang; [email protected] 主题: RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

We should BUG and not access register at all if found during RUNTIME && IRQ_context By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

do you think that's okay ?

Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:

Quoted 249 lines

Yeah, make sense

I think before we have a full version of KIQ reg accessing without context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

That won't work. Locking a mutext like it is done in the write path can cause scheduling as well.

If we need to push writes through the KIQ in interrupt context we need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Otherwise the whole thing can easily deadlock when an interrupt happens to come while the lock is taken.

Please also test the patchset with lockdep enabled, that should yield a bunch of error messages when you try to do something like this.

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 212 lines

Because if we are in interrupt , we are forbid to do schedule, and use kiq to read register will invoke fence_wait() ...

If you think that's odds, we can use BUG_ON() to stop driver running if we need read registers in SRIOV case.

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 34 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

---
    drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
    drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 ++++++++++++++++++++++++++++++
    drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
    drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
    5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
     atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
     amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
     amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
    # add asic specific block
    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o

kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, · line 100

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
    {
     uint32_t ret;
 
+   if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+           return amdgpu_virt_kiq_rreg(adev, reg);
+

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, · line 122

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
     if ((reg * 4) < adev->rmmio_size && !always_indirect)
             ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
     else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
    {
     trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+   if (amdgpu_sriov_runtime(adev))
+           return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a register from an interrupt routine?

Christian.

Quoted 134 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
     if ((reg * 4) < adev->rmmio_size && !always_indirect)
             writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
     else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +93 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+   mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+   signed long r;
+   uint32_t val;
+   struct fence *f;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+
+   BUG_ON(!ring->funcs->emit_rreg);
+
+   mutex_lock(&adev->virt.lock);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_hdp_flush(ring);
+   amdgpu_ring_emit_rreg(ring, reg);
+   amdgpu_ring_emit_hdp_invalidate(ring);
+   amdgpu_fence_emit(ring, &f);
+   amdgpu_ring_commit(ring);
+   mutex_unlock(&adev->virt.lock);
+
+   r = fence_wait(f, false);
+   if (r)
+           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+   fence_put(f);
+
+   val = adev->wb.wb[adev->virt.reg_val_offs];
+
+   return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v) {
+   signed long r;
+   struct fence *f;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+
+   BUG_ON(!ring->funcs->emit_wreg);
+
+   mutex_lock(&adev->virt.lock);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_hdp_flush(ring);
+   amdgpu_ring_emit_wreg(ring, reg, v);
+   amdgpu_ring_emit_hdp_invalidate(ring);
+   amdgpu_fence_emit(ring, &f);
+   amdgpu_ring_commit(ring);
+   mutex_unlock(&adev->virt.lock);
+
+   r = fence_wait(f, false);
+   if (r)
+           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+   fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
    struct amdgpu_virt {
     uint32_t                caps;
     uint32_t                reg_val_offs;
+   struct mutex            lock;
    };
 
    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static

inline bool is_virtual_machine(void)

    #endif
    }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v);
+
    #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
             (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
             smc_enabled = true;
 
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+   if (amdgpu_sriov_vf(adev))
+           amdgpu_virt_init_setting(adev);
+
     adev->rev_id = vi_get_rev_id(adev);
     adev->external_rev_id = 0xFF;
     switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB11102241041CA3C7C37D25D984790-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#23 lore raw

Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 3 lines · Monk
   if (in_interrupt())
        BUG();

Current coding style is

    BUG_ON(in_interrupt());

according to https://kernelnewbies.org/FAQ/BUG .

Signature · 6 lines
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 65f6eb07-f350-bd46-d8c8-df6cb9387efa-otUistvHUpPR7s880joybQ@public.gmane.org

#25 lore raw

RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 10 lines

-----Original Message-----
From: Michel Dänzer [mailto:[email protected]]
Sent: Thursday, January 12, 2017 1:45 PM
To: Liu, Monk <[email protected]>; Christian König <[email protected]>; Yu, Xiangliang <[email protected]>
Cc: [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line · Monk
Quoted 8 lines
Quoted 2 lines
   if (in_interrupt())
        BUG();

Current coding style is

    BUG_ON(in_interrupt());

according to https://kernelnewbies.org/FAQ/BUG .

Got it. Thanks!

Quoted 5 lines

--
Earthling Michel Dänzer | http://www.amd.com
Libre software enthusiast | Mesa and X developer

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#26 lore raw

Re: 答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

In reply to a message not in the archive

I would go a step further and put the BUG_ON(in_interrupt()) into amdgpu_virt_kiq_rreg().

And BTW the problem is even worse, you also can't lock a mutex inside an atomic section and we could have plenty of that as well in the driver code.

Regards,
Christian.

Am 12.01.2017 um 04:21 schrieb Liu, Monk:

Quoted 331 lines

Xiangliang

please BUG() when register access occured in RUNTIME and IRQ context, e.g.:

for register read:

if (amdgpu_sriov_runtime(adev)) {
   if (in_interrupt())
        BUG();
   else

return amdgpu_virt_kiq_rreg(adev, reg, v);
}

and also for register write, with above addressed, Reviewed-by: Monk Liu <[email protected]>

------------------------------------------------------------------------
*发件人:* Liu, Monk
*发送时间:* 2017年1月12日 11:19:40
*收件人:* Christian König; Yu, Xiangliang; [email protected] *主题:* 答复: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Xiangliang

please BUG() when register access occured in RUNTIME and IRQ context, e.g.:

if (amdgpu_sriov_runtime(adev)) {

}
return amdgpu_virt_kiq_wreg(adev, reg, v);

with above addressed, Reviewed-by: Monk Liu <[email protected]>

------------------------------------------------------------------------
*发件人:* amd-gfx <amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org> 代表 Liu, Monk <[email protected]> *发送时间:* 2017年1月11日 22:54:52
*收件人:* Christian König; Yu, Xiangliang; [email protected]
*主题:* RE: [V3 04/11] drm/amdgpu/virt: use kiq to access registers
We should BUG and not access register at all if found during RUNTIME && IRQ_context
By far.

BR Monk
-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:44 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

do you think that's okay ?

Fine with me, but I'm not sure if the hypervisor doesn't gets a hickup when a client writes to some registers from interrupt context.

Christian.

Am 11.01.2017 um 15:38 schrieb Liu, Monk:

Quoted 3 lines

Yeah, make sense

I think before we have a full version of KIQ reg accessing without

context switch, we should totally disable KIQ reg read/write in IRQ context, do you think that's okay ?

Quoted 11 lines

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 10:16 PM
To: Liu, Monk <[email protected]>; Yu, Xiangliang <[email protected]>; [email protected]
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and

use kiq to read register will invoke fence_wait() ...

Quoted 1 line

That won't work. Locking a mutext like it is done in the write path

can cause scheduling as well.

Quoted 2 lines

If we need to push writes through the KIQ in interrupt context we

need to change the code to use a spinlock with disabled interrupts instead of a mutex.

Quoted 2 lines

Otherwise the whole thing can easily deadlock when an interrupt

happens to come while the lock is taken.

Quoted 2 lines

Please also test the patchset with lockdep enabled, that should

yield a bunch of error messages when you try to do something like this.

Quoted 6 lines

Regards,
Christian.

Am 11.01.2017 um 14:51 schrieb Liu, Monk:

Quoted 1 line

Because if we are in interrupt , we are forbid to do schedule, and

use kiq to read register will invoke fence_wait() ...

Quoted 2 lines
Quoted 2 lines

If you think that's odds, we can use BUG_ON() to stop driver

running if we need read registers in SRIOV case.

Quoted 22 lines
Quoted 22 lines

And to fully support register reading, we need to implement a new method to use KIQ access register without any schedule() chance, but that's overhead for current stage, and we don't observe registers reading in IRQ context now on 3D apps. (kfd observed such case, and @Shaoyun dispatches a kernel thread/tasklet in IRQ to do the register dumping, instead of directly reading them via KIQ )

BR Monk

-----Original Message-----
From: Christian König [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:38 PM
To: Yu, Xiangliang <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>
Subject: Re: [V3 04/11] drm/amdgpu/virt: use kiq to access registers

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 3 lines

For virtualization, it is must for driver to use KIQ to access registers when it is out of GPU full access mode.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

---

drivers/gpu/drm/amd/amdgpu/Makefile        |  2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  6 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 82 

++++++++++++++++++++++++++++++

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  5 ++
drivers/gpu/drm/amd/amdgpu/vi.c            |  3 ++
    5 files changed, 97 insertions(+), 1 deletion(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 4185b03..0b8e470 100644
@@ -30,7 +30,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
     atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
     amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
     amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
+   amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o
 
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c uint32_t amdgpu_mm_rreg(struct amdgpu_device

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
    # add asic specific block
    amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o

kv_dpm.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f82919d..9a2fd3e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
@@ -95,6 +95,9 @@ uint32_t amdgpu_mm_rreg(struct amdgpu_device 

*adev, uint32_t reg,

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
    {
     uint32_t ret;
 
+   if (amdgpu_sriov_runtime(adev) && !in_interrupt())
+           return amdgpu_virt_kiq_rreg(adev, reg);
+
Quoted 6 lines
Quoted 6 lines

Mhm, why is the in_interrupt() necessary here?

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
     if ((reg * 4) < adev->rmmio_size && !always_indirect)
             ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
     else {
@@ -114,6 +117,9 @@ void amdgpu_mm_wreg(struct amdgpu_device 

*adev, uint32_t reg, uint32_t v,

Quoted 7 lines
Quoted 7 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c void amdgpu_mm_wreg(struct amdgpu_device · line 122

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
    {
 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+   if (amdgpu_sriov_runtime(adev))
+           return amdgpu_virt_kiq_wreg(adev, reg, v);
+

And why it isn't necessary here? What happens when we write a

register from an interrupt routine?

Quoted 19 lines
Quoted 19 lines

Christian.

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 -0
     if ((reg * 4) < adev->rmmio_size && !always_indirect)
             writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
     else {
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +6 -0
new file mode 100644
index 0000000..7861b6b
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
Quoted 8 lines
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c line 14

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +8 -0
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to
+whom the
+ * Software is furnished to do so, subject to the following 

conditions:

Quoted 121 lines
Quoted 110 lines
Quoted 110 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c new file +79 -0
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_virt.h"
+
+void amdgpu_virt_init_setting(struct amdgpu_device *adev) {
+   mutex_init(&adev->virt.lock);
+}
+
+uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t
+reg) {
+   signed long r;
+   uint32_t val;
+   struct fence *f;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+
+   BUG_ON(!ring->funcs->emit_rreg);
+
+   mutex_lock(&adev->virt.lock);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_hdp_flush(ring);
+   amdgpu_ring_emit_rreg(ring, reg);
+   amdgpu_ring_emit_hdp_invalidate(ring);
+   amdgpu_fence_emit(ring, &f);
+   amdgpu_ring_commit(ring);
+   mutex_unlock(&adev->virt.lock);
+
+   r = fence_wait(f, false);
+   if (r)
+           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+   fence_put(f);
+
+   val = adev->wb.wb[adev->virt.reg_val_offs];
+
+   return val;
+}
+
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v) {
+   signed long r;
+   struct fence *f;
+   struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+   struct amdgpu_ring *ring = &kiq->ring;
+
+   BUG_ON(!ring->funcs->emit_wreg);
+
+   mutex_lock(&adev->virt.lock);
+   amdgpu_ring_alloc(ring, 32);
+   amdgpu_ring_emit_hdp_flush(ring);
+   amdgpu_ring_emit_wreg(ring, reg, v);
+   amdgpu_ring_emit_hdp_invalidate(ring);
+   amdgpu_fence_emit(ring, &f);
+   amdgpu_ring_commit(ring);
+   mutex_unlock(&adev->virt.lock);
+
+   r = fence_wait(f, false);
+   if (r)
+           DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+   fence_put(f);
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1 -0
index 3d38a9f..2869980 100644
@@ -33,6 +33,7 @@
    struct amdgpu_virt {
     uint32_t                caps;
     uint32_t                reg_val_offs;
+   struct mutex            lock;
    };
 
    #define amdgpu_sriov_enabled(adev) \ @@ -59,4 +60,8 @@ static

inline bool is_virtual_machine(void)

    #endif
    }
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +5 -0
+void amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t
+amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
+void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg,
+uint32_t v);
+
    #endif
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/vi.c index 7350a8f..dc0d4fa 100644

drivers/gpu/drm/amd/amdgpu/vi.c +0 -0
@@ -892,6 +892,9 @@ static int vi_common_early_init(void *handle)
             (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
             smc_enabled = true;
 
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/vi.c static int vi_common_early_init(void *handle) · line 900

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/vi.c +3 -0
+   if (amdgpu_sriov_vf(adev))
+           amdgpu_virt_init_setting(adev);
+
     adev->rev_id = vi_get_rev_id(adev);
     adev->external_rev_id = 0xFF;
     switch (adev->asic_type) {

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Quoted 4 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#27 lore raw

PATCH v3 5/11 drm/amdgpu/virt: add high level interfaces for virt

In reply to a message not in the archive

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57 ++++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +57 -0
index 7861b6b..4e7df8e 100644
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +14 -0
index 2869980..45771b0 100644
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations
+ */
+struct amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev);
+};
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1484140698-10548-6-git-send-email-Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org

#29 lore raw

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Hi Xiangliang

This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add my "signed-off-by: Monk Liu " line on it, Seems you add an upper layer interface to invoke on gpu_request/release routines,
And I will also take role in the reviewing of it:

+ adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+ return virt->ops->req_full_gpu(adev, init);

[ml]One nitpick , you need place " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

+ r = virt->ops->rel_full_gpu(adev, init);
+ adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ? and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return function)

+ adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+ return virt->ops->reset_gpu(adev);
+ }

[ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME" should be put to the second line as well, like:

r = virt-ops->reset_gpu(adev);
If (!r)

    adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
else
    BUG(); /* we shouldn't fail in requesting GPU_RESET, check hypervisor please */

return r;

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>
Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57 ++++++++++++++++++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +56 -0
index 7861b6b..4e7df8e 100644
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) 
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) 
+{
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +12 -1
index 2869980..45771b0 100644
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
 #define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
 #define AMDGPU_SRIOV_CAPS_RUNTIME      (1 << 4) /* is out of full access mode */
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct 
+amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev); };
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void amdgpu_virt_init_setting(struct amdgpu_device *adev);  uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);  void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
+init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, 
+bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
--

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB1110AB98F8704CD111B4883984660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#31 lore raw

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Quoted 10 lines

This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add my "signed-off-by: Monk Liu " line on it, Seems you add an upper layer interface to invoke on gpu_request/release routines, And I will also take role in the reviewing of it:

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);

[ml]One nitpick , you need place " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ? and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return function)

I think it is more flexible, we can handle error according to return result.

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
Quoted 8 lines

[ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME" should be put to the second line as well, like:

r = virt-ops->reset_gpu(adev);
If (!r)

    adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
    BUG(); /* we shouldn't fail in requesting GPU_RESET, check hypervisor

please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

Quoted 128 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>
Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
++++++++++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +56 -0
index 7861b6b..4e7df8e 100644
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
 *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 2869980..45771b0 100644
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual

function */
#define AMDGPU_PASSTHROUGH_MODE (1 << 3) /* thw whole GPU is
pass through for VM */
#define AMDGPU_SRIOV_CAPS_RUNTIME (1 << 4) /* is out of full access
mode */

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +9 -0
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct
+amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev); };
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void

amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +3 -1
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
+init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
+bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
--
Quoted 1 line

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB17019B4D879772A487182BC7EB660-rpdhrqHFk05QaJCA3gGb3wdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#33 lore raw

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

See in lines

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:27 PM
To: Liu, Monk <[email protected]>; [email protected]
Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Quoted 10 lines

This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add my "signed-off-by: Monk Liu " line on it, Seems you add an upper layer interface to invoke on gpu_request/release routines, And I will also take role in the reviewing of it:

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);

[ml]One nitpick , you need place " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

[ml] NO, you didn't get my point, you should &= ~ AMDGPU_SRIOV_CAPS_RUNTIME *after* virt->ops->req_full_gpu(adev, init)

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ? and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return
function)

I think it is more flexible, we can handle error according to return result.

[ML] then please handle the error accordingly, but your patch just ignore the return value . see it yourself

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
Quoted 8 lines

[ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME" should be put to the second line as well, like:

r = virt-ops->reset_gpu(adev);
If (!r)

    adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
    BUG(); /* we shouldn't fail in requesting GPU_RESET, check 

hypervisor please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

[ML] you still not get my point, if reset_gpu() failed. You shouldn't "&= ~AMDGPU_SRIOV_CAPS_RUNTIME", but you did it in prior to reset_gpu(), isn't it ??

Quoted 129 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>
Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
++++++++++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +56 -0
index 7861b6b..4e7df8e 100644
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device 
 *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
+init) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool 
+init) {
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 2869980..45771b0 100644
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual

function */
#define AMDGPU_PASSTHROUGH_MODE (1 << 3) /* thw whole GPU is
pass through for VM */
#define AMDGPU_SRIOV_CAPS_RUNTIME (1 << 4) /* is out of full access
mode */

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +9 -0
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct 
+amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev); };
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void 

amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +3 -1
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
+init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, 
+bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
--
Quoted 1 line

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB1110740201E46818E3F94E2084660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#35 lore raw

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Quoted 24 lines

-----Original Message-----
From: Liu, Monk
Sent: Wednesday, January 11, 2017 10:33 PM
To: Yu, Xiangliang <[email protected]>; amd- [email protected]
Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

See in lines

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:27 PM
To: Liu, Monk <[email protected]>; [email protected]
Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Quoted 8 lines

This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add my "signed-off-by: Monk Liu " line on it, Seems you add an upper layer interface to invoke on gpu_request/release routines, And I will also take role in the reviewing of it:

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
Quoted 8 lines
Quoted 2 lines

[ml]One nitpick , you need place " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

[ml] NO, you didn't get my point, you should &= ~ AMDGPU_SRIOV_CAPS_RUNTIME *after* virt->ops->req_full_gpu(adev, init)

Good catch.

Quoted 13 lines
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ? and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return
function)

I think it is more flexible, we can handle error according to return result.

Quoted 2 lines

[ML] then please handle the error accordingly, but your patch just ignore the return value . see it yourself

I have no time to think of the design, maybe later.

Quoted 153 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}

[ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME" should be

put

Quoted 7 lines

to the second line as well, like:

r = virt-ops->reset_gpu(adev);
If (!r)

    adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
    BUG(); /* we shouldn't fail in requesting GPU_RESET, check

hypervisor please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

[ML] you still not get my point, if reset_gpu() failed. You shouldn't "&= ~AMDGPU_SRIOV_CAPS_RUNTIME", but you did it in prior to reset_gpu(), isn't it ??

Quoted 96 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>
Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
++++++++++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +56 -0
index 7861b6b..4e7df8e 100644
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device
 *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
+init) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool
+init) {
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 2869980..45771b0 100644
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual
Quoted 2 lines

function */
#define AMDGPU_PASSTHROUGH_MODE (1 << 3) /* thw whole GPU

is

Quoted 2 lines

pass through for VM */
#define AMDGPU_SRIOV_CAPS_RUNTIME (1 << 4) /* is out of full

access

Quoted 27 lines

mode */

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +9 -0
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct
+amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev); };
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void

amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +3 -0
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool
+init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev,
+bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
Quoted 3 lines
Quoted 3 lines

#endif
--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#36 lore raw

RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

In reply to a message not in the archive
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ? and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return
function)

I think it is more flexible, we can handle error according to return result.

[ML] then please handle the error accordingly, but you always do "|= AMDGPU_SRIOV_CAPS_RUNTIME" no matter above rel_full_gpu() return okay or not , isn't it ?

That RUNTIME flag should only be set after successfully release the full access mode on GPU...

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:27 PM
To: Liu, Monk <[email protected]>; [email protected]
Subject: RE: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Quoted 10 lines

This patch is not derived from my work (amd-sriov-4.3/4.6), please don't add my "signed-off-by: Monk Liu " line on it, Seems you add an upper layer interface to invoke on gpu_request/release routines, And I will also take role in the reviewing of it:

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);

[ml]One nitpick , you need place " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;" in the second line,

I think it make more clear as it is not runtime when entering full access mode.

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2 -0
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;

[ml] what about rel_full_gpu() failed ? and if rel_full_gpu is designed not allowed to fail then you should type it as non-return function. ( I remember in my first initial implementation on rel_gpu function it is is non-return
function)

I think it is more flexible, we can handle error according to return result.

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3 -0
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
Quoted 8 lines

[ML] " adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME" should be put to the second line as well, like:

r = virt-ops->reset_gpu(adev);
If (!r)

    adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; else
    BUG(); /* we shouldn't fail in requesting GPU_RESET, check 

hypervisor please */ return r;

I think we can't hang out VM so rudely, need to print out error message and do error handling to enhance the process later.

Quoted 129 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>
Subject: [V3 05/11] drm/amdgpu/virt: add high level interfaces for virt

Add high level interfaces that is not relate to specific asic. So asic files just need to implement the interfaces to support virtualization.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 57
++++++++++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 14 ++++++++
 2 files changed, 71 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +56 -0
index 7861b6b..4e7df8e 100644
@@ -80,3 +80,60 @@ void amdgpu_virt_kiq_wreg(struct amdgpu_device 
 *adev, uint32_t reg, uint32_t v)
 		DRM_ERROR("wait for kiq fence error: %ld.\n", r);
 	fence_put(f);
 }
+
+/**
+ * amdgpu_virt_request_full_gpu() - request full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When start to init/fini driver, first need to request full gpu access.
+ * Return: Zero if request success, otherwise will return error.
+ */
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
+init) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_full_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->req_full_gpu(adev, init);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_virt_release_full_gpu() - release full gpu access
+ * @amdgpu:	amdgpu device.
+ * @init:	is driver init time.
+ * When finishing driver init/fini, need to release full gpu access.
+ * Return: Zero if release success, otherwise will returen error.
+ */
+int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool 
+init) {
+	struct amdgpu_virt *virt = &adev->virt;
+	int r;
+
+	if (virt->ops && virt->ops->rel_full_gpu) {
+		r = virt->ops->rel_full_gpu(adev, init);
+		adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
+		return r;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_virt_reset_gpu() - reset gpu
+ * @amdgpu:	amdgpu device.
+ * Send reset command to GPU hypervisor to reset GPU that VM is using
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) {
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->reset_gpu) {
+		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+		return virt->ops->reset_gpu(adev);
+	}
+
+	return 0;
+}
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +0 -0
index 2869980..45771b0 100644
@@ -29,11 +29,22 @@
 #define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual

function */
#define AMDGPU_PASSTHROUGH_MODE (1 << 3) /* thw whole GPU is
pass through for VM */
#define AMDGPU_SRIOV_CAPS_RUNTIME (1 << 4) /* is out of full access
mode */

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +9 -0
+
+/**
+ * struct amdgpu_virt_ops - amdgpu device virt operations  */ struct 
+amdgpu_virt_ops {
+	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
+	int (*reset_gpu)(struct amdgpu_device *adev); };
+
 /* GPU virtualization */
 struct amdgpu_virt {
 	uint32_t		caps;
 	uint32_t		reg_val_offs;
 	struct mutex		lock;
+	const struct amdgpu_virt_ops	*ops;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -63,5 +74,8 @@ static inline bool is_virtual_machine(void)  void 

amdgpu_virt_init_setting(struct amdgpu_device *adev); uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +3 -1
+int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool 
+init); int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, 
+bool init); int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
 
 #endif
--
Quoted 1 line

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#37 lore raw

PATCH v3 6/11 drm/amdgpu/virt: implement VI virt operation interfaces

In reply to a message not in the archive

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592 +++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
 amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +592 -0

592 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +55 -0

55 lines not on this page · raw message

Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1484140698-10548-7-git-send-email-Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org

#39 lore raw

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ int r;
+
+ adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+ r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+ /* do nothing for other msg */
+ if (r)
+ return 0;
+
+ /* TODO: need to save gfx states */
+ schedule_delayed_work(&adev->virt.flr_work,
+ msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+ return 0;
+}

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is IDH_FLR_NOTIFICATION_CMPL,

The previous one means GIM told us GIM will start do FLR
The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL,

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592 +++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \  amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +581 -0

581 lines not on this page · raw message

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +55 -0

55 lines not on this page · raw message

+ IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void +xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int +xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int +xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void +xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB1110D2A287766F891E89CC7184660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#41 lore raw

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 37 lines

-----Original Message-----
From: Liu, Monk
Sent: Wednesday, January 11, 2017 10:28 PM
To: Yu, Xiangliang <[email protected]>; amd- [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +18 -0
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is IDH_FLR_NOTIFICATION_CMPL,

The previous one means GIM told us GIM will start do FLR The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

Quoted 5 lines

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL,

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

Quoted 745 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
+++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode 100644
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

cik_ih.o kv_smc.o kv_dpm.o \ amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o

amdgpu-y += \

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +24 -0
new file mode 100644
index 0000000..d2622b6
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +263 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: [email protected]
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_settings_a10,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_settings_a10));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_mgcg_cgcg_init));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_settings_a11,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_settings_a11));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +121 -0
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs  */ static void
+xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, bool
+val) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true); }
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum idh_event
+event) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
+	return xgpu_vi_send_access_requests(adev,

IDH_REQ_GPU_RESET_ACCESS); }

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +7 -0
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS :

IDH_REQ_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +9 -0
+	return xgpu_vi_send_access_requests(adev, event); }
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS :

IDH_REL_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +73 -0
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */ }
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_ack_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs xgpu_vi_mailbox_rcv_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +39 -0
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work,

xgpu_vi_mailbox_flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +14 -0
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +24 -0
new file mode 100644
index 0000000..fd6216e
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +37 -1
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void
+xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
+xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
+xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
+xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
--
Quoted 1 line

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB1701F42FB13FE1C323E51D80EB660-rpdhrqHFk05QaJCA3gGb3wdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#43 lore raw

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

We will discuss it tomorrow, I'm not sure of your approach right now

We can stop running driver right after IDH_FLR_NOTIFICATION received and continue driver right after IDH_FLR_NOTIFICATION_CMPL

BR Monk

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:43 PM
To: Liu, Monk <[email protected]>; [email protected]
Cc: Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 35 lines

-----Original Message-----
From: Liu, Monk
Sent: Wednesday, January 11, 2017 10:28 PM
To: Yu, Xiangliang <[email protected]>; amd- [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +17 -0
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is IDH_FLR_NOTIFICATION_CMPL,

The previous one means GIM told us GIM will start do FLR The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

Quoted 5 lines

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL,

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

Quoted 748 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
+++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode 
100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o 

kv_smc.o kv_dpm.o \ amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o

amdgpu-y += \

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +24 -0
new file mode 100644
index 0000000..d2622b6
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +263 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: [email protected]
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_settings_a10,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_settings_a10));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_mgcg_cgcg_init));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_settings_a11,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_settings_a11));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +123 -0
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs  */ static 
+void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, 
+bool
+val) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true); }
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum 
+idh_event
+event) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
+	return xgpu_vi_send_access_requests(adev,

IDH_REQ_GPU_RESET_ACCESS); }

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +7 -0
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS :

IDH_REQ_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +9 -0
+	return xgpu_vi_send_access_requests(adev, event); }
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS :

IDH_REL_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +72 -0
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */ }
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs 
+xgpu_vi_mailbox_ack_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs 
+xgpu_vi_mailbox_rcv_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +39 -0
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work,

xgpu_vi_mailbox_flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +14 -0
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +24 -0
new file mode 100644
index 0000000..fd6216e
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom 
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +37 -1
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void 
+xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int 
+xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int 
+xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void 
+xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
--
Quoted 1 line

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): BY2PR1201MB11109FEFDEBA34BD58EFE3AE84660-O28G1zQ8oGliQkyLPkmea2rFom/aUZj6nBOFsp37pqbUKgpGm//BTAC/[email protected]

#45 lore raw

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 4 lines

We will discuss it tomorrow, I'm not sure of your approach right now

We can stop running driver right after IDH_FLR_NOTIFICATION received and continue driver right after IDH_FLR_NOTIFICATION_CMPL

I have verified the method on my platform and work fine. Let's discuss it tomorrow if you have any concern.

Quoted 800 lines

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:43 PM
To: Liu, Monk <[email protected]>; [email protected]
Cc: Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 28 lines

-----Original Message-----
From: Liu, Monk
Sent: Wednesday, January 11, 2017 10:28 PM
To: Yu, Xiangliang <[email protected]>; amd- [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +17 -0
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
Quoted 2 lines

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is

IDH_FLR_NOTIFICATION_CMPL,

Quoted 10 lines

The previous one means GIM told us GIM will start do FLR The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL,

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

drivers/gpu/drm/amd/amdgpu/Makefile amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

Quoted 34 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
+++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode
100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

cik_ih.o

Quoted 1 line

kv_smc.o kv_dpm.o \ amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o

gmc_v6_0.o

Quoted 46 lines

gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o

amdgpu-y += \

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +10 -0
new file mode 100644
index 0000000..d2622b6
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c line 20

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +10 -0
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

KIND,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +4 -0
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN

NO

Quoted 593 lines

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +263 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: [email protected]
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_settings_a10,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_settings_a10));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_mgcg_cgcg_init));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_settings_a11,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_settings_a11));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +123 -0
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs  */ static
+void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
+bool
+val) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true); }
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
+idh_event
+event) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
+	return xgpu_vi_send_access_requests(adev,

IDH_REQ_GPU_RESET_ACCESS); }

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +7 -0
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS :

IDH_REQ_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +9 -0
+	return xgpu_vi_send_access_requests(adev, event); }
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device *adev,
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS :

IDH_REL_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +72 -0
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */
+static int xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */ }
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs
+xgpu_vi_mailbox_ack_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq,
+};
+
+static const struct amdgpu_irq_src_funcs
+xgpu_vi_mailbox_rcv_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +39 -0
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq,
+};
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work,

xgpu_vi_mailbox_flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +14 -0
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +10 -0
new file mode 100644
index 0000000..fd6216e
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h line 20

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +10 -0
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom
+the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

KIND,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h line 24

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +4 -0
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN

NO

Quoted 39 lines

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +31 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); void
Quoted 8 lines
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +6 -1
+xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
+xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
+xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
+xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
+#endif
--

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB17019BC05EC206D150F4F109EB660-rpdhrqHFk05QaJCA3gGb3wdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#47 lore raw

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 4 lines
Quoted 4 lines

We will discuss it tomorrow, I'm not sure of your approach right now

We can stop running driver right after IDH_FLR_NOTIFICATION received and continue driver right after IDH_FLR_NOTIFICATION_CMPL

If missing the interrupt, I think we can't easy to handle the error. That what i want to avoid it.

Quoted 806 lines
Quoted 571 lines

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:43 PM
To: Liu, Monk <[email protected]>; [email protected]
Cc: Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 28 lines

-----Original Message-----
From: Liu, Monk
Sent: Wednesday, January 11, 2017 10:28 PM
To: Yu, Xiangliang <[email protected]>; amd- [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

(quoted diff) +17 -0
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
Quoted 2 lines

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is

IDH_FLR_NOTIFICATION_CMPL,

Quoted 10 lines

The previous one means GIM told us GIM will start do FLR The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL,

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

drivers/gpu/drm/amd/amdgpu/Makefile amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

Quoted 34 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
+++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode
100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

cik_ih.o

Quoted 1 line

kv_smc.o kv_dpm.o \ amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o

gmc_v6_0.o

Quoted 46 lines

gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o

amdgpu-y += \

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +10 -0
new file mode 100644
index 0000000..d2622b6
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c line 20

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +10 -0
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

KIND,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c line 24

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +4 -0
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN

NO

Quoted 418 lines

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +263 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: [email protected]
+ */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_settings_a10,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_settings_a10));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_mgcg_cgcg_init));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_settings_a11,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_settings_a11));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +118 -0
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs  */ static
+void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev,
+bool
+val) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true); }
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum
+idh_event
+event) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
+	return xgpu_vi_send_access_requests(adev,

IDH_REQ_GPU_RESET_ACCESS); }

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device

*adev,

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS :

IDH_REQ_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +3 -0
+	return xgpu_vi_send_access_requests(adev, event); }
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device

*adev,

Quoted 216 lines
Quoted 156 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS :

IDH_REL_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +72 -0
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */ static int
+xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */ }
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs
+xgpu_vi_mailbox_ack_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq, };
+
+static const struct amdgpu_irq_src_funcs
+xgpu_vi_mailbox_rcv_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +38 -0
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq, };
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work,

xgpu_vi_mailbox_flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +14 -0
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +10 -0
new file mode 100644
index 0000000..fd6216e
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+obtaining a
+ * copy of this software and associated documentation files (the
+"Software"),
+ * to deal in the Software without restriction, including without
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h line 20

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +10 -0
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

KIND,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h line 24

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +4 -0
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN

NO

Quoted 44 lines

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM,
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +36 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */
+enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev);
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); int
+xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int
+xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void
+xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
Quoted 8 lines
Quoted 3 lines
Quoted 3 lines

+#endif
--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CY4PR12MB1701C566CD09B3B39CE5A043EB660-rpdhrqHFk05QaJCA3gGb3wdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org

#49 lore raw

RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Sounds make sense!
BR Monk

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 11:04 PM
To: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; [email protected]
Cc: Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 4 lines
Quoted 4 lines

We will discuss it tomorrow, I'm not sure of your approach right now

We can stop running driver right after IDH_FLR_NOTIFICATION received and continue driver right after IDH_FLR_NOTIFICATION_CMPL

If missing the interrupt, I think we can't easy to handle the error. That what i want to avoid it.

Quoted 804 lines
Quoted 570 lines

-----Original Message-----
From: Yu, Xiangliang
Sent: Wednesday, January 11, 2017 10:43 PM
To: Liu, Monk <[email protected]>; [email protected]
Cc: Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

Quoted 28 lines

-----Original Message-----
From: Liu, Monk
Sent: Wednesday, January 11, 2017 10:28 PM
To: Yu, Xiangliang <[email protected]>; amd- [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: RE: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

(quoted diff) +17 -0
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
Quoted 2 lines

[ML] we should be able to receive at least two type of IRQ from mailbox:
One is FLR_NOTIFICATION and another one is

IDH_FLR_NOTIFICATION_CMPL,

Quoted 10 lines

The previous one means GIM told us GIM will start do FLR The later one means GIM told us GIM already finished FLR and from that on Guest driver can do re-initialize .

You only handle the first IRQ, and dispatch a delay_work to monitor event of IDH_FLR_NOTIFICATION_CMPL,

I suspect that correct scheme may be you handle both of those two type of IRQ , please check with @Frank and me on the detail tomorrow .

As I think we may suspend gpu after receive FLR interrupt, polling method is safer.

drivers/gpu/drm/amd/amdgpu/Makefile amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

Quoted 34 lines

BR Monk

-----Original Message-----
From: Xiangliang Yu [mailto:[email protected]]
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>; Liu, Monk <[email protected]>; Liu, Shaoyun <[email protected]>
Subject: [V3 06/11] drm/amdgpu/virt: implement VI virt operation interfaces

VI has asic specific virt support, which including mailbox and golden registers init.

---
 drivers/gpu/drm/amd/amdgpu/Makefile      |   2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h |   9 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c    | 592
+++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h    |  55 +++
 4 files changed, 654 insertions(+), 4 deletions(-)  create mode
100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
drivers/gpu/drm/amd/amdgpu/Makefile +0 -0
index 0b8e470..388e7b0 100644
@@ -40,7 +40,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o

cik_ih.o

Quoted 1 line

kv_smc.o kv_dpm.o \ amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o

gmc_v6_0.o

Quoted 46 lines

gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o

amdgpu-y += \

drivers/gpu/drm/amd/amdgpu/Makefile +1 -1
-	vi.o
+	vi.o mxgpu_vi.o
 
 # add GMC block
 amdgpu-y += \
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +6 -3
index 45771b0..a409eec 100644
@@ -41,9 +41,12 @@ struct amdgpu_virt_ops {
 
 /* GPU virtualization */
 struct amdgpu_virt {
-	uint32_t		caps;
-	uint32_t		reg_val_offs;
-	struct mutex		lock;
+	uint32_t			caps;
+	uint32_t			reg_val_offs;
+	struct mutex			lock;
+	struct amdgpu_irq_src		ack_irq;
+	struct amdgpu_irq_src		rcv_irq;
+	struct delayed_work		flr_work;
 	const struct amdgpu_virt_ops	*ops;
 };
 
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +10 -0
new file mode 100644
index 0000000..d2622b6
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c line 20

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +10 -0
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to 
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

KIND,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c line 24

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +4 -0
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN

NO

Quoted 417 lines

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +262 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: [email protected]  */
+
+#include "amdgpu.h"
+#include "vi.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "vid.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gmc_v8_0.h"
+#include "gfx_v8_0.h"
+#include "sdma_v3_0.h"
+#include "tonga_ih.h"
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu/smu_7_1_3_d.h"
+#include "mxgpu_vi.h"
+
+/* VI golden setting */
+static const u32 xgpu_fiji_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+	mmPCIE_DATA, 0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100, };
+
+static const u32 xgpu_fiji_golden_settings_a10[] = {
+	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_fiji_golden_common_all[] = {
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
+	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, };
+
+static const u32 xgpu_tonga_mgcg_cgcg_init[] = {
+	mmRLC_CGTT_MGCG_OVERRIDE,   0xffffffff, 0xffffffff,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCB_CGTT_SCLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_BCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CP_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_CPC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_CPF_CLK_CTRL,        0xffffffff, 0x40000100,
+	mmCGTT_DRM_CLK_CTRL0,       0xffffffff, 0x00600100,
+	mmCGTT_GDS_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_IA_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PA_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_WD_CLK_CTRL,         0xffffffff, 0x06000100,
+	mmCGTT_PC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_RLC_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SC_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SPI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SQ_CLK_CTRL,         0xffffffff, 0x00000100,
+	mmCGTT_SQG_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL0,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL1,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL2,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL3,        0xffffffff, 0x00000100,
+	mmCGTT_SX_CLK_CTRL4,        0xffffffff, 0x00000100,
+	mmCGTT_TCI_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_TCP_CLK_CTRL,        0xffffffff, 0x00000100,
+	mmCGTT_VGT_CLK_CTRL,        0xffffffff, 0x06000100,
+	mmDB_CGTT_CLK_CTRL_0,       0xffffffff, 0x00000100,
+	mmTA_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmTCA_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTCC_CGTT_SCLK_CTRL,       0xffffffff, 0x00000100,
+	mmTD_CGTT_CTRL,             0xffffffff, 0x00000100,
+	mmGRBM_GFX_INDEX,           0xffffffff, 0xe0000000,
+	mmCGTS_CU0_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU0_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU1_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU1_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU1_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU2_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU2_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU2_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU3_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU3_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU3_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU4_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
+	mmCGTS_CU4_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU5_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU5_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU5_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU6_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU6_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU6_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_CU7_SP0_CTRL_REG,    0xffffffff, 0x00010000,
+	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
+	mmCGTS_CU7_TA_CTRL_REG,     0xffffffff, 0x00040007,
+	mmCGTS_CU7_SP1_CTRL_REG,    0xffffffff, 0x00060005,
+	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
+	mmCGTS_SM_CTRL_REG,         0xffffffff, 0x96e00200,
+	mmCP_RB_WPTR_POLL_CNTL,     0xffffffff, 0x00900100,
+	mmRLC_CGCG_CGLS_CTRL,       0xffffffff, 0x0020003c,
+	mmPCIE_INDEX,               0xffffffff, 0x0140001c,
+	mmPCIE_DATA,                0x000f0000, 0x00000000,
+	mmSMC_IND_INDEX_4,          0xffffffff, 0xC060000C,
+	mmSMC_IND_DATA_4,           0xc0000fff, 0x00000100,
+	mmXDMA_CLOCK_GATING_CNTL,   0xffffffff, 0x00000100,
+	mmXDMA_MEM_POWER_CNTL,      0x00000101, 0x00000000,
+	mmMC_MEM_POWER_LS,          0xffffffff, 0x00000104,
+	mmCGTT_DRM_CLK_CTRL0,       0xff000fff, 0x00000100,
+	mmHDP_XDP_CGTT_BLK_CTRL,    0xc0000fff, 0x00000104,
+	mmCP_MEM_SLP_CNTL,          0x00000001, 0x00000001,
+	mmSDMA0_CLK_CTRL,           0xff000ff0, 0x00000100,
+	mmSDMA1_CLK_CTRL,           0xff000ff0, 0x00000100,
+};
+
+static const u32 xgpu_tonga_golden_settings_a11[] = {
+	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
+	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
+	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+	mmFBC_MISC, 0x1f311fff, 0x12300000,
+	mmGB_GPU_ID, 0x0000000f, 0x00000000,
+	mmHDMI_CONTROL, 0x31000111, 0x00000011,
+	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
+	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
+	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
+	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
+	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
+	mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+	mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000,
+	mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
+	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
+	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
+	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
+	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
+	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
+	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
+	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, };
+
+static const u32 xgpu_tonga_golden_common_all[] = {
+	mmGRBM_GFX_INDEX,               0xffffffff, 0xe0000000,
+	mmPA_SC_RASTER_CONFIG,          0xffffffff, 0x16000012,
+	mmPA_SC_RASTER_CONFIG_1,        0xffffffff, 0x0000002A,
+	mmGB_ADDR_CONFIG,               0xffffffff, 0x22011002,
+	mmSPI_RESOURCE_RESERVE_CU_0,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_CU_1,    0xffffffff, 0x00000800,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, };
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) {
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_fiji_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+						 xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_settings_a10,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_settings_a10));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_fiji_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_fiji_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+		break;
+	case CHIP_TONGA:
+		amdgpu_program_register_sequence(adev,
+						 xgpu_tonga_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_mgcg_cgcg_init));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_settings_a11,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_settings_a11));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+		amdgpu_program_register_sequence(adev,
+

xgpu_tonga_golden_common_all,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+						 (const u32)ARRAY_SIZE(
+

xgpu_tonga_golden_common_all));

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +118 -0
+		break;
+	default:
+		BUG_ON("Doesn't support chip type.\n");
+		break;
+	}
+}
+
+/*
+ * Mailbox communication between GPU hypervisor and VFs  */ 
+static void xgpu_vi_mailbox_send_ack(struct amdgpu_device *adev) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL, RCV_MSG_ACK, 1);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_set_valid(struct amdgpu_device *adev, 
+bool
+val) {
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	reg = REG_SET_FIELD(reg, MAILBOX_CONTROL,
+			    TRN_MSG_VALID, val ? 1 : 0);
+	WREG32(mmMAILBOX_CONTROL, reg);
+}
+
+static void xgpu_vi_mailbox_trans_msg(struct amdgpu_device *adev,
+				      enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_TRN_DW0);
+	reg = REG_SET_FIELD(reg, MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, event);
+	WREG32(mmMAILBOX_MSGBUF_TRN_DW0, reg);
+
+	xgpu_vi_mailbox_set_valid(adev, true); }
+
+static int xgpu_vi_mailbox_rcv_msg(struct amdgpu_device *adev,
+				   enum idh_event event)
+{
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_MSGBUF_RCV_DW0);
+	if (reg != event)
+		return -ENOENT;
+
+	/* send ack to PF */
+	xgpu_vi_mailbox_send_ack(adev);
+
+	return 0;
+}
+
+static int xgpu_vi_poll_ack(struct amdgpu_device *adev) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+	u32 mask = REG_FIELD_MASK(MAILBOX_CONTROL, TRN_MSG_ACK);
+	u32 reg;
+
+	reg = RREG32(mmMAILBOX_CONTROL);
+	while (!(reg & mask)) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		reg = RREG32(mmMAILBOX_CONTROL);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_poll_msg(struct amdgpu_device *adev, enum 
+idh_event
+event) {
+	int r = 0, timeout = VI_MAILBOX_TIMEDOUT;
+
+	r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	while (r) {
+		if (timeout <= 0) {
+			pr_err("Doesn't get ack from pf.\n");
+			r = -ETIME;
+			break;
+		}
+		msleep(1);
+		timeout -= 1;
+
+		r = xgpu_vi_mailbox_rcv_msg(adev, event);
+	}
+
+	return r;
+}
+
+static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request request)
+{
+	int r;
+
+	xgpu_vi_mailbox_trans_msg(adev, request);
+
+	/* start to poll ack */
+	r = xgpu_vi_poll_ack(adev);
+	if (r)
+		return r;
+
+	xgpu_vi_mailbox_set_valid(adev, false);
+
+	/* start to check msg if request is idh_req_gpu_init_access */
+	if (request == IDH_REQ_GPU_INIT_ACCESS) {
+		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
+		if (r)
+			return r;
+	}
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+	return 0;
+}
+
+static int xgpu_vi_request_reset(struct amdgpu_device *adev) {
+	return xgpu_vi_send_access_requests(adev,

IDH_REQ_GPU_RESET_ACCESS); }

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +2 -0
+
+static int xgpu_vi_request_full_gpu_access(struct amdgpu_device

*adev,

Quoted 9 lines

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+					   bool init)
+{
+	enum idh_event event;
+
+	event = init ? IDH_REQ_GPU_INIT_ACCESS :

IDH_REQ_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +3 -0
+	return xgpu_vi_send_access_requests(adev, event); }
+
+static int xgpu_vi_release_full_gpu_access(struct amdgpu_device

*adev,

Quoted 215 lines
Quoted 156 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +6 -0
+					   bool init)
+{
+	enum idh_event event;
+	int r = 0;
+
+	event = init ? IDH_REL_GPU_INIT_ACCESS :

IDH_REL_GPU_FINI_ACCESS;

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +72 -0
+	r = xgpu_vi_send_access_requests(adev, event);
+
+	return r;
+}
+
+/* add support mailbox interrupts */ static int 
+xgpu_vi_mailbox_ack_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	DRM_DEBUG("get ack intr and do nothing.\n");
+	return 0;
+}
+
+static int xgpu_vi_set_mailbox_ack_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, ACK_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static void xgpu_vi_mailbox_flr_work(struct work_struct *work) {
+	struct amdgpu_virt *virt = container_of(work,
+					struct amdgpu_virt, flr_work.work);
+	struct amdgpu_device *adev = container_of(virt,
+					struct amdgpu_device, virt);
+	int r = 0;
+
+	r = xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+	if (r)
+		DRM_ERROR("failed to get flr cmpl msg from hypervior.\n");
+
+	/* TODO: need to restore gfx states */ }
+
+static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *src,
+				       unsigned type,
+				       enum amdgpu_interrupt_state state) {
+	u32 tmp = RREG32(mmMAILBOX_INT_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, MAILBOX_INT_CNTL, VALID_INT_EN,
+			    (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
+	WREG32(mmMAILBOX_INT_CNTL, tmp);
+
+	return 0;
+}
+
+static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry) {
+	int r;
+
+	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+	r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
+	/* do nothing for other msg */
+	if (r)
+		return 0;
+
+	/* TODO: need to save gfx states */
+	schedule_delayed_work(&adev->virt.flr_work,
+			      msecs_to_jiffies(VI_MAILBOX_RESET_TIME));
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs 
+xgpu_vi_mailbox_ack_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +5 -0
+	.set = xgpu_vi_set_mailbox_ack_irq,
+	.process = xgpu_vi_mailbox_ack_irq, };
+
+static const struct amdgpu_irq_src_funcs 
+xgpu_vi_mailbox_rcv_irq_funcs =

{

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +38 -0
+	.set = xgpu_vi_set_mailbox_rcv_irq,
+	.process = xgpu_vi_mailbox_rcv_irq, };
+
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev) {
+	adev->virt.ack_irq.num_types = 1;
+	adev->virt.ack_irq.funcs = &xgpu_vi_mailbox_ack_irq_funcs;
+	adev->virt.rcv_irq.num_types = 1;
+	adev->virt.rcv_irq.funcs = &xgpu_vi_mailbox_rcv_irq_funcs; }
+
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_add_id(adev, 135, &adev->virt.rcv_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 138, &adev->virt.ack_irq);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	return 0;
+}
+
+int xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev) {
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
+	if (r)
+		return r;
+	r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
+		return r;
+	}
+
+	INIT_DELAYED_WORK(&adev->virt.flr_work,

xgpu_vi_mailbox_flr_work);

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c new file +14 -0
+
+	return 0;
+}
+
+void xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev) {
+	cancel_delayed_work_sync(&adev->virt.flr_work);
+	amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
+	amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); }
+
+const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
+	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
+	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
+	.reset_gpu		= xgpu_vi_request_reset,
+};
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +10 -0
new file mode 100644
index 0000000..fd6216e
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person 
+obtaining a
+ * copy of this software and associated documentation files (the 
+"Software"),
+ * to deal in the Software without restriction, including without 
+limitation
+ * the rights to use, copy, modify, merge, publish, distribute, 

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h line 20

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +10 -0
+sublicense,
+ * and/or sell copies of the Software, and to permit persons to 
+whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

KIND,

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h line 24

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +4 -0
+EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN

NO

Quoted 43 lines

EVENT

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +6 -0
+SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR

THE USE

drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h new file +35 -0
+OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __MXGPU_VI_H__
+#define __MXGPU_VI_H__
+
+#define VI_MAILBOX_TIMEDOUT	150
+#define VI_MAILBOX_RESET_TIME	12
+
+/* VI mailbox messages request */ enum idh_request {
+	IDH_REQ_GPU_INIT_ACCESS	= 1,
+	IDH_REL_GPU_INIT_ACCESS,
+	IDH_REQ_GPU_FINI_ACCESS,
+	IDH_REL_GPU_FINI_ACCESS,
+	IDH_REQ_GPU_RESET_ACCESS
+};
+
+/* VI mailbox messages data */
+enum idh_event {
+	IDH_CLR_MSG_BUF = 0,
+	IDH_READY_TO_ACCESS_GPU,
+	IDH_FLR_NOTIFICATION,
+	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_EVENT_MAX
+};
+
+extern const struct amdgpu_virt_ops xgpu_vi_virt_ops;
+
+void xgpu_vi_init_golden_registers(struct amdgpu_device *adev); 
+void xgpu_vi_mailbox_set_irq_funcs(struct amdgpu_device *adev); 
+int xgpu_vi_mailbox_add_irq_id(struct amdgpu_device *adev); int 
+xgpu_vi_mailbox_get_irq(struct amdgpu_device *adev); void 
+xgpu_vi_mailbox_put_irq(struct amdgpu_device *adev);
+
Quoted 8 lines
Quoted 3 lines
Quoted 3 lines

+#endif
--
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#50 lore raw

PATCH v3 7/11 drm/amdgpu: request/release full gpu access if device is vf

In reply to a message not in the archive

For gpu vf device, first need to request full gpu access before accessing gpu registers, and release full gpu access after the access is done.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 9 +++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 6 ++++++
 2 files changed, 15 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +9 -0
index 9a2fd3e..03d28a1 100644
@@ -1330,6 +1330,12 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_virt_request_full_gpu(adev, true);
+		if (r)
+			return r;
+	}
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
 			DRM_ERROR("disabled ip block: %d\n", i);
@@ -1523,6 +1529,9 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.late_initialized = false;
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, false);
+
 	return 0;
 }
 
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +6 -0
index 47bc8e1..eb228c7 100644
@@ -60,6 +60,9 @@ int amdgpu_driver_unload_kms(struct drm_device *dev)
 	if (adev->rmmio == NULL)
 		goto done_free;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_request_full_gpu(adev, false);
+
 	if (amdgpu_device_is_px(dev)) {
 		pm_runtime_get_sync(dev->dev);
 		pm_runtime_forbid(dev->dev);
@@ -139,6 +142,9 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
 		pm_runtime_put_autosuspend(dev->dev);
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, true);
+
 out:
 	if (r) {
 		/* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#51 lore raw

PATCH v3 8/11 drm/amdgpu/vi: add support virtualization

In reply to a message not in the archive

Call VI virtualization functions if device is Vf.

---
 drivers/gpu/drm/amd/amdgpu/vi.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/vi.c +32 -2
index dc0d4fa..34aa603 100644
@@ -77,6 +77,7 @@
 #include "amdgpu_dm.h"
 #include "amdgpu_powerplay.h"
 #include "dce_virtual.h"
+#include "mxgpu_vi.h"
 
 /*
  * Indirect registers accessor
@@ -273,6 +274,12 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 	mutex_lock(&adev->grbm_idx_mutex);
 
+	if (amdgpu_sriov_vf(adev)) {
+		xgpu_vi_init_golden_registers(adev);
+		mutex_unlock(&adev->grbm_idx_mutex);
+		return;
+	}
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		amdgpu_program_register_sequence(adev,
@@ -892,8 +899,10 @@ static int vi_common_early_init(void *handle)
 		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
-	if (amdgpu_sriov_vf(adev))
+	if (amdgpu_sriov_vf(adev)) {
 		amdgpu_virt_init_setting(adev);
+		xgpu_vi_mailbox_set_irq_funcs(adev);
+	}
 
 	adev->rev_id = vi_get_rev_id(adev);
 	adev->external_rev_id = 0xFF;
@@ -1059,8 +1068,23 @@ static int vi_common_early_init(void *handle)
 	return 0;
 }
 
+static int vi_common_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_get_irq(adev);
+
+	return 0;
+}
+
 static int vi_common_sw_init(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_add_irq_id(adev);
+
 	return 0;
 }
 
@@ -1092,6 +1116,9 @@ static int vi_common_hw_fini(void *handle)
 	/* enable the doorbell aperture */
 	vi_enable_doorbell_aperture(adev, false);
 
+	if (amdgpu_sriov_vf(adev))
+		xgpu_vi_mailbox_put_irq(adev);
+
 	return 0;
 }
 
@@ -1376,7 +1403,7 @@ static int vi_common_set_powergating_state(void *handle,
 static const struct amd_ip_funcs vi_common_ip_funcs = {
 	.name = "vi_common",
 	.early_init = vi_common_early_init,
-	.late_init = NULL,
+	.late_init = vi_common_late_init,
 	.sw_init = vi_common_sw_init,
 	.sw_fini = vi_common_sw_fini,
 	.hw_init = vi_common_hw_init,
@@ -1404,6 +1431,9 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
 	/* in early init stage, vbios code won't work */
 	vi_detect_hw_virtualization(adev);
 
+	if (amdgpu_sriov_vf(adev))
+		adev->virt.ops = &xgpu_vi_virt_ops;
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		/* topaz has no DCE, UVD, VCE */
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#52 lore raw

PATCH v3 9/11 drm/amdgpu/virt: enable virtual display

In reply to a message not in the archive

Virtual display is default setting for virtualization, enable it.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 4 ++++
 1 file changed, 4 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -0
index 4e7df8e..4bc373e 100644
@@ -25,6 +25,10 @@
 
 void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 {
+	/* enable virtual display */
+	adev->mode_info.num_crtc = 1;
+	adev->enable_virtual_display = true;
+
 	mutex_init(&adev->virt.lock);
 }
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1484140698-10548-10-git-send-email-Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org

Reviewed-by Monk Liu
#54 lore raw

RE: [V3 09/11] drm/amdgpu/virt: enable virtual display

Reviewed-by Monk Liu

-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Xiangliang Yu
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>
Subject: [V3 09/11] drm/amdgpu/virt: enable virtual display

Virtual display is default setting for virtualization, enable it.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 4 ++++
 1 file changed, 4 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4 -1
index 4e7df8e..4bc373e 100644
@@ -25,6 +25,10 @@
 
 void amdgpu_virt_init_setting(struct amdgpu_device *adev)  {
+	/* enable virtual display */
+	adev->mode_info.num_crtc = 1;
+	adev->enable_virtual_display = true;
+
 	mutex_init(&adev->virt.lock);
 }
 
--

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#55 lore raw

PATCH v3 10/11 drm/amdgpu/virt: disable dc

In reply to a message not in the archive

Virtualization don't need the dc, disable it.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
index 03d28a1..a6d828c 100644
@@ -1619,6 +1619,9 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
  */
 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
 }
 
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1484140698-10548-11-git-send-email-Xiangliang.Yu-5C7GfCeVMHo@public.gmane.org

#57 lore raw

RE: [V3 10/11] drm/amdgpu/virt: disable dc

My RB as well

-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Xiangliang Yu
Sent: Wednesday, January 11, 2017 9:18 PM
To: [email protected]
Cc: Yu, Xiangliang <[email protected]>
Subject: [V3 10/11] drm/amdgpu/virt: disable dc

Virtualization don't need the dc, disable it.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -1
index 03d28a1..a6d828c 100644
@@ -1619,6 +1619,9 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
  */
 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)  {
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
 }
 
--

2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#58 lore raw

PATCH v3 11/11 drm/amdgpu: do not reset gpu for virtualization

In reply to a message not in the archive

Current amdgpu reset process only works on bare-metal and for SRIOV many inside it need re-work to adapt to vf device.

This is a temporary workaround to skip gpu reset.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +++
 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3 -0
index a6d828c..5705fbd 100644
@@ -2324,6 +2324,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 	int resched;
 	bool need_full_reset;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	if (!amdgpu_check_soft_reset(adev)) {
 		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
 		return 0;
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#59 lore raw
In reply to a message not in the archive

Patches #1-#3, #5 and #8 are Reviewed-by: Christian König <[email protected]>.

Patch #6, #9 and #10 are Acked-by: Christian König <[email protected]>.

Patch #4: Just send out a question on this.

Patch #7 is missing a comment that this applies during driver load/unload, cause that isn't obvious.

With that fixed the patch is Reviewed-by: Christian König <[email protected]> as well.

Patch #10 please add the comment that this is just a temporary workaround to the code as well.

With that fixed the patch is also Reviewed-by: Christian König <[email protected]>.

Regards,
Christian.

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 40 lines

This patch series will enable AMD GPU virtualization and wires it into amdgpu modules.

Changes in V3:
1. rename field name;
2. remove extra fence_get()/fence_put();
3. fixed coding style;
4. update patch comment;

Changes in V2:
1. define new high level interface for non-asic specific;
2. add asic specific file to support specific asic;
3. not include KIQ patch as it has been merged into kernel;
4. not include CSA patch as monk will submit it;

Xiangliang Yu (11):
drm/amdgpu/ring: add two interfaces to support r/w registers with kiq drm/amdgpu/gfx8: implement emit_rreg/wreg function drm/amdgpu/virt: add runtime flag
drm/amdgpu/virt: use kiq to access registers drm/amdgpu/virt: add high level interfaces for virt drm/amdgpu/virt: implement VI virt operation interfaces drm/amdgpu: request/release full gpu access if device is vf drm/amdgpu/vi: add support virtualization drm/amdgpu/virt: enable virtual display
drm/amdgpu/virt: disable dc
drm/amdgpu: do not reset gpu for virtualization

  drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
  drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592 +++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
  drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
  11 files changed, 926 insertions(+), 4 deletions(-)
Quoted 4 lines

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 28f25605-b2cf-2982-2bb0-59ee68a891a8-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#61 lore raw
Quoted 2 lines

Patch #7 is missing a comment that this applies during driver load/unload, cause that isn't obvious.

Not just only for loading/unloading, resume process will need to request/release full gpu access after FLR (function level reset). Later I'll upstream FLR patches.

Quoted 50 lines

With that fixed the patch is Reviewed-by: Christian König <[email protected]> as well.

Patch #10 please add the comment that this is just a temporary workaround to the code as well.

With that fixed the patch is also Reviewed-by: Christian König <[email protected]>.

Regards,
Christian.

Am 11.01.2017 um 14:18 schrieb Xiangliang Yu:

Quoted 27 lines

This patch series will enable AMD GPU virtualization and wires it into amdgpu modules.

Changes in V3:
1. rename field name;
2. remove extra fence_get()/fence_put(); 3. fixed coding style; 4. update patch comment;

Changes in V2:
1. define new high level interface for non-asic specific; 2. add asic specific file to support specific asic; 3. not include KIQ patch as it has been merged into kernel; 4. not include CSA patch as monk will submit it;

Xiangliang Yu (11):
drm/amdgpu/ring: add two interfaces to support r/w registers with kiq drm/amdgpu/gfx8: implement emit_rreg/wreg function drm/amdgpu/virt: add runtime flag
drm/amdgpu/virt: use kiq to access registers drm/amdgpu/virt: add high level interfaces for virt drm/amdgpu/virt: implement VI virt operation interfaces drm/amdgpu: request/release full gpu access if device is vf drm/amdgpu/vi: add support virtualization drm/amdgpu/virt: enable virtual display
drm/amdgpu/virt: disable dc
drm/amdgpu: do not reset gpu for virtualization

Quoted 9 lines
  drivers/gpu/drm/amd/amdgpu/Makefile        |   4 +-
  drivers/gpu/drm/amd/amdgpu/amdgpu.h        |   2 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  21 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    |   6 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h   |   2 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c   | 146 +++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h   |  30 +-
  drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c      |  37 ++
  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c      | 592
Quoted 8 lines

+++++++++++++++++++++++++++++

Quoted 7 lines
  drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h      |  55 +++
  drivers/gpu/drm/amd/amdgpu/vi.c            |  35 +-
  11 files changed, 926 insertions(+), 4 deletions(-)

create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c create mode 100644 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx