Skip to contentSkip to conversation

drm/amdkfd: Check int source id for utcl2 poison event

PATCH In review Hawking Zhang 2 participants 4 messages Aug 19, 2024 – Aug 20, 2024

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 drm/amdkfd: Check int source id for utcl2 poison event

Traditional utcl2 fault_status polling does not work in SRIOV environment. The polling of fault status register from guest side will be dropped by hardware.

Driver should switch to check utcl2 interrupt source id to identify utcl2 poison event. It is set to 1 when poisoned data interrupts are signaled.

---
 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 3 +--
 drivers/gpu/drm/amd/amdkfd/soc15_int.h          | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +1 -2
index a9c3580be8c9..1196dccbe6bc 100644
@@ -448,8 +448,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 		if (vmid_type && client_id == SOC15_IH_CLIENTID_VMC)
 			hub_inst = node_id / 4;
 
-		if (amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev,
-					hub_inst, vmid_type)) {
+		if (source_id == SOC15_INTSRC_VMC_UTCL2_POISON) {
 			event_interrupt_poison_consumption_v9(dev, pasid, client_id);
 			return;
 		}
drivers/gpu/drm/amd/amdkfd/soc15_int.h +1 -0
index 10138676f27f..e5c0205f2618 100644
@@ -29,6 +29,7 @@
 #define SOC15_INTSRC_CP_BAD_OPCODE	183
 #define SOC15_INTSRC_SQ_INTERRUPT_MSG	239
 #define SOC15_INTSRC_VMC_FAULT		0
+#define SOC15_INTSRC_VMC_UTCL2_POISON	1
 #define SOC15_INTSRC_SDMA_TRAP		224
 #define SOC15_INTSRC_SDMA_ECC		220
 #define SOC21_INTSRC_SDMA_TRAP		49
Signature · 1 line
2.17.1
#1 lore raw

PATCH 2/3 drm/amdkfd: Drop poison hanlding from gfx v10

Not supported.

---
 .../gpu/drm/amd/amdkfd/kfd_int_process_v10.c  | 71 -------------------
 1 file changed, 71 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c +0 -71
index 8e0d0356e810..bb8cbfc39b90 100644
@@ -129,63 +129,6 @@ enum SQ_INTERRUPT_ERROR_TYPE {
 				KFD_DEBUG_CP_BAD_OP_ECODE_MASK)		\
 				>> KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT)
 
-static void event_interrupt_poison_consumption(struct kfd_node *dev,
-				uint16_t pasid, uint16_t client_id)
-{
-	enum amdgpu_ras_block block = 0;
-	int old_poison, ret = -EINVAL;
-	uint32_t reset = 0;
-	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
-
-	if (!p)
-		return;
-
-	/* all queues of a process will be unmapped in one time */
-	old_poison = atomic_cmpxchg(&p->poison, 0, 1);
-	kfd_unref_process(p);
-	if (old_poison)
-		return;
-
-	switch (client_id) {
-	case SOC15_IH_CLIENTID_SE0SH:
-	case SOC15_IH_CLIENTID_SE1SH:
-	case SOC15_IH_CLIENTID_SE2SH:
-	case SOC15_IH_CLIENTID_SE3SH:
-	case SOC15_IH_CLIENTID_UTCL2:
-		ret = kfd_dqm_evict_pasid(dev->dqm, pasid);
-		block = AMDGPU_RAS_BLOCK__GFX;
-		if (ret)
-			reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
-		break;
-	case SOC15_IH_CLIENTID_SDMA0:
-	case SOC15_IH_CLIENTID_SDMA1:
-	case SOC15_IH_CLIENTID_SDMA2:
-	case SOC15_IH_CLIENTID_SDMA3:
-	case SOC15_IH_CLIENTID_SDMA4:
-		block = AMDGPU_RAS_BLOCK__SDMA;
-		reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
-		break;
-	default:
-		break;
-	}
-
-	kfd_signal_poison_consumed_event(dev, pasid);
-
-	/* resetting queue passes, do page retirement without gpu reset
-	 * resetting queue fails, fallback to gpu reset solution
-	 */
-	if (!ret)
-		dev_warn(dev->adev->dev,
-			"RAS poison consumption, unmap queue flow succeeded: client id %d\n",
-			client_id);
-	else
-		dev_warn(dev->adev->dev,
-			"RAS poison consumption, fall back to gpu reset flow: client id %d\n",
-			client_id);
-
-	amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, reset);
-}
-
 static bool event_interrupt_isr_v10(struct kfd_node *dev,
 					const uint32_t *ih_ring_entry,
 					uint32_t *patched_ihre,
@@ -332,11 +275,6 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 					REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1,
 							WGP_ID),
 					sq_intr_err_type);
-				if (sq_intr_err_type != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
-					sq_intr_err_type != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
-					event_interrupt_poison_consumption(dev, pasid, source_id);
-					return;
-				}
 				break;
 			default:
 				break;
@@ -362,9 +300,6 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 		   client_id == SOC15_IH_CLIENTID_SDMA7) {
 		if (source_id == SOC15_INTSRC_SDMA_TRAP) {
 			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
-		} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
-			event_interrupt_poison_consumption(dev, pasid, source_id);
-			return;
 		}
 	} else if (client_id == SOC15_IH_CLIENTID_VMC ||
 		   client_id == SOC15_IH_CLIENTID_VMC1 ||
@@ -388,12 +323,6 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 		if (vmid_type && client_id == SOC15_IH_CLIENTID_VMC)
 			hub_inst = node_id / 4;
 
-		if (amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev,
-					hub_inst, vmid_type)) {
-			event_interrupt_poison_consumption(dev, pasid, client_id);
-			return;
-		}
-
 		info.vmid = vmid;
 		info.mc_id = client_id;
 		info.page_addr = ih_ring_entry[4] |
Signature · 1 line
2.17.1
#2 lore raw

PATCH 3/3 drm/amdgpu: Retire query_utcl2_poison_status callback

In reply to Hawking Zhang

Driver switches to interrupt source id to identify utcl2 poison event. polling interface is not needed.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 16 ----------------
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  2 --
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h |  2 --
 drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h  |  2 --
 drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c   | 18 ------------------
 drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c   | 17 -----------------
 drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c    | 17 -----------------
 7 files changed, 74 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -16
index 64a989cbc301..4f08b153cb66 100644
@@ -783,22 +783,6 @@ int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev,
-			int hub_inst, int hub_type)
-{
-	if (!hub_type) {
-		if (adev->gfxhub.funcs->query_utcl2_poison_status)
-			return adev->gfxhub.funcs->query_utcl2_poison_status(adev, hub_inst);
-		else
-			return false;
-	} else {
-		if (adev->mmhub.funcs->query_utcl2_poison_status)
-			return adev->mmhub.funcs->query_utcl2_poison_status(adev, hub_inst);
-		else
-			return false;
-	}
-}
-
 int amdgpu_amdkfd_check_and_lock_kfd(struct amdgpu_device *adev)
 {
 	return kgd2kfd_check_and_lock_kfd();
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -2
index 825c7ffe4bc9..f9d119448442 100644
@@ -350,8 +350,6 @@ bool amdgpu_amdkfd_is_fed(struct amdgpu_device *adev);
 bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem);
 void amdgpu_amdkfd_block_mmu_notifications(void *p);
 int amdgpu_amdkfd_criu_resume(void *p);
-bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev,
-			int hub_inst, int hub_type);
 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag, int8_t xcp_id);
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h +0 -2
index 103a837ccc71..c7b44aeb671b 100644
@@ -38,8 +38,6 @@ struct amdgpu_gfxhub_funcs {
 	void (*mode2_save_regs)(struct amdgpu_device *adev);
 	void (*mode2_restore_regs)(struct amdgpu_device *adev);
 	void (*halt)(struct amdgpu_device *adev);
-	bool (*query_utcl2_poison_status)(struct amdgpu_device *adev,
-			int xcc_id);
 };
 
 struct amdgpu_gfxhub {
drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h +0 -2
index 95d676ee207f..1ca9d4ed8063 100644
@@ -63,8 +63,6 @@ struct amdgpu_mmhub_funcs {
 				uint64_t page_table_base);
 	void (*update_power_gating)(struct amdgpu_device *adev,
                                 bool enable);
-	bool (*query_utcl2_poison_status)(struct amdgpu_device *adev,
-				int hub_inst);
 };
 
 struct amdgpu_mmhub {
drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c +0 -18
index d200310d1731..0e3ddea7b8e0 100644
@@ -443,23 +443,6 @@ static void gfxhub_v1_0_init(struct amdgpu_device *adev)
 		mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
 }
 
-static bool gfxhub_v1_0_query_utcl2_poison_status(struct amdgpu_device *adev,
-				int xcc_id)
-{
-	u32 status = 0;
-	struct amdgpu_vmhub *hub;
-
-	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
-		return false;
-
-	hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
-	status = RREG32(hub->vm_l2_pro_fault_status);
-	/* reset page fault status */
-	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
-
-	return REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
-}
-
 const struct amdgpu_gfxhub_funcs gfxhub_v1_0_funcs = {
 	.get_mc_fb_offset = gfxhub_v1_0_get_mc_fb_offset,
 	.setup_vm_pt_regs = gfxhub_v1_0_setup_vm_pt_regs,
@@ -468,5 +451,4 @@ const struct amdgpu_gfxhub_funcs gfxhub_v1_0_funcs = {
 	.set_fault_enable_default = gfxhub_v1_0_set_fault_enable_default,
 	.init = gfxhub_v1_0_init,
 	.get_xgmi_info = gfxhub_v1_1_get_xgmi_info,
-	.query_utcl2_poison_status = gfxhub_v1_0_query_utcl2_poison_status,
 };
drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c +0 -17
index 72109abe7c86..ed8e130c7d19 100644
@@ -622,22 +622,6 @@ static int gfxhub_v1_2_get_xgmi_info(struct amdgpu_device *adev)
 	return 0;
 }
 
-static bool gfxhub_v1_2_query_utcl2_poison_status(struct amdgpu_device *adev,
-				int xcc_id)
-{
-	u32 fed, status;
-
-	status = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regVM_L2_PROTECTION_FAULT_STATUS);
-	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
-	if (!amdgpu_sriov_vf(adev)) {
-		/* clear page fault status and address */
-		WREG32_P(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id),
-			 regVM_L2_PROTECTION_FAULT_CNTL), 1, ~1);
-	}
-
-	return fed;
-}
-
 const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs = {
 	.get_mc_fb_offset = gfxhub_v1_2_get_mc_fb_offset,
 	.setup_vm_pt_regs = gfxhub_v1_2_setup_vm_pt_regs,
@@ -646,7 +630,6 @@ const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs = {
 	.set_fault_enable_default = gfxhub_v1_2_set_fault_enable_default,
 	.init = gfxhub_v1_2_init,
 	.get_xgmi_info = gfxhub_v1_2_get_xgmi_info,
-	.query_utcl2_poison_status = gfxhub_v1_2_query_utcl2_poison_status,
 };
 
 static int gfxhub_v1_2_xcp_resume(void *handle, uint32_t inst_mask)
drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c +0 -17
index 915203b91c5f..b01bb759d0f4 100644
@@ -559,22 +559,6 @@ static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags)
 
 }
 
-static bool mmhub_v1_8_query_utcl2_poison_status(struct amdgpu_device *adev,
-				int hub_inst)
-{
-	u32 fed, status;
-
-	status = RREG32_SOC15(MMHUB, hub_inst, regVM_L2_PROTECTION_FAULT_STATUS);
-	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
-	if (!amdgpu_sriov_vf(adev)) {
-		/* clear page fault status and address */
-		WREG32_P(SOC15_REG_OFFSET(MMHUB, hub_inst,
-			 regVM_L2_PROTECTION_FAULT_CNTL), 1, ~1);
-	}
-
-	return fed;
-}
-
 const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = {
 	.get_fb_location = mmhub_v1_8_get_fb_location,
 	.init = mmhub_v1_8_init,
@@ -584,7 +568,6 @@ const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = {
 	.setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs,
 	.set_clockgating = mmhub_v1_8_set_clockgating,
 	.get_clockgating = mmhub_v1_8_get_clockgating,
-	.query_utcl2_poison_status = mmhub_v1_8_query_utcl2_poison_status,
 };
 
 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = {
Signature · 1 line
2.17.1
#3 lore raw
In reply to Hawking Zhang

[AMD Official Use Only - AMD Internal Distribution Only]

Quoted 27 lines

-----Original Message-----
From: Hawking Zhang <[email protected]>
Sent: Monday, August 19, 2024 11:15 PM
To: [email protected]; Zhou1, Tao <[email protected]>; Yang, Stanley <[email protected]>
Cc: Zhang, Hawking <[email protected]>; Fan, Shikang <[email protected]>
Subject: [PATCH 1/3] drm/amdkfd: Check int source id for utcl2 poison event

Traditional utcl2 fault_status polling does not work in SRIOV environment. The polling of fault status register from guest side will be dropped by hardware.

Driver should switch to check utcl2 interrupt source id to identify utcl2 poison event. It is set to 1 when poisoned data interrupts are signaled.

---
 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 3 +--
 drivers/gpu/drm/amd/amdkfd/soc15_int.h          | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +0 -0
index a9c3580be8c9..1196dccbe6bc 100644

drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c static void event_interrupt_wq_v9(struct kfd_node *dev, · line 452

Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +0 -2
@@ -448,8 +448,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
              if (vmid_type && client_id == SOC15_IH_CLIENTID_VMC)
                      hub_inst = node_id / 4;
 
-             if (amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev,
-                                     hub_inst, vmid_type)) {

[Tao] the node_id, vmid_type and hub_inst can be also dropped.

Quoted 20 lines
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +1 -0
+             if (source_id == SOC15_INTSRC_VMC_UTCL2_POISON) {
                      event_interrupt_poison_consumption_v9(dev, pasid,
 client_id);
                      return;
              }
drivers/gpu/drm/amd/amdkfd/soc15_int.h +1 -1
index 10138676f27f..e5c0205f2618 100644
@@ -29,6 +29,7 @@
 #define SOC15_INTSRC_CP_BAD_OPCODE   183
 #define SOC15_INTSRC_SQ_INTERRUPT_MSG        239
 #define SOC15_INTSRC_VMC_FAULT               0
+#define SOC15_INTSRC_VMC_UTCL2_POISON        1
 #define SOC15_INTSRC_SDMA_TRAP               224
 #define SOC15_INTSRC_SDMA_ECC                220
 #define SOC21_INTSRC_SDMA_TRAP               49
--

2.17.1