Skip to contentSkip to conversation

drm/amdkfd: Add GPU reset SMI event

PATCH Superseded Mukul Joshi 2 participants 2 messages Aug 26, 2020 – Aug 27, 2020

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 drm/amdkfd: Add GPU reset SMI event

Add support for reporting GPU reset events through SMI. KFD would report both pre and post GPU reset events.

---
 drivers/gpu/drm/amd/amdkfd/kfd_device.c     |  4 +++
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h       |  2 ++
 drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c | 30 +++++++++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h |  1 +
 include/uapi/linux/kfd_ioctl.h              |  2 ++
 5 files changed, 39 insertions(+)
drivers/gpu/drm/amd/amdkfd/kfd_device.c +4 -0
index e1cd6599529f..aad1ecfa1239 100644
@@ -812,6 +812,8 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
 	if (!kfd->init_complete)
 		return 0;
 
+	kfd_smi_event_update_gpu_reset(kfd, false);
+
 	kfd->dqm->ops.pre_reset(kfd->dqm);
 
 	kgd2kfd_suspend(kfd, false);
@@ -833,6 +835,8 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
 	if (!kfd->init_complete)
 		return 0;
 
+	kfd_smi_event_update_gpu_reset(kfd, true);
+
 	ret = kfd_resume(kfd);
 	if (ret)
 		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -0
index 18bc711f97ae..b1a2979e086f 100644
@@ -312,6 +312,8 @@ struct kfd_dev {
 	/* Clients watching SMI events */
 	struct list_head smi_clients;
 	spinlock_t smi_lock;
+
+	uint64_t reset_seq_num;
 };
 
 enum kfd_mempool {
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +30 -0
index 4d4b6e3ab697..4f0590bcb1a3 100644
@@ -174,6 +174,36 @@ static void add_event_to_kfifo(struct kfd_dev *dev, unsigned int smi_event,
 	rcu_read_unlock();
 }
 
+void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset)
+{
+	/*
+	 * GpuReset msg = Reset seq number (incremented for
+	 * every reset message sent before GPU reset).
+	 * 1 byte event + 1 byte space + 16 bytes seq num +
+	 * 1 byte \n + 1 byte \0 = 20
+	 */
+	char fifo_in[20];
+	int len;
+	unsigned int event;
+
+	if (list_empty(&dev->smi_clients)) {
+		return;
+	}
+
+	memset(fifo_in, 0x0, sizeof(fifo_in));
+
+	if (post_reset) {
+		event = KFD_SMI_EVENT_GPU_POST_RESET;
+	} else {
+		event = KFD_SMI_EVENT_GPU_PRE_RESET;
+		++(dev->reset_seq_num);
+	}
+
+	len = snprintf(fifo_in, 20, "%x %llx\n", event, dev->reset_seq_num);
+
+	add_event_to_kfifo(dev, event, fifo_in, len);
+}
+
 void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
 					     uint32_t throttle_bitmask)
 {
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h +1 -0
index 15537b2cccb5..b9b0438202e2 100644
@@ -27,5 +27,6 @@ int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd);
 void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid);
 void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
 					     uint32_t throttle_bitmask);
+void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset);
 
 #endif
include/uapi/linux/kfd_ioctl.h +2 -0
index cb1f963a84e0..8b7368bfbd84 100644
@@ -453,6 +453,8 @@ enum kfd_smi_event {
         KFD_SMI_EVENT_NONE = 0, /* not used */
         KFD_SMI_EVENT_VMFAULT = 1, /* event start counting at 1 */
         KFD_SMI_EVENT_THERMAL_THROTTLE = 2,
+	KFD_SMI_EVENT_GPU_PRE_RESET = 3,
+	KFD_SMI_EVENT_GPU_POST_RESET = 4,
 };
 
 #define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1))
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
maintainer
#1 lore raw

Am 2020-08-26 um 4:01 p.m. schrieb Mukul Joshi:

Quoted 26 lines

Add support for reporting GPU reset events through SMI. KFD would report both pre and post GPU reset events.

---
 drivers/gpu/drm/amd/amdkfd/kfd_device.c     |  4 +++
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h       |  2 ++
 drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c | 30 +++++++++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h |  1 +
 include/uapi/linux/kfd_ioctl.h              |  2 ++
 5 files changed, 39 insertions(+)
drivers/gpu/drm/amd/amdkfd/kfd_device.c +2 -0
index e1cd6599529f..aad1ecfa1239 100644
@@ -812,6 +812,8 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
 	if (!kfd->init_complete)
 		return 0;
 
+	kfd_smi_event_update_gpu_reset(kfd, false);
+
 	kfd->dqm->ops.pre_reset(kfd->dqm);
 
 	kgd2kfd_suspend(kfd, false);

drivers/gpu/drm/amd/amdkfd/kfd_device.c int kgd2kfd_post_reset(struct kfd_dev *kfd) · line 839

Quoted 5 lines
drivers/gpu/drm/amd/amdkfd/kfd_device.c +2 -0
@@ -833,6 +835,8 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
 	if (!kfd->init_complete)
 		return 0;
 
+	kfd_smi_event_update_gpu_reset(kfd, true);
+

You should generate the message only after successfully resuming KFD.

drivers/gpu/drm/amd/amdkfd/kfd_priv.h struct kfd_dev { · line 316

Quoted 13 lines
drivers/gpu/drm/amd/amdkfd/kfd_device.c +0 -0
 	ret = kfd_resume(kfd);
 	if (ret)
 		return ret;
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -0
index 18bc711f97ae..b1a2979e086f 100644
@@ -312,6 +312,8 @@ struct kfd_dev {
 	/* Clients watching SMI events */
 	struct list_head smi_clients;
 	spinlock_t smi_lock;
+
+	uint64_t reset_seq_num;

64-bit may be overkill. If you're resetting your GPU so frequently that a 32-bit counter overflows, nobody wants to buy that GPU. ;)

Quoted 24 lines
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +0 -0
 };
 
 enum kfd_mempool {
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +13 -0
index 4d4b6e3ab697..4f0590bcb1a3 100644
@@ -174,6 +174,36 @@ static void add_event_to_kfifo(struct kfd_dev *dev, unsigned int smi_event,
 	rcu_read_unlock();
 }
 
+void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset)
+{
+	/*
+	 * GpuReset msg = Reset seq number (incremented for
+	 * every reset message sent before GPU reset).
+	 * 1 byte event + 1 byte space + 16 bytes seq num +
+	 * 1 byte \n + 1 byte \0 = 20
+	 */
+	char fifo_in[20];
+	int len;
+	unsigned int event;
+
+	if (list_empty(&dev->smi_clients)) {

drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c static void add_event_to_kfifo(struct kfd_dev *dev, unsigned int smi_event, · line 199

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +10 -0
+		return;
+	}
+
+	memset(fifo_in, 0x0, sizeof(fifo_in));
+
+	if (post_reset) {
+		event = KFD_SMI_EVENT_GPU_POST_RESET;
+	} else {
+		event = KFD_SMI_EVENT_GPU_PRE_RESET;
+		++(dev->reset_seq_num);

This sequence number is per-device. When multiple GPUs in an XGMI hive get reset together, each GPU would get its own uncorrelated reset sequence number. It may happen to be the same number, but I don't think it can be guaranteed.

On the other hand, when two non-XGMI GPUs are reset at different times, they could get the same sequence number, although the two resets were caused by different unrelated events, and happened at different times.

It would be better to have a global reset sequence number that can be used to correlate GPU resets that happen to multiple GPUs (in an XGMI hive) at the same time, caused by the same event. The easiest way to achieve this would be, to use a global reset_seq_num and increment it only in kgd2kfd_pre_reset if atomic_read(&kfd_locked) is 0 (i.e. on the first GPU that's getting reset at the same time).

drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c static void add_event_to_kfifo(struct kfd_dev *dev, unsigned int smi_event, · line 202

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +3 -0
+	}
+
+	len = snprintf(fifo_in, 20, "%x %llx\n", event, dev->reset_seq_num);

A safer, more maintainable way would be to write "sizeof(fifo_in)" instead of hard-coding 20.

Regards,
  Felix

Quoted 23 lines
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +4 -0
+
+	add_event_to_kfifo(dev, event, fifo_in, len);
+}
+
 void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
 					     uint32_t throttle_bitmask)
 {
drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h +1 -0
index 15537b2cccb5..b9b0438202e2 100644
@@ -27,5 +27,6 @@ int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd);
 void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid);
 void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
 					     uint32_t throttle_bitmask);
+void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset);
 
 #endif
include/uapi/linux/kfd_ioctl.h +0 -0
index cb1f963a84e0..8b7368bfbd84 100644

include/uapi/linux/kfd_ioctl.h enum kfd_smi_event { · line 460

Quoted 8 lines
include/uapi/linux/kfd_ioctl.h +2 -0
@@ -453,6 +453,8 @@ enum kfd_smi_event {
         KFD_SMI_EVENT_NONE = 0, /* not used */
         KFD_SMI_EVENT_VMFAULT = 1, /* event start counting at 1 */
         KFD_SMI_EVENT_THERMAL_THROTTLE = 2,
+	KFD_SMI_EVENT_GPU_PRE_RESET = 3,
+	KFD_SMI_EVENT_GPU_POST_RESET = 4,
 };
 
 #define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1))

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx