Skip to contentSkip to conversation

tests/intel/xe_exec_reset: add GT reset fault injection stress coverage

PATCH In review [email protected] 2 participants 2 messages Sep 1

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 tests/intel/xe_exec_reset: add GT reset fault injection stress coverage

From: Nishit Sharma <[email protected]>

Inject a GT reset failure via the KMD fail_gt_reset debugfs hook, verify the device wedges as expected, tolerate the resulting -ECANCELED errors in the submitting threads, then recover by rebind and confirm the driver is usable again.

---
 tests/intel/xe_exec_reset.c | 123 ++++++++++++++++++++++++++++++++++--
 1 file changed, 118 insertions(+), 5 deletions(-)
tests/intel/xe_exec_reset.c +118 -5
index 6eda71c32..f3226579f 100644
@@ -16,6 +16,8 @@
 
 #include "igt.h"
 #include "igt_sysfs.h"
+#include "igt_device.h"
+#include "igt_kmod.h"
 #include "lib/igt_syncobj.h"
 #include "lib/intel_reg.h"
 #include "xe_drm.h"
@@ -140,6 +142,7 @@ static void test_spin(int fd, struct drm_xe_engine_class_instance *eci,
 #define DESTROY_VM_CTX_STRESS		(0x1 << 20)
 #define MIXED_ENGINE_STRESS		(0x1 << 21)
 #define PM_TRANSITION_STRESS		(0x1 << 22)
+#define FAULT_INJECT_STRESS		(0x1 << 23)
 
 /**
  * SUBTEST: %s-cat-error
@@ -760,7 +763,7 @@ static void submit_jobs(struct gt_thread_data *t)
 	uint32_t pressure_bos[PRESSURE_COUNT];
 	uint32_t *data;
 	int pressure_count;
-	int i = 0;
+	int i = 0, exec_ret;
 
 	bo = xe_bo_create(fd, vm, bo_size, vram_if_possible(fd, t->gt),
 			  DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM);
@@ -794,8 +797,25 @@ static void submit_jobs(struct gt_thread_data *t)
 			continue;
 		}
 
-		xe_exec(fd, &exec);
-		xe_exec_queue_destroy(fd, exec.exec_queue_id);
+		/*
+		 * Once an injected GT reset failure wedges the device, exec and
+		 * queue teardown return -ECANCELED. That is the expected outcome
+		 * for the fault-injection stress
+		 */
+		if (t->flags & FAULT_INJECT_STRESS) {
+			struct drm_xe_exec_queue_destroy destroy = {
+				.exec_queue_id = exec.exec_queue_id,
+		};
+
+		exec_ret = __xe_exec(fd, &exec);
+		igt_assert_f(exec_ret == 0 || exec_ret == -ECANCELED,
+			     "exec returned unexpected error %d (expected 0 or -ECANCELED)\n",
+			     exec_ret);
+		igt_ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_DESTROY, &destroy);
+		} else {
+			xe_exec(fd, &exec);
+			xe_exec_queue_destroy(fd, exec.exec_queue_id);
+		}
 		(*t->num_submit)++;
 
 		if ((t->flags & MEM_PRESSURE_STRESS) && !(i % 128)) {
@@ -829,8 +849,20 @@ static void submit_jobs(struct gt_thread_data *t)
 		pressure_bo_destroy(fd, pressure_bos, PRESSURE_COUNT);
 
 	munmap(data, bo_size);
-	gem_close(fd, bo);
-	xe_vm_destroy(fd, vm);
+	/*
+	 * On a device wedged by injected GT reset failures, BO close and VM
+	 * destroy also return -ECANCELED.
+	 */
+	if (t->flags & FAULT_INJECT_STRESS) {
+		struct drm_gem_close close_bo = { .handle = bo };
+		struct drm_xe_vm_destroy vm_destroy = { .vm_id = vm };
+
+		igt_ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close_bo);
+		igt_ioctl(fd, DRM_IOCTL_XE_VM_DESTROY, &vm_destroy);
+	} else {
+		gem_close(fd, bo);
+		xe_vm_destroy(fd, vm);
+	}
 }
 
 static void *gt_reset_thread(void *data)
@@ -850,6 +882,54 @@ static void *gt_reset_thread(void *data)
 	return NULL;
 }
 
+static int fault_inject_fd = -1;
+
+static void gt_reset_disable_fault_injection(int sig)
+{
+	if (fault_inject_fd < 0)
+		return;
+
+	igt_debugfs_write(fault_inject_fd, "fail_gt_reset/probability", "0");
+	igt_debugfs_write(fault_inject_fd, "fail_gt_reset/times", "1");
+	fault_inject_fd = -1;
+}
+
+static void gt_reset_enable_fault_injection(int fd)
+{
+	static bool exit_handler_installed;
+
+	fault_inject_fd = fd;
+
+	if (!exit_handler_installed) {
+		igt_install_exit_handler(gt_reset_disable_fault_injection);
+		exit_handler_installed = true;
+	}
+
+	igt_debugfs_write(fd, "fail_gt_reset/probability", "100");
+	igt_debugfs_write(fd, "fail_gt_reset/times", "2");
+}
+
+static int try_vm_create(int fd)
+{
+	struct drm_xe_vm_create create = { 0 };
+	int err = 0;
+
+	if (igt_ioctl(fd, DRM_IOCTL_XE_VM_CREATE, &create))
+		err = -errno;
+	else
+		xe_vm_destroy(fd, create.vm_id);
+
+	return err;
+}
+
+static void ignore_gt_reset_fault_dmesg(void)
+{
+	igt_emit_ignore_dmesg_regex("reset failed \\(-ECANCELED\\)"
+				    "|declared device .* as wedged"
+				    "|GPU HANG"
+				    "|Failed to reset");
+}
+
 /**
  * SUBTEST: gt-reset-stress
  * Description: Stress GT reset
@@ -891,6 +971,9 @@ static void *gt_reset_thread(void *data)
  * Description: Test GT reset while long spinner workload is active
  * Test category: stress test
  *
+ * SUBTEST: gt-reset-fault-injection
+ * Description: Stress concurrent GT resets and job submissions with GT reset failures injected via debugfs
+ * Test category: fault injection
  */
 static void
 gt_reset(int fd, int gt, int n_threads, int n_sec, unsigned int flags)
@@ -907,6 +990,9 @@ gt_reset(int fd, int gt, int n_threads, int n_sec, unsigned int flags)
 	pthread_mutex_init(&mutex, 0);
 	pthread_cond_init(&cond, 0);
 
+	if (flags & FAULT_INJECT_STRESS)
+		gt_reset_enable_fault_injection(fd);
+
 	for (i = 0; i < n_threads; ++i) {
 		threads[i].mutex = &mutex;
 		threads[i].cond = &cond;
@@ -940,6 +1026,9 @@ gt_reset(int fd, int gt, int n_threads, int n_sec, unsigned int flags)
 	igt_info("number of resets %d, submissions %d, submit fails %d vm_recreate %d\n",
 		 num_reset, num_submit, num_submit_fail, num_vm_recreate);
 
+	if (flags & FAULT_INJECT_STRESS)
+		gt_reset_disable_fault_injection(0);
+
 	igt_assert_neq(num_reset, 0);
 	igt_assert_neq(num_submit, 0);
 	free(threads);
@@ -1122,6 +1211,7 @@ int igt_main()
 	int gt;
 	int class;
 	int fd;
+	char pci_slot[NAME_MAX];
 
 	igt_fixture()
 		fd = drm_open_driver(DRIVER_XE);
@@ -1326,6 +1416,29 @@ int igt_main()
 			break;
 		}
 
+	igt_subtest("gt-reset-fault-injection") {
+		igt_require_f(igt_debugfs_exists(fd, "fail_gt_reset/probability",
+						 O_RDWR),
+			      "GT reset fault injection not available; "
+			      "CONFIG_DRM_XE_KUNIT_TEST/fault-injection must be "
+			      "enabled in the KMD\n");
+
+		igt_device_get_pci_slot_name(fd, pci_slot);
+		ignore_gt_reset_fault_dmesg();
+
+		gt_reset(fd, 0, 8, 2, FAULT_INJECT_STRESS);
+
+		igt_assert_f(try_vm_create(fd) != 0,
+			     "Device did not wedge after injected GT reset failure\n");
+
+		drm_close_driver(fd);
+		igt_kmod_rebind("xe", pci_slot);
+		fd = drm_open_driver(DRIVER_XE);
+
+		igt_assert_f(try_vm_create(fd) == 0,
+			     "Device not functional after rebind recovery\n");
+	}
+
 	igt_subtest("gt-mocs-reset")
 		xe_for_each_gt(fd, gt)
 			gt_mocs_reset(fd, gt);
Signature · 1 line
2.43.0
#1 lore raw

Hi Nishit,

Quoted 17 lines · On 2026-09-01 at 03:44:03 +0000, [email protected] wrote:

From: Nishit Sharma <[email protected]>

Inject a GT reset failure via the KMD fail_gt_reset debugfs hook, verify the device wedges as expected, tolerate the resulting -ECANCELED errors in the submitting threads, then recover by rebind and confirm the driver is usable again.

---
 tests/intel/xe_exec_reset.c | 123 ++++++++++++++++++++++++++++++++++--
 1 file changed, 118 insertions(+), 5 deletions(-)
tests/intel/xe_exec_reset.c +0 -0
index 6eda71c32..f3226579f 100644

tests/intel/xe_exec_reset.c line 20

Quoted 5 lines
tests/intel/xe_exec_reset.c +2 -0
@@ -16,6 +16,8 @@
 
 #include "igt.h"
 #include "igt_sysfs.h"
+#include "igt_device.h"
+#include "igt_kmod.h"

Move these up before igt_sysfs.h, keep it sorted.

Quoted 76 lines
tests/intel/xe_exec_reset.c +36 -5
 #include "lib/igt_syncobj.h"
 #include "lib/intel_reg.h"
 #include "xe_drm.h"
@@ -140,6 +142,7 @@ static void test_spin(int fd, struct drm_xe_engine_class_instance *eci,
 #define DESTROY_VM_CTX_STRESS		(0x1 << 20)
 #define MIXED_ENGINE_STRESS		(0x1 << 21)
 #define PM_TRANSITION_STRESS		(0x1 << 22)
+#define FAULT_INJECT_STRESS		(0x1 << 23)
 
 /**
  * SUBTEST: %s-cat-error
@@ -760,7 +763,7 @@ static void submit_jobs(struct gt_thread_data *t)
 	uint32_t pressure_bos[PRESSURE_COUNT];
 	uint32_t *data;
 	int pressure_count;
-	int i = 0;
+	int i = 0, exec_ret;
 
 	bo = xe_bo_create(fd, vm, bo_size, vram_if_possible(fd, t->gt),
 			  DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM);
@@ -794,8 +797,25 @@ static void submit_jobs(struct gt_thread_data *t)
 			continue;
 		}
 
-		xe_exec(fd, &exec);
-		xe_exec_queue_destroy(fd, exec.exec_queue_id);
+		/*
+		 * Once an injected GT reset failure wedges the device, exec and
+		 * queue teardown return -ECANCELED. That is the expected outcome
+		 * for the fault-injection stress
+		 */
+		if (t->flags & FAULT_INJECT_STRESS) {
+			struct drm_xe_exec_queue_destroy destroy = {
+				.exec_queue_id = exec.exec_queue_id,
+		};
+
+		exec_ret = __xe_exec(fd, &exec);
+		igt_assert_f(exec_ret == 0 || exec_ret == -ECANCELED,
+			     "exec returned unexpected error %d (expected 0 or -ECANCELED)\n",
+			     exec_ret);
+		igt_ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_DESTROY, &destroy);
+		} else {
+			xe_exec(fd, &exec);
+			xe_exec_queue_destroy(fd, exec.exec_queue_id);
+		}
 		(*t->num_submit)++;
 
 		if ((t->flags & MEM_PRESSURE_STRESS) && !(i % 128)) {
@@ -829,8 +849,20 @@ static void submit_jobs(struct gt_thread_data *t)
 		pressure_bo_destroy(fd, pressure_bos, PRESSURE_COUNT);
 
 	munmap(data, bo_size);
-	gem_close(fd, bo);
-	xe_vm_destroy(fd, vm);
+	/*
+	 * On a device wedged by injected GT reset failures, BO close and VM
+	 * destroy also return -ECANCELED.
+	 */
+	if (t->flags & FAULT_INJECT_STRESS) {
+		struct drm_gem_close close_bo = { .handle = bo };
+		struct drm_xe_vm_destroy vm_destroy = { .vm_id = vm };
+
+		igt_ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close_bo);
+		igt_ioctl(fd, DRM_IOCTL_XE_VM_DESTROY, &vm_destroy);
+	} else {
+		gem_close(fd, bo);
+		xe_vm_destroy(fd, vm);
+	}
 }
 
 static void *gt_reset_thread(void *data)
@@ -850,6 +882,54 @@ static void *gt_reset_thread(void *data)
 	return NULL;
 }
 
+static int fault_inject_fd = -1;

tests/intel/xe_exec_reset.c static void *gt_reset_thread(void *data) · line 895

Quoted 10 lines
tests/intel/xe_exec_reset.c +10 -0
+
+static void gt_reset_disable_fault_injection(int sig)
+{
+	if (fault_inject_fd < 0)
+		return;
+
+	igt_debugfs_write(fault_inject_fd, "fail_gt_reset/probability", "0");
+	igt_debugfs_write(fault_inject_fd, "fail_gt_reset/times", "1");
+	fault_inject_fd = -1;
+}

Now this looks ok but see a note at a test.

Quoted 88 lines
tests/intel/xe_exec_reset.c +53 -0
+
+static void gt_reset_enable_fault_injection(int fd)
+{
+	static bool exit_handler_installed;
+
+	fault_inject_fd = fd;
+
+	if (!exit_handler_installed) {
+		igt_install_exit_handler(gt_reset_disable_fault_injection);
+		exit_handler_installed = true;
+	}
+
+	igt_debugfs_write(fd, "fail_gt_reset/probability", "100");
+	igt_debugfs_write(fd, "fail_gt_reset/times", "2");
+}
+
+static int try_vm_create(int fd)
+{
+	struct drm_xe_vm_create create = { 0 };
+	int err = 0;
+
+	if (igt_ioctl(fd, DRM_IOCTL_XE_VM_CREATE, &create))
+		err = -errno;
+	else
+		xe_vm_destroy(fd, create.vm_id);
+
+	return err;
+}
+
+static void ignore_gt_reset_fault_dmesg(void)
+{
+	igt_emit_ignore_dmesg_regex("reset failed \\(-ECANCELED\\)"
+				    "|declared device .* as wedged"
+				    "|GPU HANG"
+				    "|Failed to reset");
+}
+
 /**
  * SUBTEST: gt-reset-stress
  * Description: Stress GT reset
@@ -891,6 +971,9 @@ static void *gt_reset_thread(void *data)
  * Description: Test GT reset while long spinner workload is active
  * Test category: stress test
  *
+ * SUBTEST: gt-reset-fault-injection
+ * Description: Stress concurrent GT resets and job submissions with GT reset failures injected via debugfs
+ * Test category: fault injection
  */
 static void
 gt_reset(int fd, int gt, int n_threads, int n_sec, unsigned int flags)
@@ -907,6 +990,9 @@ gt_reset(int fd, int gt, int n_threads, int n_sec, unsigned int flags)
 	pthread_mutex_init(&mutex, 0);
 	pthread_cond_init(&cond, 0);
 
+	if (flags & FAULT_INJECT_STRESS)
+		gt_reset_enable_fault_injection(fd);
+
 	for (i = 0; i < n_threads; ++i) {
 		threads[i].mutex = &mutex;
 		threads[i].cond = &cond;
@@ -940,6 +1026,9 @@ gt_reset(int fd, int gt, int n_threads, int n_sec, unsigned int flags)
 	igt_info("number of resets %d, submissions %d, submit fails %d vm_recreate %d\n",
 		 num_reset, num_submit, num_submit_fail, num_vm_recreate);
 
+	if (flags & FAULT_INJECT_STRESS)
+		gt_reset_disable_fault_injection(0);
+
 	igt_assert_neq(num_reset, 0);
 	igt_assert_neq(num_submit, 0);
 	free(threads);
@@ -1122,6 +1211,7 @@ int igt_main()
 	int gt;
 	int class;
 	int fd;
+	char pci_slot[NAME_MAX];
 
 	igt_fixture()
 		fd = drm_open_driver(DRIVER_XE);
@@ -1326,6 +1416,29 @@ int igt_main()
 			break;
 		}
 
+	igt_subtest("gt-reset-fault-injection") {
+		igt_require_f(igt_debugfs_exists(fd, "fail_gt_reset/probability",
+						 O_RDWR),
+			      "GT reset fault injection not available; "
+			      "CONFIG_DRM_XE_KUNIT_TEST/fault-injection must be "
+			      "enabled in the KMD\n");

tests/intel/xe_exec_reset.c int igt_main() · line 1434

Quoted 10 lines
tests/intel/xe_exec_reset.c +10 -0
+
+		igt_device_get_pci_slot_name(fd, pci_slot);
+		ignore_gt_reset_fault_dmesg();
+
+		gt_reset(fd, 0, 8, 2, FAULT_INJECT_STRESS);
+
+		igt_assert_f(try_vm_create(fd) != 0,
+			     "Device did not wedge after injected GT reset failure\n");
+
+		drm_close_driver(fd);

Now fault_inject_fd becomes stale.

tests/intel/xe_exec_reset.c int igt_main() · line 1435

Quoted 1 line
tests/intel/xe_exec_reset.c +1 -0
+		igt_kmod_rebind("xe", pci_slot);

After rebind there is no point in restoring params, or do I miss something?

Regards,
Kamil

Quoted 12 lines
tests/intel/xe_exec_reset.c +6 -0
+		fd = drm_open_driver(DRIVER_XE);
+
+		igt_assert_f(try_vm_create(fd) == 0,
+			     "Device not functional after rebind recovery\n");
+	}
+
 	igt_subtest("gt-mocs-reset")
 		xe_for_each_gt(fd, gt)
 			gt_mocs_reset(fd, gt);
Signature · 1 line
2.43.0