Skip to contentSkip to conversation

tests/core_hotunplug: add *-with-load subtests exercising GPU workload during hotplug

PATCH In review Nitin Gote 2 participants 2 messages May 18

lore
Share & tools
lore

Nitin Gote

author
#0 lore raw

PATCH 1/1 tests/core_hotunplug: add *-with-load subtests exercising GPU workload during hotplug

The six fd-holding hot* subtests keep a DRM fd open across unbind/unplug but leave the GPU idle, so the kernel hotplug path is never exercised against an active VM, exec queue and BOs.

Add six new dedicated *-with-load subtests that run a GPU spinner across the unbind/unplug sequence so the device is removed while a workload is actively using it.

The workload helpers use an explicit chipset dispatch block in workload_start()/stop() so that other vendors can contribute support by adding an else-if branch. New subtests are gated on workload_available(), which currently covers DRIVER_XE and DRIVER_INTEL, and returns false (SKIP) on any other driver.

igt_spin_free() is avoided in workload_stop() since the device may be gone by then — igt_spin_end() writes the stop condition via a userspace mmap and lets the DRM fd close reclaim kernel-side resources.

Validated on BMG and DG2 with a KASAN-enabled kernel:
all six new subtests pass with no KASAN reports.

v2: Pass the hotunplug priv struct to workload_start()/stop()
and dispatch via an explicit if/else-if chipset block to invite other vendors to contribute. (Janusz)

Cc Matthew Auld <[email protected]>
Cc Janusz Krzysztofik <[email protected]>
Cc Kamil Konieczny <[email protected]>
Assisted-by Copilot:claude-opus-4.7
---
 tests/core_hotunplug.c | 290 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 290 insertions(+)
tests/core_hotunplug.c +290 -0
index 7c9dae1bf..38dcbb97d 100644
@@ -80,6 +80,34 @@
  * SUBTEST: unplug-rescan
  * Description: Check if a device believed to be closed can be cleanly
  *		unplugged, then restored
+ *
+ * SUBTEST: hotunbind-rebind-with-load
+ * Description: Check if the driver can be cleanly unbound from an open device
+ *		with a background GPU workload in flight, then released and
+ *		rebound
+ *
+ * SUBTEST: hotunplug-rescan-with-load
+ * Description: Check if an open device with a background GPU workload in
+ *		flight can be cleanly unplugged, then released and restored
+ *
+ * SUBTEST: hotrebind-with-load
+ * Description: Check if the driver can be cleanly rebound to a device with a
+ *		still open hot unbound driver instance while a background GPU
+ *		workload is in flight
+ *
+ * SUBTEST: hotreplug-with-load
+ * Description: Check if a hot unplugged and still open device can be cleanly
+ *		restored while a background GPU workload is in flight
+ *
+ * SUBTEST: hotrebind-lateclose-with-load
+ * Description: Check if a hot unbound driver instance still open after hot
+ *		rebind with a background GPU workload in flight can be cleanly
+ *		released
+ *
+ * SUBTEST: hotreplug-lateclose-with-load
+ * Description: Check if an instance of a still open while hot replugged
+ *		device with a background GPU workload in flight can be cleanly
+ *		released
  */
 
 IGT_TEST_DESCRIPTION("Examine behavior of a driver on device hot unplug");
@@ -565,6 +593,52 @@ static void set_filter_from_device(int fd)
 	igt_assert_eq(igt_device_filter_add(filter), 1);
 }
 
+/* GPU workload helpers */
+
+struct gpu_workload {
+	igt_spin_t *spin;
+	uint64_t ahnd;
+};
+
+static bool workload_available(int chipset)
+{
+	return chipset == DRIVER_XE || chipset == DRIVER_INTEL;
+}
+
+static void workload_start(struct hotunplug *priv, int fd,
+			   struct gpu_workload *w)
+{
+	if (priv->chipset == DRIVER_XE || priv->chipset == DRIVER_INTEL) {
+		local_debug("%s\n", "starting GPU spinner");
+		priv->failure = "GPU workload start failure!";
+		w->ahnd = intel_allocator_open(fd, 0, INTEL_ALLOCATOR_RELOC);
+		w->spin = igt_spin_new(fd, .ahnd = w->ahnd);
+		priv->failure = NULL;
+	}
+}
+
+static void workload_stop(struct hotunplug *priv, struct gpu_workload *w)
+{
+	if (priv->chipset == DRIVER_XE || priv->chipset == DRIVER_INTEL) {
+		if (!w->spin)
+			return;
+
+		/*
+		 * The device may be gone (unbind/unplug), so avoid
+		 * igt_spin_free(): on Xe it would wait forever on a syncobj
+		 * that never signals; on i915 it would touch the dead fd via
+		 * gem_munmap()/gem_close(). Instead, end the spinner via a
+		 * userspace mmap write and let the DRM fd close reclaim the
+		 * kernel-side resources.
+		 */
+		local_debug("%s\n", "stopping GPU spinner");
+		igt_spin_end(w->spin);
+		put_ahnd(w->ahnd);
+		w->spin = NULL;
+		w->ahnd = 0;
+	}
+}
+
 /* Subtests */
 
 static void unbind_rebind(struct hotunplug *priv)
@@ -679,6 +753,150 @@ static void hotreplug_lateclose(struct hotunplug *priv)
 	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
 }
 
+static void hotunbind_rebind_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot unbind");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	driver_unbind(priv, "hot ", 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "unbound ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	driver_bind(priv, 0);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotunplug_rescan_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot unplug");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	device_unplug(priv, "hot ", 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "removed ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	bus_rescan(priv, 0);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotrebind_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot rebind");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	driver_unbind(priv, "hot ", 60);
+
+	driver_bind(priv, 0);
+
+	workload_stop(priv, &w);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotreplug_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot replug");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	device_unplug(priv, "hot ", 60);
+
+	bus_rescan(priv, 0);
+
+	workload_stop(priv, &w);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotrebind_lateclose_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot rebind");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	driver_unbind(priv, "hot ", 60);
+
+	driver_bind(priv, 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "unbound ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotreplug_lateclose_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot replug");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	device_unplug(priv, "hot ", 60);
+
+	bus_rescan(priv, 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "removed ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
 /* Main */
 
 int igt_main()
@@ -817,6 +1035,78 @@ int igt_main()
 			recover(&priv);
 	}
 
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if the driver can be cleanly unbound from an open device with a background GPU workload in flight, then released and rebound");
+		igt_subtest("hotunbind-rebind-with-load")
+			hotunbind_rebind_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if an open device with a background GPU workload in flight can be cleanly unplugged, then released and restored");
+		igt_subtest("hotunplug-rescan-with-load")
+			hotunplug_rescan_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if the driver can be cleanly rebound to a device with a still open hot unbound driver instance while a background GPU workload is in flight");
+		igt_subtest("hotrebind-with-load")
+			hotrebind_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if a hot unplugged and still open device can be cleanly restored while a background GPU workload is in flight");
+		igt_subtest("hotreplug-with-load")
+			hotreplug_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if a hot unbound driver instance still open after hot rebind with a background GPU workload in flight can be cleanly released");
+		igt_subtest("hotrebind-lateclose-with-load")
+			hotrebind_lateclose_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if an instance of a still open while hot replugged device with a background GPU workload in flight can be cleanly released");
+		igt_subtest("hotreplug-lateclose-with-load")
+			hotreplug_lateclose_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
 	igt_fixture() {
 		post_healthcheck(&priv);
 
Signature · 1 line
2.50.1
#1 lore raw

Hi Nitin,

Quoted 74 lines · Nitin Gote

The six fd-holding hot* subtests keep a DRM fd open across unbind/unplug but leave the GPU idle, so the kernel hotplug path is never exercised against an active VM, exec queue and BOs.

Add six new dedicated *-with-load subtests that run a GPU spinner across the unbind/unplug sequence so the device is removed while a workload is actively using it.

The workload helpers use an explicit chipset dispatch block in workload_start()/stop() so that other vendors can contribute support by adding an else-if branch. New subtests are gated on workload_available(), which currently covers DRIVER_XE and DRIVER_INTEL, and returns false (SKIP) on any other driver.

igt_spin_free() is avoided in workload_stop() since the device may be gone by then — igt_spin_end() writes the stop condition via a userspace mmap and lets the DRM fd close reclaim kernel-side resources.

Validated on BMG and DG2 with a KASAN-enabled kernel:
all six new subtests pass with no KASAN reports.

v2: Pass the hotunplug priv struct to workload_start()/stop()
and dispatch via an explicit if/else-if chipset block to invite other vendors to contribute. (Janusz)

Cc Matthew Auld <[email protected]>
Cc Janusz Krzysztofik <[email protected]>
Cc Kamil Konieczny <[email protected]>
Assisted-by Copilot:claude-opus-4.7
---
 tests/core_hotunplug.c | 290 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 290 insertions(+)
tests/core_hotunplug.c +28 -0
index 7c9dae1bf..38dcbb97d 100644
@@ -80,6 +80,34 @@
  * SUBTEST: unplug-rescan
  * Description: Check if a device believed to be closed can be cleanly
  *		unplugged, then restored
+ *
+ * SUBTEST: hotunbind-rebind-with-load
+ * Description: Check if the driver can be cleanly unbound from an open device
+ *		with a background GPU workload in flight, then released and
+ *		rebound
+ *
+ * SUBTEST: hotunplug-rescan-with-load
+ * Description: Check if an open device with a background GPU workload in
+ *		flight can be cleanly unplugged, then released and restored
+ *
+ * SUBTEST: hotrebind-with-load
+ * Description: Check if the driver can be cleanly rebound to a device with a
+ *		still open hot unbound driver instance while a background GPU
+ *		workload is in flight
+ *
+ * SUBTEST: hotreplug-with-load
+ * Description: Check if a hot unplugged and still open device can be cleanly
+ *		restored while a background GPU workload is in flight
+ *
+ * SUBTEST: hotrebind-lateclose-with-load
+ * Description: Check if a hot unbound driver instance still open after hot
+ *		rebind with a background GPU workload in flight can be cleanly
+ *		released
+ *
+ * SUBTEST: hotreplug-lateclose-with-load
+ * Description: Check if an instance of a still open while hot replugged
+ *		device with a background GPU workload in flight can be cleanly
+ *		released
  */
 
 IGT_TEST_DESCRIPTION("Examine behavior of a driver on device hot unplug");

tests/core_hotunplug.c static void set_filter_from_device(int fd) · line 601

Quoted 9 lines
tests/core_hotunplug.c +6 -0
@@ -565,6 +593,52 @@ static void set_filter_from_device(int fd)
 	igt_assert_eq(igt_device_filter_add(filter), 1);
 }
 
+/* GPU workload helpers */
+
+struct gpu_workload {
+	igt_spin_t *spin;
+	uint64_t ahnd;
+};

Sorry for not touching this before when asking you to make room for other vendors in functions, but now as we have that in place, I think it would be good to have the same also for data. A union maybe, with its members named after vendors?

Thanks,
Janusz

Quoted 273 lines
tests/core_hotunplug.c +256 -0
+
+static bool workload_available(int chipset)
+{
+	return chipset == DRIVER_XE || chipset == DRIVER_INTEL;
+}
+
+static void workload_start(struct hotunplug *priv, int fd,
+			   struct gpu_workload *w)
+{
+	if (priv->chipset == DRIVER_XE || priv->chipset == DRIVER_INTEL) {
+		local_debug("%s\n", "starting GPU spinner");
+		priv->failure = "GPU workload start failure!";
+		w->ahnd = intel_allocator_open(fd, 0, INTEL_ALLOCATOR_RELOC);
+		w->spin = igt_spin_new(fd, .ahnd = w->ahnd);
+		priv->failure = NULL;
+	}
+}
+
+static void workload_stop(struct hotunplug *priv, struct gpu_workload *w)
+{
+	if (priv->chipset == DRIVER_XE || priv->chipset == DRIVER_INTEL) {
+		if (!w->spin)
+			return;
+
+		/*
+		 * The device may be gone (unbind/unplug), so avoid
+		 * igt_spin_free(): on Xe it would wait forever on a syncobj
+		 * that never signals; on i915 it would touch the dead fd via
+		 * gem_munmap()/gem_close(). Instead, end the spinner via a
+		 * userspace mmap write and let the DRM fd close reclaim the
+		 * kernel-side resources.
+		 */
+		local_debug("%s\n", "stopping GPU spinner");
+		igt_spin_end(w->spin);
+		put_ahnd(w->ahnd);
+		w->spin = NULL;
+		w->ahnd = 0;
+	}
+}
+
 /* Subtests */
 
 static void unbind_rebind(struct hotunplug *priv)
@@ -679,6 +753,150 @@ static void hotreplug_lateclose(struct hotunplug *priv)
 	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
 }
 
+static void hotunbind_rebind_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot unbind");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	driver_unbind(priv, "hot ", 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "unbound ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	driver_bind(priv, 0);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotunplug_rescan_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot unplug");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	device_unplug(priv, "hot ", 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "removed ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	bus_rescan(priv, 0);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotrebind_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot rebind");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	driver_unbind(priv, "hot ", 60);
+
+	driver_bind(priv, 0);
+
+	workload_stop(priv, &w);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotreplug_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot replug");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	device_unplug(priv, "hot ", 60);
+
+	bus_rescan(priv, 0);
+
+	workload_stop(priv, &w);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotrebind_lateclose_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot rebind");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	driver_unbind(priv, "hot ", 60);
+
+	driver_bind(priv, 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "unbound ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
+static void hotreplug_lateclose_with_load(struct hotunplug *priv)
+{
+	struct gpu_workload w = { 0 };
+
+	pre_check(priv);
+
+	igt_require_f(workload_available(priv->chipset),
+		      "No GPU workload support for this driver\n");
+
+	priv->fd.drm = local_drm_open_driver(false, "", " for hot replug");
+
+	workload_start(priv, priv->fd.drm, &w);
+
+	device_unplug(priv, "hot ", 60);
+
+	bus_rescan(priv, 0);
+
+	workload_stop(priv, &w);
+
+	priv->fd.drm = close_device(priv->fd.drm, "late ", "removed ");
+	igt_assert_eq(priv->fd.drm, -1);
+
+	igt_assert_f(healthcheck(priv, false), "%s\n", priv->failure);
+}
+
 /* Main */
 
 int igt_main()
@@ -817,6 +1035,78 @@ int igt_main()
 			recover(&priv);
 	}
 
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if the driver can be cleanly unbound from an open device with a background GPU workload in flight, then released and rebound");
+		igt_subtest("hotunbind-rebind-with-load")
+			hotunbind_rebind_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if an open device with a background GPU workload in flight can be cleanly unplugged, then released and restored");
+		igt_subtest("hotunplug-rescan-with-load")
+			hotunplug_rescan_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if the driver can be cleanly rebound to a device with a still open hot unbound driver instance while a background GPU workload is in flight");
+		igt_subtest("hotrebind-with-load")
+			hotrebind_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if a hot unplugged and still open device can be cleanly restored while a background GPU workload is in flight");
+		igt_subtest("hotreplug-with-load")
+			hotreplug_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if a hot unbound driver instance still open after hot rebind with a background GPU workload in flight can be cleanly released");
+		igt_subtest("hotrebind-lateclose-with-load")
+			hotrebind_lateclose_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
+	igt_fixture()
+		post_healthcheck(&priv);
+
+	igt_subtest_group() {
+		igt_describe("Check if an instance of a still open while hot replugged device with a background GPU workload in flight can be cleanly released");
+		igt_subtest("hotreplug-lateclose-with-load")
+			hotreplug_lateclose_with_load(&priv);
+
+		igt_fixture()
+			recover(&priv);
+	}
+
 	igt_fixture() {
 		post_healthcheck(&priv);