support/runtime-test: extend graphic testing for weston + flutter (branch yem/flutter)

In 4edb0e3456ef (support/testing/tests/package/test_weston.py: new runtime test), the weston test was introduced, and thus was the first that needed to test that rendering was happening.

Now we also have a test for a flutter application, and we'll want to have it test the rendering too.

Move the corresponding code to a helper that can be reused by other tests, rather than duplicate (or reinvent) it.

Switch weston to using that new helper.

Signed-off-by Yann E. MORIN <[email protected]>

Cc Julien Olivain <[email protected]>

---
 support/testing/tests/graphics_base.py       | 39 +++++++++++++
 support/testing/tests/package/test_weston.py | 61 ++++----------------
 2 files changed, 49 insertions(+), 51 deletions(-)
 create mode 100644 support/testing/tests/graphics_base.py

support/testing/tests/graphics_base.py new file +0 -0

new file mode 100644
index 0000000000..15a4c00bb2

support/testing/tests/graphics_base.py new file +14 -0

+    def get_n_fb_crc(self, *, count=10, uniq=False, timeout=-1):
+        """
+        Return count DRM CRC from the framebuffer. If uniq is True,
+        only unique CRCs are returned (which may be less than the
+        requested cont).
+        Returns a possibly empty list of integers.
+        Set timeout to -1 for no timeout, or to a positive number for
+        a timeout of that many seconds.
+        """
+        # DRM CRCs are exposed through a sysfs pseudo file
+        try:
+            self.debugfs_mounted
+        except AttributeError:
+            # Note: some init system (e.g. systemd) may have this 

already

support/testing/tests/graphics_base.py new file +5 -1

+            # mounted, so check beforehand
+            self.assertRunOk("mountpoint /sys/kernel/debug/ || mount 
-t debugfs none /sys/kernel/debug/")
+            self.debugfs_mounted = True
+
+        # The first column is the frame number, the second column is 

the

support/testing/tests/graphics_base.py new file +17 -0

+        # CRC measure. We use "head" to get the needed CRC count.
+        disp_crc_path = "/sys/kernel/debug/dri/0/crtc-0/crc/data"
+        cmd = f"head -{count} {disp_crc_path}"
+
+        # The DRM CRC sysfs pseudo file lines are terminated by '\n'
+        # and '\0'. We remove the '\0' to have a text-only output.
+        cmd += " | tr -d '\\000'"
+
+        # Finally, we drop the frame counter, and keep only the second
+        # column (CRC values)
+        cmd += " | cut -f 2 -d ' '"
+
+        if uniq:
+            cmd += " | sort -u"
+
+        output, exit_code = self.emulator.run(cmd, timeout=timeout)
+        self.assertTrue(exit_code == 0, f"'{cmd}' failed with exit 

code {exit_code}")

support/testing/tests/graphics_base.py new file +2 -0

+
+        return [int(crc, 16) for crc in output]

support/testing/tests/package/test_weston.py +2 -3

index df1b7a4135..f37a73565f 100644
@@ -2,9 +2,10 @@ import os
 import time
 
 import infra.basetest
+from ..graphics_base import GraphicsBase
 
 
-class TestWeston(infra.basetest.BRTest):
+class TestWeston(infra.basetest.BRTest, GraphicsBase):
     config = \
         """
         BR2_aarch64=y

@@ -36,31 +37,6 @@ class TestWeston(infra.basetest.BRTest):
 
 infra.filepath("tests/package/test_weston/linux-vkms.fragment")
              )
 
-    def gen_read_disp_crcs_cmd(self, count=1):
-        # DRM CRCs are exposed through a sysfs pseudo file, one 

measure

support/testing/tests/package/test_weston.py +0 -23

-        # per line. The first column is the frame number, the second
-        # column is the CRC measure. We use "head" to get the needed
-        # CRC count.
-        disp_crc_path = "/sys/kernel/debug/dri/0/crtc-0/crc/data"
-        cmd = f"head -{count} {disp_crc_path}"
-
-        # The DRM CRC sysfs pseudo file lines are terminated by '\n'
-        # and '\0'. We remove the '\0' to have a text-only output.
-        cmd += " | tr -d '\\000'"
-
-        # Finally, we drop the frame counter, and keep only the second
-        # column (CRC values)
-        cmd += " | cut -f 2 -d ' '"
-
-        return cmd
-
-    def gen_count_unique_disp_crcs_cmd(self, count=10):
-        # We get the command generating one CRC per line...
-        cmd = self.gen_read_disp_crcs_cmd(count)
-        # ...then count the number of unique values
-        cmd += " | uniq | wc -l"
-        return cmd
-
     def start_weston(self):
         self.assertRunOk("export XDG_RUNTIME_DIR=/tmp")

@@ -106,25 +82,14 @@ class TestWeston(infra.basetest.BRTest):
         # Check a simple info client can communicate with the 

compositor
self.assertRunOk("wayland-info", timeout=10)

support/testing/tests/package/test_weston.py +0 -3

-        # This test will use the Kernel VKMS DRM Display CRC support,
-        # which is exposed in debugfs. See:
-        # 

https://docs.kernel.org/gpu/drm-uapi.html#display-crc-support

support/testing/tests/package/test_weston.py +6 -18

-        self.assertRunOk("mount -t debugfs none /sys/kernel/debug/")
-
         # We get 10 consecutive DRM frame CRCs and count how many
         # unique CRCs we have. Since weston is supposed to run idle,
         # we should have 10 times the same display CRC.
-        cmd = self.gen_count_unique_disp_crcs_cmd()
-        output, exit_code = self.emulator.run(cmd)
-        self.assertEqual(exit_code, 0)
-        self.assertEqual(int(output[0]), 1)
+        self.assertTrue(len(self.get_n_fb_crc(uniq=True)) == 1)
 
         # We save the CRC value of an empty weston desktop for
         # later...
-        cmd = self.gen_read_disp_crcs_cmd()
-        output, exit_code = self.emulator.run(cmd)
-        self.assertEqual(exit_code, 0)
-        weston_desktop_crc = int(output[0], 16)
+        weston_desktop_crc = self.get_n_fb_crc(count=1)[0]
 
         # We start the weston-simple-egl in background...  Every
         # rendered frame is supposed to be different (as the triangle

@@ -138,10 +103,8 @@ class TestWeston(infra.basetest.BRTest):
         # display something, we are now supposed to measure a
         # different display CRC than the one we measured when the
         # desktop was empty.
-        cmd = self.gen_read_disp_crcs_cmd()
-        output, exit_code = self.emulator.run(cmd)
-        self.assertEqual(exit_code, 0)
-        self.assertNotEqual(int(output[0], 16), weston_desktop_crc)
+        crc = self.get_n_fb_crc(count=1)[0]
+        self.assertNotEqual(crc, weston_desktop_crc)
 
         # While weston-simple-egl is running, we check the VKMS DRM
         # CRCs are now changing. We get many CRCs, one per display

@@ -152,10 +115,8 @@ class TestWeston(infra.basetest.BRTest):
         # remain very permissive to slow emulation situations.
         # Increase timeout, as the command is expected to run about 
 5s,
         # which is the default timeout.
-        cmd = self.gen_count_unique_disp_crcs_cmd(300)
-        output, exit_code = self.emulator.run(cmd, timeout=10)
-        self.assertEqual(exit_code, 0)
-        self.assertGreaterEqual(int(output[0]), 5)
+        crcs = self.get_n_fb_crc(count=300, timeout=10)
+        self.assertGreaterEqual(len(crcs), 5)
 
         # We stop weston-simple-egl, and sleep a bit to let Weston do

         # its cleanup and desktop repaint refresh...

support/testing/tests/package/test_weston.py +2 -5

@@ -165,10 +126,8 @@ class TestWeston(infra.basetest.BRTest):
         # After we stopped the application, we should have the initial
         # weston desktop background. The CRC we measure now should be
         # the same as the one we saved earlier.
-        cmd = self.gen_read_disp_crcs_cmd()
-        output, exit_code = self.emulator.run(cmd)
-        self.assertEqual(exit_code, 0)
-        self.assertEqual(int(output[0], 16), weston_desktop_crc)
+        crc = self.get_n_fb_crc(count=1)[0]
+        self.assertEqual(crc, weston_desktop_crc)
 
         self.stop_weston()
 
--

2.25.1

In 4edb0e3456ef (support/testing/tests/package/test_weston.py: new runtime test), the weston test was introduced, and thus was the first that needed to test that rendering was happening.

Now we also have a test for a flutter application, and we'll want to have it test the rendering too.

Move the corresponding code to a helper that can be reused by other tests, rather than duplicate (or reinvent) it.

Switch weston to using that new helper.

Signed-off-by Yann E. MORIN <[email protected]>

Cc Julien Olivain <[email protected]>

Commit 7f0af11cee31 (support/testing/tests/package/test_flutter.py: new runtime test) added a simple test that just checked that the systemd unit launching the application, was active.

It is perfectly possible, from a systemd perspective, that the unit is active, while the application actually crashes.

Instead, what we need to check, is that the application does actually render "something"; we don't really care what, as long as we know it is actually rendering, thus the graphical stack is working and the aplication stack is running.

Extend the flutter runtime test to also check that the framebuffer is modified by the application when it is running, similarly to what we do in the weston test.

We drop the activation of the unit and start it manually, as we want to check the state of the CRC before, while, and after the application runs.

We also need to disable the blinking cursor on the console, or we would not be able to detect whether a change in CRC is due to the application starting rendering, or to the cursor blinking. We tell the kernel to disable the cursor with the appropriate kernel command line parameter.

Signed-off-by Yann E. MORIN <[email protected]>

Cc Adam Duskett <[email protected]>

---
Changes v1 -> v2:
- don't mistake blinking cursor for application start (Adam)
- properly terminate application (Adam)

---
 support/testing/tests/package/test_flutter.py | 35 +++++++++++++++----
 .../flutter-gallery.service                   |  1 -
 2 files changed, 29 insertions(+), 7 deletions(-)
 delete mode 120000

support/testing/tests/package/test_flutter/overlay/etc/systemd/system/multi-user.target.wants/flutter-gallery.service

support/testing/tests/package/test_flutter.py +5 -2

index a3e98a43e5..515c95409f 100644
@@ -1,8 +1,11 @@
 import os
+import time
 import infra.basetest
 
+from ..graphics_base import GraphicsBase
 
-class TestFlutter(infra.basetest.BRTest):
+
+class TestFlutter(infra.basetest.BRTest, GraphicsBase):
     config = f"""
         BR2_aarch64=y
         BR2_TOOLCHAIN_EXTERNAL=y

@@ -39,7 +42,7 @@ class TestFlutter(infra.basetest.BRTest):
         self.emulator.boot(
             arch="aarch64",
             kernel=kern,
-            kernel_cmdline=["root=/dev/vda console=ttyAMA0"],
+            kernel_cmdline=["root=/dev/vda console=ttyAMA0
 vt.global_cursor_default=0"],
             options=["-M", "virt",
                      "-cpu", "cortex-a57",

                      "-m", "512M",

support/testing/tests/package/test_flutter.py +15 -4

@@ -48,7 +51,27 @@ class TestFlutter(infra.basetest.BRTest):
                      "-vnc", "none",
                      "-drive", f"file={img},if=virtio,format=raw"])
         self.emulator.login()
-        cmd = "systemctl is-active flutter-gallery"
-        output, exit_code = self.emulator.run(cmd, 10)
-        self.assertEqual(exit_code, 0)
-        self.assertEqual(output[0], "active")
+
+        # Get the CRC from the current ramebuffer
+        empty_crc = self.get_n_fb_crc(count=1)[0]
+
+        # Start the gallery App. It can take a bit of time to start,
+        # so lets try a few times. 600 samples should cover about 10s
+        # @60Hz (although, the rendering could be much slower on slow
+        # machines)
+        self.assertRunOk("systemctl start flutter-gallery", timeout=10)
+        for i in range(600):
+            gallery_crc = self.get_n_fb_crc(count=1)[0]
+            if gallery_crc != empty_crc:
+                break
+            time.sleep(1)
+        self.assertNotEqual(gallery_crc, empty_crc, "gallery app did not

render anything on screen")

support/testing/tests/package/test_flutter.py +2 -0

+
+        # Stop the application, and check it restored the framebuffer

content

support/testing/tests/package/test_flutter.py +7 -0

+        self.assertRunOk("systemctl stop flutter-gallery", timeout=10)
+        for i in range(600):
+            gallery_crc = self.get_n_fb_crc(count=1)[0]
+            if gallery_crc == empty_crc:
+                break
+            time.sleep(1)
+        self.assertEqual(gallery_crc, empty_crc, "gallery app did not

stop rendering")
diff --git a/support/testing/tests/package/test_flutter/overlay/etc/systemd/system/multi-user.target.wants/flutter-gallery.service b/support/testing/tests/package/test_flutter/overlay/etc/systemd/system/multi-user.target.wants/flutter-gallery.service deleted file mode 120000
index 40993fb16c..0000000000
---
a/support/testing/tests/package/test_flutter/overlay/etc/systemd/system/multi-user.target.wants/flutter-gallery.service +++ /dev/null

support/testing/tests/package/test_flutter.py +0 -2

@@ -1 +0,0 @@
-../../../../usr/lib/systemd/system/flutter-gallery.service
\ No newline at end of file
--

2.25.1

Yann E. MORIN (2):
support/runtime-test: add helper to test graphics rendering suport/runtime-test: extend flutter test to check rendering is happening