Skip to contentSkip to conversation

Create class for building rust unit test

PATCH Superseded [email protected] 1 participant 5 messages Apr 27, 2023

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v3 cover Create class for building rust unit test

From: Frederic Martinsons <[email protected]>

This brings the possibility to use this class to build and ship unit tests of rust projects, the class also create (or modified) standard run-ptest script to run the generated rust test suite.

It has been tested successfully with core-image-sato under qemu for zvariant-ptest and python3-bcrypt-ptest (though the last one didn't define any unit tests).

Note that I tried to do the same with python3-cryptography but fail to build the test suite and I don't know how to do it with the rust extension module build by python setuptools. There must be some kind of way for doing that so maybe someone will put some work in it ( because python3-cryptography rust extension do have unit tests).

Moreover, in the class, I didn't manage to share data between do_compile_ptest_base and do_install_ptest_base cleanly (I tried to d.setVar in compile and d.getVar in install but it seems that the data store doens't recognize my new variable) so I used a file for that. I'm sure there is a clever way for doing that, so feel free to tell me.

The following changes since commit 45a8bb6e4676899d40525e7d5ad1c1ddefee3185:

apt-util: Fix ptest on musl (2023-04-20 11:56:03 +0100)

are available in the Git repository at:

https://gitlab.com/fmartinsons/openembedded-core cargo-add-ptest

Frederic Martinsons (3):
ptest-cargo.bbclass: create class
python3-bcrypt: enable build of unit tests
zvariant: add ptest feature for zvariant test suite

 .../zvariant/zvariant_3.12.0.bb               |   9 +-
 meta/classes-recipe/ptest-cargo.bbclass       | 129 ++++++++++++++++++
 meta/lib/oeqa/selftest/cases/devtool.py       |   2 +-
 .../python/python3-bcrypt_4.0.1.bb            |   4 +-
 4 files changed, 141 insertions(+), 3 deletions(-)
 create mode 100644 meta/classes-recipe/ptest-cargo.bbclass
Signature · 1 line
2.34.1
#1 lore raw

PATCH v3 1/3 ptest-cargo.bbclass: create class

From: Frederic Martinsons <[email protected]>

This new class offer the capbility to build rust tests and find them correctly.
Due to non deterministic name of generated binaries, a custom parsing of build result must be performed.
See https://github.com/rust-lang/cargo/issues/1924

All rust project will generate a test binary even if there are not test defined in source code (the binary will just output that it ran 0 tests)

---
 meta/classes-recipe/ptest-cargo.bbclass | 129 ++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 meta/classes-recipe/ptest-cargo.bbclass
meta/classes-recipe/ptest-cargo.bbclass new file +129 -0
new file mode 100644
index 0000000000..f26d4225d3
@@ -0,0 +1,129 @@
+inherit cargo ptest
+
+CARGO_TEST_BINARIES_FILES ?= "${B}/test_binaries_list"
+
+# sadly generated test binary have no deterministic names (https://github.com/rust-lang/cargo/issues/1924)
+# which force us to parse the cargo output in json format to find those test binaries
+python do_compile_ptest_cargo() {
+    import subprocess
+    import json
+
+    cargo = bb.utils.which(d.getVar("PATH"), d.getVar("CARGO", True))
+    cargo_build_flags = d.getVar("CARGO_BUILD_FLAGS", True)
+    rust_flags = d.getVar("RUSTFLAGS", True)
+    manifest_path = d.getVar("MANIFEST_PATH", True)
+
+    env = os.environ.copy()
+    env['RUSTFLAGS'] = rust_flags
+    cmd = f"{cargo} build --tests --message-format json {cargo_build_flags}"
+    bb.note(f"Building tests with cargo ({cmd})")
+
+    try:
+        proc = subprocess.Popen(cmd, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        bb.fatal(f"Cannot build test with cargo: {e}")
+
+    lines = []
+    for line in proc.stdout:
+        data = line.decode('utf-8').strip('\n')
+        lines.append(data)
+        bb.note(data)
+    proc.communicate()
+    if proc.returncode != 0:
+        bb.fatal(f"Unable to compile test with cargo, '{cmd}' failed")
+
+    # Definition of the format: https://doc.rust-lang.org/cargo/reference/external-tools.html#json-messages
+    test_bins = []
+    for line in lines:
+        try:
+            data = json.loads(line)
+        except json.JSONDecodeError:
+            # skip lines that are not a json
+            pass
+        else:
+            try:
+                # Filter the test packages coming from the current manifest
+                current_manifest_path = os.path.normpath(data['manifest_path'])
+                project_manifest_path = os.path.normpath(manifest_path)
+                if current_manifest_path == project_manifest_path:
+                    if data['target']['test'] or data['target']['doctest'] and data['executable']:
+                        test_bins.append(data['executable'])
+            except KeyError as e:
+                # skip lines that do not meet the requirements
+                pass
+
+    # All rust project will genrate at least one unit test binary
+    # It will just run a test suite with 0 tests if the project didn't define some
+    # So it is not expected to have an empty list here
+    if not test_bins:
+        bb.fatal("Unable to find any test binaries")
+
+    cargo_test_binaries_file = d.getVar('CARGO_TEST_BINARIES_FILES', True)
+    bb.note(f"Found {len(test_bins)} tests, write their path into {cargo_test_binaries_file}")
+    with open(cargo_test_binaries_file, "w") as f:
+        for test_bin in test_bins:
+            f.write(f"{test_bin}\n")
+
+}
+
+python do_install_ptest_cargo() {
+    import shutil
+
+    dest_dir = d.getVar("D", True)
+    pn = d.getVar("PN", True)
+    ptest_path = d.getVar("PTEST_PATH", True)
+    cargo_test_binaries_file = d.getVar('CARGO_TEST_BINARIES_FILES', True)
+
+    ptest_dir = os.path.join(dest_dir, ptest_path.lstrip('/'))
+    os.makedirs(ptest_dir, exist_ok=True)
+
+    test_bins = []
+    with open(cargo_test_binaries_file, "r") as f:
+        for line in f.readlines():
+            test_bins.append(line.strip('\n'))
+
+    test_paths = []
+    for test_bin in test_bins:
+        shutil.copy2(test_bin, ptest_dir)
+        test_paths.append(os.path.join(ptest_path, os.path.basename(test_bin)))
+
+    ptest_script = os.path.join(ptest_dir, "run-ptest")
+    if os.path.exists(ptest_script):
+        with open(ptest_script, "a") as f:
+            f.write(f"\necho \"\"\n")
+            f.write(f"echo \"## starting to run rust tests ##\"\n")
+            for test_path in test_paths:
+                f.write(f"{test_path}\n")
+    else:
+        with open(ptest_script, "a") as f:
+            f.write("#!/bin/sh\n")
+            for test_path in test_paths:
+                f.write(f"{test_path}\n")
+        os.chmod(ptest_script, 0o755)
+
+    # this is chown -R root:root ${D}${PTEST_PATH}
+    for root, dirs, files in os.walk(ptest_dir):
+        for d in dirs:
+            shutil.chown(os.path.join(root, d), "root", "root")
+        for f in files:
+            shutil.chown(os.path.join(root, f), "root", "root")
+}
+
+do_install_ptest_cargo[dirs] = "${B}"
+do_install_ptest_cargo[doc] = "Create or update the run-ptest script with rust test binaries generated"
+do_compile_ptest_cargo[dirs] = "${B}"
+do_compile_ptest_cargo[doc] = "Generate rust test binaries through cargo"
+
+addtask compile_ptest_cargo after do_compile            before do_compile_ptest_base
+addtask install_ptest_cargo after do_install_ptest_base before do_package
+
+python () {
+    if not bb.data.inherits_class('native', d) and not bb.data.inherits_class('cross', d):
+        d.setVarFlag('do_install_ptest_cargo', 'fakeroot', '1')
+        d.setVarFlag('do_install_ptest_cargp', 'umask', '022')
+
+    # Remove all '*ptest_cargo' tasks when ptest is not enabled
+    if not(d.getVar('PTEST_ENABLED') == "1"):
+        for i in ['do_compile_ptest_cargo', 'do_install_ptest_cargo']:
+            bb.build.deltask(i, d)
+}
Signature · 1 line
2.34.1
#3 lore raw

PATCH v3 2/3 python3-bcrypt: enable build of unit tests

In reply to [email protected]

From: Frederic Martinsons <[email protected]>

The source code of bcrypt extension doesn't define any tests but it is to show the ptest-cargo usage

Below is the updated test suite output for the recipe (cut):

PASS: tests/test_bcrypt.py:test_invalid_params[password-$2b$04$cVWp4XaNU8a4v1uMRum2SO-513-10-ValueError]
PASS: tests/test_bcrypt.py:test_invalid_params[password-$2b$04$cVWp4XaNU8a4v1uMRum2SO-20-0-ValueError]
PASS: tests/test_bcrypt.py:test_2a_wraparound_bug
============================================================================
Testsuite summary

running 0 tests

test result: ok$<2>. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s

root@qemux86-64:/usr/lib/python3-bcrypt/ptest#

---
 meta/recipes-devtools/python/python3-bcrypt_4.0.1.bb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
meta/recipes-devtools/python/python3-bcrypt_4.0.1.bb +3 -1
index 21f2eb6ba4..b4f245530d 100644
@@ -7,12 +7,14 @@ DEPENDS += "${PYTHON_PN}-cffi-native"
 
 SRC_URI[sha256sum] = "27d375903ac8261cfe4047f6709d16f7d18d39b1ec92aaf72af989552a650ebd"
 
-inherit pypi python_setuptools3_rust ptest cargo-update-recipe-crates
+inherit pypi python_setuptools3_rust ptest-cargo cargo-update-recipe-crates
 
 SRC_URI += " \
 	file://run-ptest \
 "
 
+CARGO_SRC_DIR = "src/_bcrypt"
+
 require ${BPN}-crates.inc
 
 RDEPENDS:${PN}-ptest += " \
Signature · 1 line
2.34.1
#4 lore raw

PATCH v3 3/3 zvariant: add ptest feature for zvariant test suite

In reply to [email protected]

From: Frederic Martinsons <[email protected]>

Below is the output of run-ptest script under qemu

root@qemux86-64:~# /usr/lib/zvariant/ptest/run-ptest

running 37 tests
test framing_offset_size::tests::framing_offset_size_bump ... ok$<2> test owned_value::tests::from_value ... ok$<2>
test object_path::unit::owned_from_reader ... ok$<2>
test str::tests::from_string ... ok$<2>
test signature::tests::signature_slicing ... ok$<2>
test str::tests::test_ordering ... ok$<2>
test owned_value::tests::map_conversion ... ok$<2>
test owned_value::tests::serde ... ok$<2>
test tests::enums ... ok$<2>
test tests::derive ... ok$<2>
test tests::f64_value ... ok$<2>
test tests::i16_value ... ok$<2>
test tests::fd_value ... ok$<2>
test tests::i32_value ... ok$<2>
test tests::i8_value ... ok$<2>
test tests::i64_value ... ok$<2>
test tests::ip_addr ... ok$<2>
test tests::issue_59 ... ok$<2>
test tests::issue_99 ... ok$<2>
test tests::array_value ... ok$<2>
test tests::object_path_value ... ok$<2>
test tests::dict_value ... ok$<2>
test tests::signature_value ... ok$<2>
test tests::serialized_size ... ok$<2>
test tests::struct_byte_array ... ok$<2>
test tests::struct_ref ... ok$<2>
test tests::str_value ... ok$<2>
test tests::option_value ... ok$<2>
test tests::struct_value ... ok$<2>
test tests::struct_with_hashmap ... ok$<2>
test tests::u16_value ... ok$<2>
test tests::u32_value ... ok$<2>
test tests::unit ... ok$<2>
test tests::u8_value ... ok$<2>
test tests::unit_fds ... ok$<2>
test tests::value_value ... ok$<2>
test tests::recursion_limits ... ok$<2>

test result: ok$<2>. 37 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.21s

root@qemux86-64:~#

---
 .../recipes-extended/zvariant/zvariant_3.12.0.bb         | 9 ++++++++-
 meta/lib/oeqa/selftest/cases/devtool.py                  | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)
meta-selftest/recipes-extended/zvariant/zvariant_3.12.0.bb +8 -1
index 4285d11b72..36f2a07ecf 100644
@@ -7,6 +7,8 @@ HOMEPAGE = "https://gitlab.freedesktop.org/dbus/zbus/"
 LICENSE = "MIT"
 LIC_FILES_CHKSUM = "file://LICENSE;md5=b377b220f43d747efdec40d69fcaa69d"
 
+DEPENDS += "${@bb.utils.contains('DISTRO_FEATURES', 'ptest', 'glib-2.0', '', d)}"
+
 SRC_URI = " \
     git://gitlab.freedesktop.org/dbus/zbus;protocol=https;branch=main;subpath=zvariant \
     file://0001-Tweak-zvariant-crate-config.patch;striplevel=2 \
@@ -21,10 +23,15 @@ python do_clean_lic_file_symlink() {
 
 addtask clean_lic_file_symlink after do_unpack before do_patch
 
-inherit cargo cargo-update-recipe-crates
+inherit ptest-cargo pkgconfig cargo-update-recipe-crates
 
 # Remove this when the recipe is reproducible
 EXCLUDE_FROM_WORLD = "1"
 
 require ${BPN}-crates.inc
 require ${BPN}-git-crates.inc
+
+# zvariant is an indermediate product for the zbus project
+# and so, it provided only a static lib (rlib) which fall only
+# in -dev package
+ALLOW_EMPTY:${PN} = "1"
meta/lib/oeqa/selftest/cases/devtool.py +1 -1
index 94873fd19f..1896944250 100644
@@ -876,7 +876,7 @@ class DevtoolModifyTests(DevtoolBase):
         self.assertTrue(any([uri.startswith('file://') and '.patch' in uri for uri in src_uri]),
                         'This test expects the %s recipe to have a patch in its src uris' % testrecipe)
 
-        self._test_recipe_contents(recipefile, {}, ['cargo'])
+        self._test_recipe_contents(recipefile, {}, ['ptest-cargo'])
 
         # Clean up anything in the workdir/sysroot/sstate cache
         bitbake('%s -c cleansstate' % testrecipe)
Signature · 1 line
2.34.1
#2 lore raw

Re: [PATCHV3 1/3] ptest-cargo.bbclass: create class

In reply to [email protected]
Quoted 161 lines · Le jeu. 27 avr. 2023, 07:21, <[email protected]> a écrit :

From: Frederic Martinsons <[email protected]>

This new class offer the capbility to build rust tests and find them correctly.
Due to non deterministic name of generated binaries, a custom parsing of build result must be performed.
See https://github.com/rust-lang/cargo/issues/1924

All rust project will generate a test binary even if there are not test defined in source code (the binary will just output that it ran 0 tests)

---
 meta/classes-recipe/ptest-cargo.bbclass | 129 ++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 meta/classes-recipe/ptest-cargo.bbclass
meta/classes-recipe/ptest-cargo.bbclass new file +5 -0
new file mode 100644
index 0000000000..f26d4225d3
@@ -0,0 +1,129 @@
+inherit cargo ptest
+
+CARGO_TEST_BINARIES_FILES ?= "${B}/test_binaries_list"
+
+# sadly generated test binary have no deterministic names (

https://github.com/rust-lang/cargo/issues/1924)
+# which force us to parse the cargo output in json format to find those test binaries

meta/classes-recipe/ptest-cargo.bbclass new file +12 -0
+python do_compile_ptest_cargo() {
+    import subprocess
+    import json
+
+    cargo = bb.utils.which(d.getVar("PATH"), d.getVar("CARGO", True))
+    cargo_build_flags = d.getVar("CARGO_BUILD_FLAGS", True)
+    rust_flags = d.getVar("RUSTFLAGS", True)
+    manifest_path = d.getVar("MANIFEST_PATH", True)
+
+    env = os.environ.copy()
+    env['RUSTFLAGS'] = rust_flags
+    cmd = f"{cargo} build --tests --message-format json

{cargo_build_flags}"

meta/classes-recipe/ptest-cargo.bbclass new file +4 -0
+    bb.note(f"Building tests with cargo ({cmd})")
+
+    try:
+        proc = subprocess.Popen(cmd, shell=True, env=env,

stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

meta/classes-recipe/ptest-cargo.bbclass new file +13 -0
+    except subprocess.CalledProcessError as e:
+        bb.fatal(f"Cannot build test with cargo: {e}")
+
+    lines = []
+    for line in proc.stdout:
+        data = line.decode('utf-8').strip('\n')
+        lines.append(data)
+        bb.note(data)
+    proc.communicate()
+    if proc.returncode != 0:
+        bb.fatal(f"Unable to compile test with cargo, '{cmd}' failed")
+
+    # Definition of the format:

https://doc.rust-lang.org/cargo/reference/external-tools.html#json-messages

meta/classes-recipe/ptest-cargo.bbclass new file +10 -0
+    test_bins = []
+    for line in lines:
+        try:
+            data = json.loads(line)
+        except json.JSONDecodeError:
+            # skip lines that are not a json
+            pass
+        else:
+            try:
+                # Filter the test packages coming from the current

manifest
+ current_manifest_path =
os.path.normpath(data['manifest_path'])

meta/classes-recipe/ptest-cargo.bbclass new file +3 -0
+                project_manifest_path = os.path.normpath(manifest_path)
+                if current_manifest_path == project_manifest_path:
+                    if data['target']['test'] or

data['target']['doctest'] and data['executable']:

meta/classes-recipe/ptest-cargo.bbclass new file +7 -0
+                        test_bins.append(data['executable'])
+            except KeyError as e:
+                # skip lines that do not meet the requirements
+                pass
+
+    # All rust project will genrate at least one unit test binary
+    # It will just run a test suite with 0 tests if the project didn't

define some

meta/classes-recipe/ptest-cargo.bbclass new file +6 -0
+    # So it is not expected to have an empty list here
+    if not test_bins:
+        bb.fatal("Unable to find any test binaries")
+
+    cargo_test_binaries_file = d.getVar('CARGO_TEST_BINARIES_FILES', True)
+    bb.note(f"Found {len(test_bins)} tests, write their path into

{cargo_test_binaries_file}")

meta/classes-recipe/ptest-cargo.bbclass new file +26 -0
+    with open(cargo_test_binaries_file, "w") as f:
+        for test_bin in test_bins:
+            f.write(f"{test_bin}\n")
+
+}
+
+python do_install_ptest_cargo() {
+    import shutil
+
+    dest_dir = d.getVar("D", True)
+    pn = d.getVar("PN", True)
+    ptest_path = d.getVar("PTEST_PATH", True)
+    cargo_test_binaries_file = d.getVar('CARGO_TEST_BINARIES_FILES', True)
+
+    ptest_dir = os.path.join(dest_dir, ptest_path.lstrip('/'))
+    os.makedirs(ptest_dir, exist_ok=True)
+
+    test_bins = []
+    with open(cargo_test_binaries_file, "r") as f:
+        for line in f.readlines():
+            test_bins.append(line.strip('\n'))
+
+    test_paths = []
+    for test_bin in test_bins:
+        shutil.copy2(test_bin, ptest_dir)
+        test_paths.append(os.path.join(ptest_path,

os.path.basename(test_bin)))

meta/classes-recipe/ptest-cargo.bbclass new file +25 -0
+
+    ptest_script = os.path.join(ptest_dir, "run-ptest")
+    if os.path.exists(ptest_script):
+        with open(ptest_script, "a") as f:
+            f.write(f"\necho \"\"\n")
+            f.write(f"echo \"## starting to run rust tests ##\"\n")
+            for test_path in test_paths:
+                f.write(f"{test_path}\n")
+    else:
+        with open(ptest_script, "a") as f:
+            f.write("#!/bin/sh\n")
+            for test_path in test_paths:
+                f.write(f"{test_path}\n")
+        os.chmod(ptest_script, 0o755)
+
+    # this is chown -R root:root ${D}${PTEST_PATH}
+    for root, dirs, files in os.walk(ptest_dir):
+        for d in dirs:
+            shutil.chown(os.path.join(root, d), "root", "root")
+        for f in files:
+            shutil.chown(os.path.join(root, f), "root", "root")
+}
+
+do_install_ptest_cargo[dirs] = "${B}"
+do_install_ptest_cargo[doc] = "Create or update the run-ptest script with

rust test binaries generated"

meta/classes-recipe/ptest-cargo.bbclass new file +4 -0
+do_compile_ptest_cargo[dirs] = "${B}"
+do_compile_ptest_cargo[doc] = "Generate rust test binaries through cargo"
+
+addtask compile_ptest_cargo after do_compile            before

do_compile_ptest_base

meta/classes-recipe/ptest-cargo.bbclass new file +4 -0
+addtask install_ptest_cargo after do_install_ptest_base before do_package
+
+python () {
+    if not bb.data.inherits_class('native', d) and not

bb.data.inherits_class('cross', d):

meta/classes-recipe/ptest-cargo.bbclass line 123

Quoted 2 lines
meta/classes-recipe/ptest-cargo.bbclass new file +2 -0
+        d.setVarFlag('do_install_ptest_cargo', 'fakeroot', '1')
+        d.setVarFlag('do_install_ptest_cargp', 'umask', '022')

Auto reader on: "You have a typo on the task name"
Me: "argh, thanks, will correct that in a V4"

:>)

+ # Remove all '*ptest_cargo' tasks when ptest is not enabled

Quoted 8 lines
meta/classes-recipe/ptest-cargo.bbclass new file +4 -1
+    if not(d.getVar('PTEST_ENABLED') == "1"):
+        for i in ['do_compile_ptest_cargo', 'do_install_ptest_cargo']:
+            bb.build.deltask(i, d)
+}
--

2.34.1