Fix .ko file pre-filter to use endswith instead of string contains
v2 cover Fix .ko file pre-filter to use endswith instead of string contains
Sam Kent (3):
package.py: fix kernel module file pre-filter and document strip
asymmetry
oe/package: add unit tests for kernel module detection helpers oeqa/selftest: add oe-selftest for kernel module pre-filter
.../selftest-ko-filter/files/module.c | 3 + .../selftest-ko-filter/selftest-ko-filter.bb | 29 +++++ meta/lib/oe/package.py | 5 +- meta/lib/oe/tests/__init__.py | 0 meta/lib/oe/tests/test_package.py | 121 ++++++++++++++++++ meta/lib/oeqa/selftest/cases/package.py | 22 ++++ 6 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/files/module.c create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb create mode 100644 meta/lib/oe/tests/__init__.py create mode 100644 meta/lib/oe/tests/test_package.py
Signature · 1 line
2.34.1
PATCH v2 1/3 package.py: fix kernel module file pre-filter and document strip asymmetry
Change the check to f.endswith(".ko"), consistent with strip_execs() and with the /lib/modules/ guard already present in is_elf() and splitdebuginfo().
Fixes [YOCTO #2348]
--- meta/lib/oe/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
meta/lib/oe/package.py +3 -2
@@ -37,7 +37,8 @@ def runstrip(file, elftype, strip, extra_strip_sections=''): stripcmd = [strip] skip_strip = False - # kernel module + # kernel module: use --strip-debug and --preserve-dates (required for + # module signing to remain valid after stripping) if elftype & 16: if is_kernel_module_signed(file): bb.debug(1, "Skip strip on signed module %s" % file)
@@ -1167,7 +1168,7 @@ def process_split_and_strip_files(d): or (s[stat.ST_MODE] & stat.S_IXOTH) \ or ((file.startswith(libdir) or file.startswith(baselibdir)) \ and (".so" in f or ".node" in f)) \ - or (f.startswith('vmlinux') or ".ko" in f): + or (f.startswith('vmlinux') or f.endswith(".ko")): if cpath.islink(file): checkelflinks[file] = ltarget
Signature · 1 line
2.34.1
PATCH v2 2/3 oe/package: add unit tests for kernel module detection helpers
In reply to Sam KentAdd unit tests for the filename pre-filter logic, is_kernel_module(), and the is_kernel_module_signed() detection.
--- meta/lib/oe/tests/__init__.py | 0 meta/lib/oe/tests/test_package.py | 121 ++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 meta/lib/oe/tests/__init__.py create mode 100644 meta/lib/oe/tests/test_package.py
meta/lib/oe/tests/__init__.py new file +0 -0
meta/lib/oe/tests/test_package.py new file +121 -0
121 lines not on this page · raw message
Signature · 1 line
2.34.1
Re: [OE-core] [PATCH v2 2/3] oe/package: add unit tests for kernel module detection helpers
Quoted 10 lines · Sam Kent via lists.openembedded.org
Add unit tests for the filename pre-filter logic,
is_kernel_module(), and theis_kernel_module_signed()detection.Signed-off-by Sam Kent <[email protected]>---
meta/lib/oe/tests/__init__.py | 0meta/lib/oe/tests/test_package.py| 121 ++++++++++++++++++++++++++++++2 files changed, 121 insertions(+)create mode 100644
meta/lib/oe/tests/__init__.pycreate mode 100644meta/lib/oe/tests/test_package.py
Thanks, this is a good step in the right direction, however if won't be executed by our standard autobuilder tests if it is in this location. We run tests via oe-selftest (amongst other places).
The tests for lib/oe are in lib/oeqa/sefltest/cases/oelib/
You can run them with "oe-selftest -r oelib"
A recent example of a change which adds tests would be:
https://git.openembedded.org/openembedded-core/commit/?id=20322406a77b178a6655427612ed25e79b308b21
If we extend them that way, they will be run as part of our regular testing and then hopefully regressions get caught.
Your base patch 1/3 looked fine and passed review so we've merged that, thanks!
Cheers,
Richard
Re: [OE-core] [PATCH v2 2/3] oe/package: add unit tests for kernel module detection helpers
In reply to Sam KentQuoted 27 lines · Sam Kent via lists.openembedded.org
Add unit tests for the filename pre-filter logic,
is_kernel_module(), and theis_kernel_module_signed()detection.Signed-off-by Sam Kent <[email protected]>---
meta/lib/oe/tests/__init__.py | 0meta/lib/oe/tests/test_package.py| 121 ++++++++++++++++++++++++++++++2 files changed, 121 insertions(+)create mode 100644
meta/lib/oe/tests/__init__.pycreate mode 100644meta/lib/oe/tests/test_package.pymeta/lib/oe/tests/test_package.py new file +16
-0+class TestIsKernelModule(unittest.TestCase): + """ + is_kernel_module() detects kernel modules by searching for the + "vermagic=" string, which is always present in genuine .ko files. + """ + + def _make_tmp(self, content: bytes) -> str: + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def tearDown(self): + # Clean up any temp files created during the test. + for attr in ("_tmpfile",): + path = getattr(self, attr, None)
meta/lib/oe/tests/test_package.py line 88
Quoted 10 lines
meta/lib/oe/tests/test_package.py new file +10
-0+ if path and os.path.exists(path): + os.unlink(path) + + def test_detects_vermagic(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 10 + b"vermagic=5.15.0" + b"\x00" * 10) + self.assertTrue(is_kernel_module(self._tmpfile)) + + def test_rejects_plain_elf(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 50) + self.assertFalse(is_kernel_module(self._tmpfile))
Sorry about the delay in replying to this, I think we were all confused about who was going to do it!
The code above is a little bit more complex than it needs to be. It can be simplified to something like:
with NamedTemporaryFile(suffix=".ko") as f:
f.write(b"\x7fELF\x00" * 10 + b"vermagic=5.15.0" + b"\x00" * 10)
self.assertTrue(is_kernel_module(f.name))which will self clean up. Could you make that tweak, then we should be able to merge, thanks!
Cheers,
Richard
PATCH v2 3/3 oeqa/selftest: add oe-selftest for kernel module pre-filter
In reply to Sam KentAdd selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify process_split_and_strip_files() handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.
AI-Generated: Uses Claude Sonnet 4.6
--- .../selftest-ko-filter/files/module.c | 3 ++ .../selftest-ko-filter/selftest-ko-filter.bb | 29 +++++++++++++++++++ meta/lib/oeqa/selftest/cases/package.py | 22 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/files/module.c create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb
meta-selftest/recipes-test/selftest-ko-filter/files/module.c new file +3 -0
@@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: MIT */ +/* Minimal relocatable object used as a .ko stand-in for pre-filter testing. */ +int selftest_ko_filter_marker =42 ;
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +29 -0
@@ -0,0 +1,29 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +SUMMARY = "Test fixture for the kernel-module file pre-filter in package.py" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://module.c" +S = "${UNPACKDIR}" + +MODDIR = "${libdir}/selftest-ko-filter" + +do_compile () { + ${CC} ${CFLAGS} -c module.c -o module.ko +} + +do_install () { + install -d ${D}${MODDIR} + install -m 0644 module.ko ${D}${MODDIR}/ + + # Fake compressed modules — the pre-filter must not pass these to strip. + # Use octal escapes: dash (the OE recipe shell) does not support \xHH in printf. + printf '\375\067\172\130\132\000' > ${D}${MODDIR}/module.ko.xz + printf '\037\213' > ${D}${MODDIR}/module.ko.gz +} + +FILES:${PN} = "${MODDIR}/*"
meta/lib/oeqa/selftest/cases/package.py +22 -0
@@ -185,6 +185,28 @@ class PackageTests(OESelftestTestCase): if not gdbtest(qemu, binary): self.fail('GDB %s failed' % binary) + def test_kmodule_prefilter(self): + # Regression test for YOCTO #2348: process_split_and_strip_files() must + # use f.endswith(".ko") so that compressed modules (.ko.xz, .ko.gz) are + # not fed to is_elf() / strip. + bitbake("selftest-ko-filter -c package") + + pkgdest = get_bb_var('PKGDEST', 'selftest-ko-filter') + libdir = get_bb_var('libdir', 'selftest-ko-filter') + moddir = pkgdest + "/selftest-ko-filter" + libdir + "/selftest-ko-filter" + + self.assertTrue(os.path.exists(moddir + "/module.ko"), + "module.ko missing from PKGDEST") + + for fname, magic in [("module.ko.xz", b"\xfd\x37\x7a\x58\x5a\x00"), + ("module.ko.gz", b"\x1f\x8b")]: + path = moddir + "/" + fname + self.assertTrue(os.path.exists(path), + "%s missing from PKGDEST" % fname) + with open(path, "rb") as f: + self.assertEqual(f.read(len(magic)), magic, + "%s header corrupted in PKGDEST" % fname) + def test_preserve_ownership(self): features = 'IMAGE_INSTALL:append = " selftest-chown"\n' self.write_config(features)
Signature · 1 line
2.34.1
PATCH v3 1/2 oelib: add unit tests for kernel module detection helpers
In reply to Sam KentAdd tests for the filename pre-filter logic, is_kernel_module(), and is_kernel_module_signed() to meta/lib/oeqa/selftest/cases/oelib/
--- meta/lib/oeqa/selftest/cases/oelib/package.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 meta/lib/oeqa/selftest/cases/oelib/package.py
meta/lib/oeqa/selftest/cases/oelib/package.py new file +108 -0
@@ -0,0 +1,108 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import mmap +import os +import tempfile +from unittest.case import TestCase + + +class TestKernelModuleFilenameFilter(TestCase): + """ + The pre-filter in process_split_and_strip_files() selects candidates to + pass to is_elf(). It must use f.endswith(".ko"), not ".ko" in f, to avoid + false-positives on compressed modules (.ko.xz, .ko.gz). + """ + + ko_files = [ + "driver.ko", + "net/foo.ko", + ] + + not_ko_files = [ + "driver.ko.xz", + "driver.ko.gz", + "driver.ko2", + "myko.c", + "vmlinux", + ] + + def test_endswith_matches_ko(self): + for f in self.ko_files: + with self.subTest(f=f): + self.assertTrue(f.endswith(".ko")) + + def test_endswith_rejects_non_ko(self): + for f in self.not_ko_files: + with self.subTest(f=f): + self.assertFalse(f.endswith(".ko")) + + def test_old_predicate_had_false_positives(self): + # The previous check (".ko" in f) matched compressed modules — this is + # the regression the fix addresses. + false_positives = [f for f in self.not_ko_files if ".ko" in f] + self.assertEqual(false_positives, ["driver.ko.xz", "driver.ko.gz", "driver.ko2"]) + + +class TestIsKernelModule(TestCase): + """ + is_kernel_module() detects kernel modules by searching for the + "vermagic=" string, which is always present in genuine .ko files. + """ + + def setUp(self): + from oe.package import is_kernel_module + self.is_kernel_module = is_kernel_module + self._tmpfile = None + + def tearDown(self): + if self._tmpfile and os.path.exists(self._tmpfile): + os.unlink(self._tmpfile) + + def _make_tmp(self, content): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def test_detects_vermagic(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 10 + b"vermagic=5.15.0" + b"\x00" * 10) + self.assertTrue(self.is_kernel_module(self._tmpfile)) + + def test_rejects_plain_elf(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 50) + self.assertFalse(self.is_kernel_module(self._tmpfile)) + + +class TestIsKernelModuleSigned(TestCase): + """ + is_kernel_module_signed() detects the "Module signature appended" tail + that the kernel's modsign infrastructure writes. + """ + + def setUp(self): + from oe.package import is_kernel_module_signed + self.is_kernel_module_signed = is_kernel_module_signed + self._tmpfile = None + + def tearDown(self): + if self._tmpfile and os.path.exists(self._tmpfile): + os.unlink(self._tmpfile) + + def _make_tmp(self, content): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def test_detects_signed(self): + tail = b"Module signature appended\n\x00\x00" + self._tmpfile = self._make_tmp(b"\x00" * 64 + tail) + self.assertTrue(self.is_kernel_module_signed(self._tmpfile)) + + def test_rejects_unsigned(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 20) + self.assertFalse(self.is_kernel_module_signed(self._tmpfile))
Signature · 1 line
2.34.1
PATCH v3 2/2 oeqa/selftest: add oe-selftest for kernel module pre-filter
Add selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify process_split_and_strip_files() handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.
AI-Generated: Uses Claude Sonnet 4.6
--- .../selftest-ko-filter/files/module.c | 3 ++ .../selftest-ko-filter/selftest-ko-filter.bb | 29 +++++++++++++++++++ meta/lib/oeqa/selftest/cases/package.py | 22 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/files/module.c create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb
meta-selftest/recipes-test/selftest-ko-filter/files/module.c new file +3 -0
@@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: MIT */ +/* Minimal relocatable object used as a .ko stand-in for pre-filter testing. */ +int selftest_ko_filter_marker =42 ;
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +29 -0
@@ -0,0 +1,29 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +SUMMARY = "Test fixture for the kernel-module file pre-filter in package.py" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://module.c" +S = "${UNPACKDIR}" + +MODDIR = "${libdir}/selftest-ko-filter" + +do_compile () { + ${CC} ${CFLAGS} -c module.c -o module.ko +} + +do_install () { + install -d ${D}${MODDIR} + install -m 0644 module.ko ${D}${MODDIR}/ + + # Fake compressed modules — the pre-filter must not pass these to strip. + # Use octal escapes: dash (the OE recipe shell) does not support \xHH in printf. + printf '\375\067\172\130\132\000' > ${D}${MODDIR}/module.ko.xz + printf '\037\213' > ${D}${MODDIR}/module.ko.gz +} + +FILES:${PN} = "${MODDIR}/*"
meta/lib/oeqa/selftest/cases/package.py +22 -0
@@ -185,6 +185,28 @@ class PackageTests(OESelftestTestCase): if not gdbtest(qemu, binary): self.fail('GDB %s failed' % binary) + def test_kmodule_prefilter(self): + # Regression test for YOCTO #2348: process_split_and_strip_files() must + # use f.endswith(".ko") so that compressed modules (.ko.xz, .ko.gz) are + # not fed to is_elf() / strip. + bitbake("selftest-ko-filter -c package") + + pkgdest = get_bb_var('PKGDEST', 'selftest-ko-filter') + libdir = get_bb_var('libdir', 'selftest-ko-filter') + moddir = pkgdest + "/selftest-ko-filter" + libdir + "/selftest-ko-filter" + + self.assertTrue(os.path.exists(moddir + "/module.ko"), + "module.ko missing from PKGDEST") + + for fname, magic in [("module.ko.xz", b"\xfd\x37\x7a\x58\x5a\x00"), + ("module.ko.gz", b"\x1f\x8b")]: + path = moddir + "/" + fname + self.assertTrue(os.path.exists(path), + "%s missing from PKGDEST" % fname) + with open(path, "rb") as f: + self.assertEqual(f.read(len(magic)), magic, + "%s header corrupted in PKGDEST" % fname) + def test_preserve_ownership(self): features = 'IMAGE_INSTALL:append = " selftest-chown"\n' self.write_config(features)
Signature · 1 line
2.34.1
Re: [OE-core] [PATCH v3 2/2] oeqa/selftest: add oe-selftest for kernel module pre-filter
Quoted 9 lines · Sam Kent via lists.openembedded.org
Add selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify
process_split_and_strip_files()handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.AI-Generated: Uses Claude Sonnet 4.6
Signed-off-by Sam Kent <[email protected]>---
Hi Sam,
Thanks for your patch.
meta/lib/oeqa/selftest/cases/package.py
Quoted 4 lines
meta/lib/oeqa/selftest/cases/package.py +0
-0
I have some trouble to find a version with this patch in. Are you based on the master branch?
Thanks,
Mathieu
Signature · 3 lines
Mathieu Dubois-Briand, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Re: [OE-core] [PATCH v3 2/2] oeqa/selftest: add oe-selftest for kernel module pre-filter
Quoted 23 lines · Mathieu Dubois-Briand
Quoted 9 lines · Sam Kent via lists.openembedded.org
Add selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify
process_split_and_strip_files()handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.AI-Generated: Uses Claude Sonnet 4.6
Signed-off-by Sam Kent <[email protected]>---
Hi Sam,
Thanks for your patch.
meta/lib/oeqa/selftest/cases/package.py
Quoted 4 lines
meta/lib/oeqa/selftest/cases/package.py +0
-0I have some trouble to find a version with this patch in. Are you based on the master branch?
Quoted 2 lines
Thanks,
Mathieu
Ignore this, Monday morning, I just messed up my patch apply.
Sorry for the noise.
Signature · 3 lines
Mathieu Dubois-Briand, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Re: [OE-core] [PATCH v3 1/2] oelib: add unit tests for kernel module detection helpers
In reply to Sam KentQuoted 5 lines · Sam Kent via lists.openembedded.org
Add tests for the filename pre-filter logic,
is_kernel_module(), andis_kernel_module_signed()to meta/lib/oeqa/selftest/cases/oelib/Signed-off-by Sam Kent <[email protected]>---
Hi Sam,
Now I remember why I had your patch flagged.
This is triggering a fail in another test, as can be seen on the autobuilder:
2026-05-05 09:36:25,307 - oe-selftest - INFO - pkgdata.OePkgdataUtilTests.test_lookup_recipe (subunit.RemotedTestCase) 2026-05-05 09:36:25,307 - oe-selftest - INFO - ... FAIL
...
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/openembedded-core/meta/lib/oeqa/selftest/cases/pkgdata.py", line 62, in test_lookup_recipe
self.assertEqual(result.output, 'zlib\nbusybox')
File "/usr/lib/python3.12/unittest/case.py", line 885, in assertEqual
assertion_func(first, second, msg=msg)
File "/usr/lib/python3.12/unittest/case.py", line 1251, in assertMultiLineEqual
self.fail(self._formatMessage(msg, standardMsg))
File "/usr/lib/python3.12/unittest/case.py", line 715, in fail
raise self.failureException(msg)
AssertionError: 'zlib' != 'zlib\nbusybox'
zlib
+ busybox
https://autobuilder.yoctoproject.org/valkyrie/#/builders/35/builds/3792
https://autobuilder.yoctoproject.org/valkyrie/#/builders/48/builds/3665
Can you have a look at the issue?
Thanks,
Mathieu
Signature · 3 lines
Mathieu Dubois-Briand, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
PATCH v4 1/3 oe-pkgdata-util: fix empty runtime-rprovides directory handling
In reply to Sam KentAn empty runtime-provides directory caused lookup-recipe, package-info and list-pkg-files to skip the runtime-reverse fallback.
Use os.listdir() to ensure the folder is not empty and use os.path.isdir( to ensure it is not a file.
--- scripts/oe-pkgdata-util | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
scripts/oe-pkgdata-util +9 -6
@@ -289,8 +289,9 @@ def lookup_recipe(args): for pkg in pkgs: providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg) - if os.path.exists(providepkgpath): - for f in os.listdir(providepkgpath): + rprovides = os.listdir(providepkgpath) if os.path.isdir(providepkgpath) else [] + if rprovides: + for f in rprovides: if f != pkg: print("%s is in the RPROVIDES of %s:" % (pkg, f)) pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
@@ -355,8 +356,9 @@ def package_info(args): for pkg in packages: providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg) - if os.path.exists(providepkgpath): - for f in os.listdir(providepkgpath): + rprovides = os.listdir(providepkgpath) if os.path.isdir(providepkgpath) else [] + if rprovides: + for f in rprovides: if f != pkg: print("%s is in the RPROVIDES of %s:" % (pkg, f)) pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
@@ -507,8 +509,9 @@ def list_pkg_files(args): else: providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg) - if os.path.exists(providepkgpath): - for f in os.listdir(providepkgpath): + rprovides = os.listdir(providepkgpath) if os.path.isdir(providepkgpath) else [] + if rprovides: + for f in rprovides: if f != pkg: print("%s is in the RPROVIDES of %s:" % (pkg, f)) pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
Signature · 1 line
2.34.1
PATCH v4 2/3 oelib: add unit tests for kernel module detection helpers
Add tests for the filename pre-filter logic, is_kernel_module(), and is_kernel_module_signed() to meta/lib/oeqa/selftest/cases/oelib/
--- meta/lib/oeqa/selftest/cases/oelib/package.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 meta/lib/oeqa/selftest/cases/oelib/package.py
meta/lib/oeqa/selftest/cases/oelib/package.py new file +108 -0
@@ -0,0 +1,108 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import mmap +import os +import tempfile +from unittest.case import TestCase + + +class TestKernelModuleFilenameFilter(TestCase): + """ + The pre-filter in process_split_and_strip_files() selects candidates to + pass to is_elf(). It must use f.endswith(".ko"), not ".ko" in f, to avoid + false-positives on compressed modules (.ko.xz, .ko.gz). + """ + + ko_files = [ + "driver.ko", + "net/foo.ko", + ] + + not_ko_files = [ + "driver.ko.xz", + "driver.ko.gz", + "driver.ko2", + "myko.c", + "vmlinux", + ] + + def test_endswith_matches_ko(self): + for f in self.ko_files: + with self.subTest(f=f): + self.assertTrue(f.endswith(".ko")) + + def test_endswith_rejects_non_ko(self): + for f in self.not_ko_files: + with self.subTest(f=f): + self.assertFalse(f.endswith(".ko")) + + def test_old_predicate_had_false_positives(self): + # The previous check (".ko" in f) matched compressed modules — this is + # the regression the fix addresses. + false_positives = [f for f in self.not_ko_files if ".ko" in f] + self.assertEqual(false_positives, ["driver.ko.xz", "driver.ko.gz", "driver.ko2"]) + + +class TestIsKernelModule(TestCase): + """ + is_kernel_module() detects kernel modules by searching for the + "vermagic=" string, which is always present in genuine .ko files. + """ + + def setUp(self): + from oe.package import is_kernel_module + self.is_kernel_module = is_kernel_module + self._tmpfile = None + + def tearDown(self): + if self._tmpfile and os.path.exists(self._tmpfile): + os.unlink(self._tmpfile) + + def _make_tmp(self, content): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def test_detects_vermagic(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 10 + b"vermagic=5.15.0" + b"\x00" * 10) + self.assertTrue(self.is_kernel_module(self._tmpfile)) + + def test_rejects_plain_elf(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 50) + self.assertFalse(self.is_kernel_module(self._tmpfile)) + + +class TestIsKernelModuleSigned(TestCase): + """ + is_kernel_module_signed() detects the "Module signature appended" tail + that the kernel's modsign infrastructure writes. + """ + + def setUp(self): + from oe.package import is_kernel_module_signed + self.is_kernel_module_signed = is_kernel_module_signed + self._tmpfile = None + + def tearDown(self): + if self._tmpfile and os.path.exists(self._tmpfile): + os.unlink(self._tmpfile) + + def _make_tmp(self, content): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def test_detects_signed(self): + tail = b"Module signature appended\n\x00\x00" + self._tmpfile = self._make_tmp(b"\x00" * 64 + tail) + self.assertTrue(self.is_kernel_module_signed(self._tmpfile)) + + def test_rejects_unsigned(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 20) + self.assertFalse(self.is_kernel_module_signed(self._tmpfile))
Signature · 1 line
2.34.1
PATCH v4 3/3 oeqa/selftest: add oe-selftest for kernel module pre-filter
In reply to Sam KentAdd selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify process_split_and_strip_files() handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.
AI-Generated: Uses Claude Sonnet 4.6
--- .../selftest-ko-filter/files/module.c | 3 ++ .../selftest-ko-filter/selftest-ko-filter.bb | 29 +++++++++++++++++++ meta/lib/oeqa/selftest/cases/package.py | 22 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/files/module.c create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb
meta-selftest/recipes-test/selftest-ko-filter/files/module.c new file +3 -0
@@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: MIT */ +/* Minimal relocatable object used as a .ko stand-in for pre-filter testing. */ +int selftest_ko_filter_marker =42 ;
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +29 -0
@@ -0,0 +1,29 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +SUMMARY = "Test fixture for the kernel-module file pre-filter in package.py" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://module.c" +S = "${UNPACKDIR}" + +MODDIR = "${libdir}/selftest-ko-filter" + +do_compile () { + ${CC} ${CFLAGS} -c module.c -o module.ko +} + +do_install () { + install -d ${D}${MODDIR} + install -m 0644 module.ko ${D}${MODDIR}/ + + # Fake compressed modules — the pre-filter must not pass these to strip. + # Use octal escapes: dash (the OE recipe shell) does not support \xHH in printf. + printf '\375\067\172\130\132\000' > ${D}${MODDIR}/module.ko.xz + printf '\037\213' > ${D}${MODDIR}/module.ko.gz +} + +FILES:${PN} = "${MODDIR}/*"
meta/lib/oeqa/selftest/cases/package.py +22 -0
@@ -185,6 +185,28 @@ class PackageTests(OESelftestTestCase): if not gdbtest(qemu, binary): self.fail('GDB %s failed' % binary) + def test_kmodule_prefilter(self): + # Regression test for YOCTO #2348: process_split_and_strip_files() must + # use f.endswith(".ko") so that compressed modules (.ko.xz, .ko.gz) are + # not fed to is_elf() / strip. + bitbake("selftest-ko-filter -c package") + + pkgdest = get_bb_var('PKGDEST', 'selftest-ko-filter') + libdir = get_bb_var('libdir', 'selftest-ko-filter') + moddir = pkgdest + "/selftest-ko-filter" + libdir + "/selftest-ko-filter" + + self.assertTrue(os.path.exists(moddir + "/module.ko"), + "module.ko missing from PKGDEST") + + for fname, magic in [("module.ko.xz", b"\xfd\x37\x7a\x58\x5a\x00"), + ("module.ko.gz", b"\x1f\x8b")]: + path = moddir + "/" + fname + self.assertTrue(os.path.exists(path), + "%s missing from PKGDEST" % fname) + with open(path, "rb") as f: + self.assertEqual(f.read(len(magic)), magic, + "%s header corrupted in PKGDEST" % fname) + def test_preserve_ownership(self): features = 'IMAGE_INSTALL:append = " selftest-chown"\n' self.write_config(features)
Signature · 1 line
2.34.1
Patchtest results for [PATCH v4 1/3] oe-pkgdata-util: fix empty runtime-rprovides directory handling
In reply to Sam KentThank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information:
---
Testing patch /home/patchtest/share/mboxes/v4-1-3-oe-pkgdata-util-fix-empty-runtime-rprovides-directory-handling.patch
FAIL: test Signed-off-by presence: Mbox is missing Signed-off-by. Add it manually or with "git commit --amend -s" (test_mbox.TestMbox.test_signed_off_by_presence)
PASS: test author valid (test_mbox.TestMbox.test_author_valid)
PASS: test commit message presence (test_mbox.TestMbox.test_commit_message_presence)
PASS: test commit message user tags (test_mbox.TestMbox.test_commit_message_user_tags)
PASS: test mbox format (test_mbox.TestMbox.test_mbox_format)
PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade)
PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format)
PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length)
PASS: test target mailing list (test_mbox.TestMbox.test_target_mailing_list)
SKIP: pretest pylint: No python related patches, skipping test (test_python_pylint.PyLint.pretest_pylint)
SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format)
SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence)
SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format)
SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format)
SKIP: test pylint: No python related patches, skipping test (test_python_pylint.PyLint.test_pylint)
SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head)
---
Please address the issues identified and
submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you!
PATCH v5 1/3 oe-pkgdata-util: fix empty runtime-rprovides directory handling
In reply to Sam KentAn empty runtime-provides directory caused lookup-recipe, package-info and list-pkg-files to skip the runtime-reverse fallback.
Use os.listdir() to ensure the folder is not empty and use os.path.isdir( to ensure it is not a file.
--- scripts/oe-pkgdata-util | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
scripts/oe-pkgdata-util +9 -6
@@ -289,8 +289,9 @@ def lookup_recipe(args): for pkg in pkgs: providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg) - if os.path.exists(providepkgpath): - for f in os.listdir(providepkgpath): + rprovides = os.listdir(providepkgpath) if os.path.isdir(providepkgpath) else [] + if rprovides: + for f in rprovides: if f != pkg: print("%s is in the RPROVIDES of %s:" % (pkg, f)) pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
@@ -355,8 +356,9 @@ def package_info(args): for pkg in packages: providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg) - if os.path.exists(providepkgpath): - for f in os.listdir(providepkgpath): + rprovides = os.listdir(providepkgpath) if os.path.isdir(providepkgpath) else [] + if rprovides: + for f in rprovides: if f != pkg: print("%s is in the RPROVIDES of %s:" % (pkg, f)) pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
@@ -507,8 +509,9 @@ def list_pkg_files(args): else: providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg) - if os.path.exists(providepkgpath): - for f in os.listdir(providepkgpath): + rprovides = os.listdir(providepkgpath) if os.path.isdir(providepkgpath) else [] + if rprovides: + for f in rprovides: if f != pkg: print("%s is in the RPROVIDES of %s:" % (pkg, f)) pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
Signature · 1 line
2.34.1
PATCH v5 2/3 oelib: add unit tests for kernel module detection helpers
Add tests for the filename pre-filter logic, is_kernel_module(), and is_kernel_module_signed() to meta/lib/oeqa/selftest/cases/oelib/
--- meta/lib/oeqa/selftest/cases/oelib/package.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 meta/lib/oeqa/selftest/cases/oelib/package.py
meta/lib/oeqa/selftest/cases/oelib/package.py new file +108 -0
@@ -0,0 +1,108 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import mmap +import os +import tempfile +from unittest.case import TestCase + + +class TestKernelModuleFilenameFilter(TestCase): + """ + The pre-filter in process_split_and_strip_files() selects candidates to + pass to is_elf(). It must use f.endswith(".ko"), not ".ko" in f, to avoid + false-positives on compressed modules (.ko.xz, .ko.gz). + """ + + ko_files = [ + "driver.ko", + "net/foo.ko", + ] + + not_ko_files = [ + "driver.ko.xz", + "driver.ko.gz", + "driver.ko2", + "myko.c", + "vmlinux", + ] + + def test_endswith_matches_ko(self): + for f in self.ko_files: + with self.subTest(f=f): + self.assertTrue(f.endswith(".ko")) + + def test_endswith_rejects_non_ko(self): + for f in self.not_ko_files: + with self.subTest(f=f): + self.assertFalse(f.endswith(".ko")) + + def test_old_predicate_had_false_positives(self): + # The previous check (".ko" in f) matched compressed modules — this is + # the regression the fix addresses. + false_positives = [f for f in self.not_ko_files if ".ko" in f] + self.assertEqual(false_positives, ["driver.ko.xz", "driver.ko.gz", "driver.ko2"]) + + +class TestIsKernelModule(TestCase): + """ + is_kernel_module() detects kernel modules by searching for the + "vermagic=" string, which is always present in genuine .ko files. + """ + + def setUp(self): + from oe.package import is_kernel_module + self.is_kernel_module = is_kernel_module + self._tmpfile = None + + def tearDown(self): + if self._tmpfile and os.path.exists(self._tmpfile): + os.unlink(self._tmpfile) + + def _make_tmp(self, content): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def test_detects_vermagic(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 10 + b"vermagic=5.15.0" + b"\x00" * 10) + self.assertTrue(self.is_kernel_module(self._tmpfile)) + + def test_rejects_plain_elf(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 50) + self.assertFalse(self.is_kernel_module(self._tmpfile)) + + +class TestIsKernelModuleSigned(TestCase): + """ + is_kernel_module_signed() detects the "Module signature appended" tail + that the kernel's modsign infrastructure writes. + """ + + def setUp(self): + from oe.package import is_kernel_module_signed + self.is_kernel_module_signed = is_kernel_module_signed + self._tmpfile = None + + def tearDown(self): + if self._tmpfile and os.path.exists(self._tmpfile): + os.unlink(self._tmpfile) + + def _make_tmp(self, content): + f = tempfile.NamedTemporaryFile(delete=False, suffix=".ko") + f.write(content) + f.close() + return f.name + + def test_detects_signed(self): + tail = b"Module signature appended\n\x00\x00" + self._tmpfile = self._make_tmp(b"\x00" * 64 + tail) + self.assertTrue(self.is_kernel_module_signed(self._tmpfile)) + + def test_rejects_unsigned(self): + self._tmpfile = self._make_tmp(b"\x7fELF\x00" * 20) + self.assertFalse(self.is_kernel_module_signed(self._tmpfile))
Signature · 1 line
2.34.1
PATCH v5 3/3 oeqa/selftest: add oe-selftest for kernel module pre-filter
In reply to Sam KentAdd selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify process_split_and_strip_files() handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.
AI-Generated: Uses Claude Sonnet 4.6
--- .../selftest-ko-filter/files/module.c | 3 ++ .../selftest-ko-filter/selftest-ko-filter.bb | 29 +++++++++++++++++++ meta/lib/oeqa/selftest/cases/package.py | 22 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/files/module.c create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb
meta-selftest/recipes-test/selftest-ko-filter/files/module.c new file +3 -0
@@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: MIT */ +/* Minimal relocatable object used as a .ko stand-in for pre-filter testing. */ +int selftest_ko_filter_marker =42 ;
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +29 -0
@@ -0,0 +1,29 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +SUMMARY = "Test fixture for the kernel-module file pre-filter in package.py" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://module.c" +S = "${UNPACKDIR}" + +MODDIR = "${libdir}/selftest-ko-filter" + +do_compile () { + ${CC} ${CFLAGS} -c module.c -o module.ko +} + +do_install () { + install -d ${D}${MODDIR} + install -m 0644 module.ko ${D}${MODDIR}/ + + # Fake compressed modules — the pre-filter must not pass these to strip. + # Use octal escapes: dash (the OE recipe shell) does not support \xHH in printf. + printf '\375\067\172\130\132\000' > ${D}${MODDIR}/module.ko.xz + printf '\037\213' > ${D}${MODDIR}/module.ko.gz +} + +FILES:${PN} = "${MODDIR}/*"
meta/lib/oeqa/selftest/cases/package.py +22 -0
@@ -185,6 +185,28 @@ class PackageTests(OESelftestTestCase): if not gdbtest(qemu, binary): self.fail('GDB %s failed' % binary) + def test_kmodule_prefilter(self): + # Regression test for YOCTO #2348: process_split_and_strip_files() must + # use f.endswith(".ko") so that compressed modules (.ko.xz, .ko.gz) are + # not fed to is_elf() / strip. + bitbake("selftest-ko-filter -c package") + + pkgdest = get_bb_var('PKGDEST', 'selftest-ko-filter') + libdir = get_bb_var('libdir', 'selftest-ko-filter') + moddir = pkgdest + "/selftest-ko-filter" + libdir + "/selftest-ko-filter" + + self.assertTrue(os.path.exists(moddir + "/module.ko"), + "module.ko missing from PKGDEST") + + for fname, magic in [("module.ko.xz", b"\xfd\x37\x7a\x58\x5a\x00"), + ("module.ko.gz", b"\x1f\x8b")]: + path = moddir + "/" + fname + self.assertTrue(os.path.exists(path), + "%s missing from PKGDEST" % fname) + with open(path, "rb") as f: + self.assertEqual(f.read(len(magic)), magic, + "%s header corrupted in PKGDEST" % fname) + def test_preserve_ownership(self): features = 'IMAGE_INSTALL:append = " selftest-chown"\n' self.write_config(features)
Signature · 1 line
2.34.1
Re: [PATCH v5 3/3] oeqa/selftest: add oe-selftest for kernel module pre-filter
Quoted 47 lines · Sam Kent
Add selftest-ko-filter recipe and a PackageTests.test_kmodule_prefilter oe-selftest that builds it to verify
process_split_and_strip_files()handles .ko.xz/.ko.gz alongside a real .ko without corrupting them. Regression test for YOCTO #2348.AI-Generated: Uses Claude Sonnet 4.6
Signed-off-by Sam Kent <[email protected]>--- .../selftest-ko-filter/files/module.c | 3 ++ .../selftest-ko-filter/selftest-ko-filter.bb | 29 +++++++++++++++++++ meta/lib/oeqa/selftest/cases/package.py | 22 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/files/module.c create mode 100644 meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bbmeta-selftest/recipes-test/selftest-ko-filter/files/module.c new file +3
-0@@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: MIT */ +/* Minimal relocatable object used as a .ko stand-in for pre-filter testing. */ +int selftest_ko_filter_marker =42 ;meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +16
-0@@ -0,0 +1,29 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# +SUMMARY = "Test fixture for the kernel-module file pre-filter in package.py" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://module.c" +S = "${UNPACKDIR}" + +MODDIR = "${libdir}/selftest-ko-filter" + +do_compile () { + ${CC} ${CFLAGS} -c module.c -o module.ko
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb line 26
Quoted 10 lines
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +10
-0+} + +do_install () { + install -d ${D}${MODDIR} + install -m 0644 module.ko ${D}${MODDIR}/ + + # Fake compressed modules — the pre-filter must not pass these to strip. + # Use octal escapes: dash (the OE recipe shell) does not support \xHH in printf. + printf '\375\067\172\130\132\000' > ${D}${MODDIR}/module.ko.xz + printf '\037\213' > ${D}${MODDIR}/module.ko.gz
Hi Sam,
Why are you writing bytes via printf here instead of using gzip and xz to compess module.ko? You don't necessarily need to change this, but please explain it in the commit message or comment.
Quoted 16 lines
meta-selftest/recipes-test/selftest-ko-filter/selftest-ko-filter.bb new file +3
-0+} + +FILES:${PN} = "${MODDIR}/*"meta/lib/oeqa/selftest/cases/package.py +5
-0@@ -185,6 +185,28 @@ class PackageTests(OESelftestTestCase): if not gdbtest(qemu, binary): self.fail('GDB %s failed' % binary) + def test_kmodule_prefilter(self): + # Regression test for YOCTO #2348: process_split_and_strip_files() must + # use f.endswith(".ko") so that compressed modules (.ko.xz, .ko.gz) are + # not fed to is_elf() / strip. + bitbake("selftest-ko-filter -c package")
meta/lib/oeqa/selftest/cases/package.py class PackageTests(OESelftestTestCase): · line 202
Quoted 10 lines
meta/lib/oeqa/selftest/cases/package.py +10
-0+ + pkgdest = get_bb_var('PKGDEST', 'selftest-ko-filter') + libdir = get_bb_var('libdir', 'selftest-ko-filter') + moddir = pkgdest + "/selftest-ko-filter" + libdir + "/selftest-ko-filter" + + self.assertTrue(os.path.exists(moddir + "/module.ko"), + "module.ko missing from PKGDEST") + + for fname, magic in [("module.ko.xz", b"\xfd\x37\x7a\x58\x5a\x00"), + ("module.ko.gz", b"\x1f\x8b")]:
I find it very confusing that we have octal escapes in do_install() and hex escapes here. If we have to use octal escapes due to dash limitations, let's also use octal escapes here.
meta/lib/oeqa/selftest/cases/package.py class PackageTests(OESelftestTestCase): · line 212
Quoted 10 lines
meta/lib/oeqa/selftest/cases/package.py +7
-0+ path = moddir + "/" + fname + self.assertTrue(os.path.exists(path), + "%s missing from PKGDEST" % fname) + with open(path, "rb") as f: + self.assertEqual(f.read(len(magic)), magic, + "%s header corrupted in PKGDEST" % fname) + def test_preserve_ownership(self): features = 'IMAGE_INSTALL:append = " selftest-chown"\n' self.write_config(features)
Best regards,
Signature · 1 line
Paul Barker