python3-kirk: Add version 4.0.0
PATCH 1/2 python3-kirk: Add version 4.0.0
From: Daniel Turull <[email protected]>
Kirk application is a fork of runltp-ng and it’s the official LTP tests executor.
--- meta/conf/distro/include/maintainers.inc | 1 + .../python3-kirk/python3-kirk_4.0.0.bb | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 meta/recipes-extended/python3-kirk/python3-kirk_4.0.0.bb
meta/conf/distro/include/maintainers.inc +1 -0
@@ -665,6 +665,7 @@ RECIPE_MAINTAINER:pn-python3-jinja2 = "Richard Purdie <richard.purdie@linuxfound RECIPE_MAINTAINER:pn-python3-jsonpointer = "Bruce Ashfield <[email protected]>" RECIPE_MAINTAINER:pn-python3-jsonschema = "Bruce Ashfield <[email protected]>" RECIPE_MAINTAINER:pn-python3-jsonschema-specifications = "Bruce Ashfield <[email protected]>" +RECIPE_MAINTAINER:pn-python3-kirk = "Daniel Turull <[email protected]>" RECIPE_MAINTAINER:pn-python3-libarchive-c = "Joshua Watt <[email protected]>" RECIPE_MAINTAINER:pn-python3-license-expression = "Wang Mingyu <[email protected]>" RECIPE_MAINTAINER:pn-python3-lxml = "Khem Raj <[email protected]>"
meta/recipes-extended/python3-kirk/python3-kirk_4.0.0.bb new file +18 -0
@@ -0,0 +1,18 @@ +SUMMARY = "Linux Test Project tests executor" +HOMEPAGE = "https://github.com/linux-test-project/kirk" +DESCRIPTION = "Kirk application is a fork of runltp-ng and it’s the official \ +LTP tests executor. It provides support for remote testing via Qemu, SSH, \ +LTX, parallel execution and much more." +SECTION = "console/utils" +LICENSE = "GPL-2.0-only" +LIC_FILES_CHKSUM = "file://LICENSE;md5=39bba7d2cf0ba1036f2a6e2be52fe3f0" + +SRC_URI = "git://github.com/linux-test-project/kirk;protocol=https;branch=master" + +PV = "4.0.0" +SRCREV = "32b224021b9c657953f5b2f61ca4744b9b3c5961" + +RDEPENDS:${PN} += "python3-asyncio python3-json" + +inherit python_setuptools_build_meta +
PATCH 2/2 oeqa: replace runltp with kirk
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
See:
https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b
--- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb | 1 + 4 files changed, 50 insertions(+), 12 deletions(-)
meta/lib/oeqa/runtime/cases/ltp.py +8 -8
@@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase):
@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200)
@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime)
@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg)
meta/lib/oeqa/runtime/cases/ltp_stress.py +12 -4
@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase):
@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time()
@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtime
meta/lib/oeqa/utils/logparser.py +29 -0
@@ -5,6 +5,7 @@ # import enum +import json import os import re
@@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsing class LtpComplianceParser(object): def __init__(self):
meta/recipes-extended/ltp/ltp_20260130.bb +1 -0
@@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
meta/lib/oeqa/runtime/cases/ltp_stress.py class LtpStressTest(LtpStressBase): · line 81
Quoted 12 lines · Daniel Turull via lists.openembedded.org
meta/lib/oeqa/runtime/cases/ltp_stress.py +10
-2- parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst)
Why is an extra copying step added?
Alex
RE: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 28 lines
-----Original Message-----
From: Alexander Kanavin <[email protected]>
Sent: Wednesday, 25 March 2026 13:39
To: Daniel Turull <[email protected]>
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
Review
Quoted 2 lines · Daniel Turull via lists.openembedded.org
(quoted diff) +0
-2- parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" %
stress_group))meta/lib/oeqa/runtime/cases/ltp_stress.py class LtpStressTest(LtpStressBase): · line 81
Quoted 10 lines
meta/lib/oeqa/runtime/cases/ltp_stress.py +10
-0+ # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst)
Quoted 1 line
Why is an extra copying step added?
Kirk creates a json file with the results that it is more machine readable friendly and needs to be copied over from the target. The old code was having the output results in the log.
Quoted 2 lines
Alex
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
In reply to [email protected]Quoted 4 lines · Daniel Turull via lists.openembedded.org
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
For something like this to be considered for merging, I'd have expected a lot more information.
Is the test coverage the same as before ?
What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etc
Switching the executable is the easy part, ensuring that it actually works a replacement is the hard part.
Bruce
Quoted 200 lines
See:
https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b
Signed-off-by Daniel Turull <[email protected]>Assisted-by Claude, Anthropic--- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb | 1 + 4 files changed, 50 insertions(+), 12 deletions(-)meta/lib/oeqa/runtime/cases/ltp.py +1
-1@@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase):@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a cleanlog
meta/lib/oeqa/runtime/cases/ltp.py +1
-1- self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" %
ltp_group)- cmd = '
/opt/ltp/runltp-f %s -q -r/opt/ltp-l
/opt/ltp/results/%s -I 1 -d/opt/ltp' % (ltp_group,ltp_group)
+ cmd = 'kirk --run-suite %s --json-report
/opt/ltp/results/%s.json -n' % (ltp_group,ltp_group)starttime =
time.time()
(status, output) =self.target.run(cmd, timeout=1200)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] =self.extras['ltpresult.rawlogs']['log'] + output
# Copy the machine-readable test results locally so we can
parse itmeta/lib/oeqa/runtime/cases/ltp.py +1
-2- dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" %
ltp_group)meta/lib/oeqa/runtime/cases/ltp.py +4
-3+ remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime)@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg)meta/lib/oeqa/runtime/cases/ltp_stress.py +2
-2@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase):@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time()(status, output) =
self.target.run(cmd)endtime =time.time()meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] =self.extras['ltpstressresult.rawlogs']['log'] + output
meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-2- parser = LtpParser() - results, sections =
parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))meta/lib/oeqa/runtime/cases/ltp_stress.py +2
-0+ # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" %
stress_group)meta/lib/oeqa/runtime/cases/ltp_stress.py +8
-0+ remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtimemeta/lib/oeqa/utils/logparser.py +29
-0@@ -5,6 +5,7 @@ # import enum +import json import os import re@@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsingclass LtpComplianceParser(object):
def __init__(self):meta/recipes-extended/ltp/ltp_20260130.bb +1
-1@@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \ -=-=-=-=-=-=-=-=-=-=-=-Links: You receive all messages sent to this group. View/Reply Online (#233871):
https://lists.openembedded.org/g/openembedded-core/message/233871
Mute This Topic: https://lists.openembedded.org/mt/118498904/1050810
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
- Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end - "Use the force Harry" - Gandalf, Star Trek II
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 12 lines · Bruce Ashfield via lists.openembedded.org
For something like this to be considered for merging, I'd have expected a lot more information.
Is the test coverage the same as before ?
What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etcSwitching the executable is the easy part, ensuring that it actually works a replacement is the hard part.
Quoted 1 line
Assisted-by Claude, Anthropic
I have to add that this line ^^^ doesn't inspire confidence in the patch either. On the contrary, it raises suspicions that the code was written quickly after the issue was raised yesterday in the tech meeting, and wasn't actually tested well or at all. The commit message *must* reassure that it was.
Alex
RE: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Hi,
I added for transparency the "assisted by". I have tested the patch with the ltp math test set to be able to do verify it's output. It generates two files: a raw file with the log and a json file. The default output from kirk is a json file with the results, instead of dumping them into a stdout.
I'm attaching the output with the patches:
ls tmp/work/qemux86_64-oe-linux/core-image-minimal/1.0/testimage/ltp_log/ math.json math-raw.log
I'll modify the commit message to reflect that.
Daniel
Quoted 34 lines
-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Alexander Kanavin via lists.openembedded.org
Sent: Wednesday, 25 March 2026 14:02
To: [email protected]
Cc: Daniel Turull <[email protected]>; openembedded- [email protected]; [email protected]; [email protected]Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 12 lines · Bruce Ashfield via lists.openembedded.org
For something like this to be considered for merging, I'd have expected a lot more information.
Is the test coverage the same as before ?
What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etcSwitching the executable is the easy part, ensuring that it actually works a replacement is the hard part.
Quoted 1 line
Assisted-by Claude, AnthropicI have to add that this line ^^^ doesn't inspire confidence in the patch either. On the contrary, it raises suspicions that the code was written quickly after the issue was raised yesterday in the tech meeting, and wasn't actually tested well or at all. The commit message
*must* reassure that it was.Alex
RE: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
In reply to Bruce AshfieldHi,
Kirk is official way to run ltp since runltp has been dropped. Richard mentioned yesterday in the Weekly Project Engineering Sync if no one was fixing the runltp issue, he will drop ltp from oe-core.
The test coverage is the same since the logic there is not touched but the output is different. Now it is a json file with the results, which is machine readable without any extra parsing. Do we need a conversion from the new to the old to keep compatibility?
I’m attaching the old test output with the new one for the math test suite.
New: math-raw.log and math.json
Old: math.old math-raw.log.old.
You can see at the end of the old output:
-------------------------------------------
INFO: runltp script is deprecated, try kirk
https://github.com/linux-test-project/kirk
-------------------------------------------
Best regards,
Daniel
From: Bruce Ashfield <[email protected]>
Sent: Wednesday, 25 March 2026 13:59
To: Daniel Turull <[email protected]>
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
On Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org<http://lists.openembedded.org/> <[email protected]<mailto:[email protected]>> wrote:
From: Daniel Turull <[email protected]<mailto:[email protected]>>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
For something like this to be considered for merging, I'd have expected a lot more information.
Is the test coverage the same as before ?
What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etc
Switching the executable is the easy part, ensuring that it actually works a replacement is the hard part.
Bruce
See:
https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b
--- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> | 1 + 4 files changed, 50 insertions(+), 12 deletions(-)
meta/lib/oeqa/runtime/cases/ltp.py +8 -8
@@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase):
@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200)
@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime)
@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg)
meta/lib/oeqa/runtime/cases/ltp_stress.py +12 -4
@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase):
@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time()
@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtime
meta/lib/oeqa/utils/logparser.py +29 -0
@@ -5,6 +5,7 @@ # import enum +import json import os import re
@@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsing class LtpComplianceParser(object): def __init__(self):
meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> +1 -0
@@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#233871): https://lists.openembedded.org/g/openembedded-core/message/233871
Mute This Topic: https://lists.openembedded.org/mt/118498904/1050810
Group Owner: [email protected]<mailto:openembedded-core%[email protected]>
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]<mailto:[email protected]>]
-=-=-=-=-=-=-=-=-=-=-=-
--
- Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end
- "Use the force Harry" - Gandalf, Star Trek II
Host information
Hostname: qemux86-64 Python: 3.14.3 (main, Feb 3 2026, 15:32:20) [GCC 15.2.0] Directory: /tmp/kirk.root/tmp02mw5o2n
Connecting to SUT: default
Suite: math
─────────── abs01: pass (0.003s) atof01: pass (0.003s) float_bessel: pass (0.504s) float_exp_log: pass (0.304s) float_iperb: pass (0.129s) float_power: pass (0.246s) float_trigo: pass (0.295s) fptest01: pass (0.002s) fptest02: pass (0.002s) nextafter01: pass (0.002s)
Execution time: 1.615s
Disconnecting from SUT: default
Target information
──────────────────
Kernel: Linux 6.18.13-yocto-standard #1 SMP PREEMPT_DYNAMIC Tue Mar 3 16:48:55 UTC 2026
Cmdline: root=/dev/vda
rw
ip=192.168.7.2::192.168.7.1:255.255.255.0::eth0:off:8.8.8.8
net.ifnames=0
console=ttyS0
console=ttyS1
oprofile.timer=1
tsc=reliable
no_timer_check
rcupdate.rcu_expedited=1
swiotlb=0
printk.time=1
Machine: unknown
Arch: x86_64
RAM: 222368 kB
Swap: 0 kB
Distro: nodistro nodistro.0────────────────────────
TEST SUMMARY
──────────────────────── Suite: math Runtime: 1.489s Runs: 10
Results:
Passed: 22
Failed: 0
Broken: 0
Skipped: 0Warnings: 0
Session stopped
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 10 lines · Daniel Turull
Hi,
Kirk is official way to run ltp since runltp has been dropped. Richard mentioned yesterday in the Weekly Project Engineering Sync if no one was fixing the runltp issue, he will drop ltp from oe-core.
Yes, we know this, it's been mentioned for months. We are fundamentally too late for the LTS release in my opinion, since Richard would likely be left fixing issues with LTP failures that could gate the release.
It isn't even the results that are the issue. How will this perform on a fully loaded machine (i.e the auto builder), will we get inconsistent results ? Intermittent failures ? Different results based on the architecture ?
Quoted 5 lines
The test coverage is the same since the logic there is not touched but the output is different. Now it is a json file with the results, which is machine readable without any extra parsing. Do we need a conversion from the new to the old to keep compatibility?
Only if it impacts the reset of the QA or Autobuilder testing. If everything in the test stack continues to work with the new format, I'd say it is ok as-is.
Quoted 4 lines
I’m attaching the old test output with the new one for the math test suite.
Quoted 8 lines
New: math-raw.log and math.json
Old: math.old math-raw.log.old.
You can see at the end of the old output:
This all needs to be in the commit itself. As well as a results summary from all the supported primary architectures.
Bruce
Quoted 279 lines
-------------------------------------------
INFO: runltp script is deprecated, try kirk
https://github.com/linux-test-project/kirk
-------------------------------------------
Best regards,
Daniel
*From:* Bruce Ashfield <[email protected]>
*Sent:* Wednesday, 25 March 2026 13:59
*To:* Daniel Turull <[email protected]>
*Cc:* [email protected]; [email protected]; [email protected]
*Subject:* Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirkOn Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org <[email protected]> wrote:
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
For something like this to be considered for merging, I'd have expected
a lot more information.
Is the test coverage the same as before ?
What are the results of the tests versus the previous harness ?
Is it truly a drop in replacement for the previous runner ?
.. etc
Switching the executable is the easy part, ensuring that it actually works
a replacement is the hard part.
Bruce
See:
https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b
Signed-off-by Daniel Turull <[email protected]>Assisted-by Claude, Anthropic--- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb | 1 + 4 files changed, 50 insertions(+), 12 deletions(-)meta/lib/oeqa/runtime/cases/ltp.py +1
-1@@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase):@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a cleanlog
meta/lib/oeqa/runtime/cases/ltp.py +1
-1- self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" %
ltp_group)- cmd = '
/opt/ltp/runltp-f %s -q -r/opt/ltp-l
/opt/ltp/results/%s -I 1 -d/opt/ltp' % (ltp_group,ltp_group)
+ cmd = 'kirk --run-suite %s --json-report
/opt/ltp/results/%s.json -n' % (ltp_group,ltp_group)starttime =
time.time()
(status, output) =self.target.run(cmd, timeout=1200)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] =self.extras['ltpresult.rawlogs']['log'] + output
# Copy the machine-readable test results locally so we can
parse itmeta/lib/oeqa/runtime/cases/ltp.py +1
-2- dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" %
ltp_group)meta/lib/oeqa/runtime/cases/ltp.py +4
-3+ remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime)@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg)meta/lib/oeqa/runtime/cases/ltp_stress.py +2
-2@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase):@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time()(status, output) =
self.target.run(cmd)endtime =time.time()meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] =self.extras['ltpstressresult.rawlogs']['log'] + output
meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-2- parser = LtpParser() - results, sections =
parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))meta/lib/oeqa/runtime/cases/ltp_stress.py +2
-0+ # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" %
stress_group)meta/lib/oeqa/runtime/cases/ltp_stress.py +8
-0+ remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtimemeta/lib/oeqa/utils/logparser.py +29
-0@@ -5,6 +5,7 @@ # import enum +import json import os import re@@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsingclass LtpComplianceParser(object):
def __init__(self):meta/recipes-extended/ltp/ltp_20260130.bb +1
-1@@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \ -=-=-=-=-=-=-=-=-=-=-=-Links: You receive all messages sent to this group. View/Reply Online (#233871):
https://lists.openembedded.org/g/openembedded-core/message/233871
Mute This Topic: https://lists.openembedded.org/mt/118498904/1050810
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=---
- Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end
- "Use the force Harry" - Gandalf, Star Trek II
Signature · 3 lines
- Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end - "Use the force Harry" - Gandalf, Star Trek II
RE: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Ok. I’ll run it with the supported architectures and update the commit message with the information.
Daniel
From: [email protected] <[email protected]> On Behalf Of Bruce Ashfield via lists.openembedded.org
Sent: Wednesday, 25 March 2026 15:22
To: Daniel Turull <[email protected]>
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
On Wed, Mar 25, 2026 at 9:44 AM Daniel Turull <[email protected]<mailto:[email protected]>> wrote:
Hi,
Kirk is official way to run ltp since runltp has been dropped. Richard mentioned yesterday in the Weekly Project Engineering Sync if no one was fixing the runltp issue, he will drop ltp from oe-core.
Yes, we know this, it's been mentioned for months. We are fundamentally too late for the LTS release in my opinion, since Richard would likely be left fixing issues with LTP failures that could gate the release.
It isn't even the results that are the issue. How will this perform on a fully loaded machine (i.e the auto builder), will we get inconsistent results ? Intermittent failures ? Different results based on the architecture ?
The test coverage is the same since the logic there is not touched but the output is different. Now it is a json file with the results, which is machine readable without any extra parsing. Do we need a conversion from the new to the old to keep compatibility?
Only if it impacts the reset of the QA or Autobuilder testing. If everything in the test stack continues to work with the new format, I'd say it is ok as-is.
I’m attaching the old test output with the new one for the math test suite.
New: math-raw.log and math.json
Old: math.old math-raw.log.old.
You can see at the end of the old output:
This all needs to be in the commit itself. As well as a results summary from all the supported primary architectures.
Bruce
-------------------------------------------
INFO: runltp script is deprecated, try kirk
https://github.com/linux-test-project/kirk
-------------------------------------------
Best regards,
Daniel
From: Bruce Ashfield <[email protected]<mailto:[email protected]>>
Sent: Wednesday, 25 March 2026 13:59
To: Daniel Turull <[email protected]<mailto:[email protected]>>
Cc: [email protected]<mailto:[email protected]>; [email protected]<mailto:[email protected]>; [email protected]<mailto:[email protected]>
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
On Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org<http://lists.openembedded.org/> <[email protected]<mailto:[email protected]>> wrote:
From: Daniel Turull <[email protected]<mailto:[email protected]>>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
For something like this to be considered for merging, I'd have expected a lot more information.
Is the test coverage the same as before ?
What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etc
Switching the executable is the easy part, ensuring that it actually works a replacement is the hard part.
Bruce
See:
https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b
--- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> | 1 + 4 files changed, 50 insertions(+), 12 deletions(-)
meta/lib/oeqa/runtime/cases/ltp.py +8 -8
@@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase):
@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200)
@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime)
@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg)
meta/lib/oeqa/runtime/cases/ltp_stress.py +12 -4
@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase):
@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time()
@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtime
meta/lib/oeqa/utils/logparser.py +29 -0
@@ -5,6 +5,7 @@ # import enum +import json import os import re
@@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsing class LtpComplianceParser(object): def __init__(self):
meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> +1 -0
@@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \
--
- Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end
- "Use the force Harry" - Gandalf, Star Trek II
--
- Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end
- "Use the force Harry" - Gandalf, Star Trek II
Re: [PATCH 2/2] oeqa: replace runltp with kirk
In reply to [email protected]Quoted 147 lines · On Wed, 2026-03-25 at 09:40 +0100, [email protected] wrote:
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
See:
https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1bSigned-off-by Daniel Turull <[email protected]>Assisted-by Claude, Anthropic---
meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++-------meta/lib/oeqa/runtime/cases/ltp_stress.py| 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb | 1 +4 files changed, 50 insertions(+), 12 deletions(-)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -12,7 +12,7 @@ import pprintfrom oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage
meta/lib/oeqa/runtime/cases/ltp.py +1
-1-from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser
class LtpTestBase(OERuntimeTestCase):meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase):
defrunltp(self, ltp_group):
# LTP appends to log files, so ensure we start with a clean log
-self.target.deleteFiles("/opt/ltp/results/", ltp_group)
+self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group)
- cmd = '/opt/ltp/runltp-f %s -q -r/opt/ltp-l /opt/ltp/results/%s -I 1 -d/opt/ltp' % (ltp_group,ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group,ltp_group)
starttime =time.time()
(status, output) =self.target.run(cmd, timeout=1200)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase):self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output
# Copy the machine-readable test results locally so we can parse it
- dst =os.path.join(self.ltptest_log_dir, ltp_group)
-remote_src= "/opt/ltp/results/%s" %ltp_group
+ dst =os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group)
+remote_src= "/opt/ltp/results/%s.json" %ltp_group
(status, output) =self.target.copyFrom(remote_src, dst, True)
if status:
msg = 'File could not be copied. Output: %s' % outputself.target.logger.warning(msg)
- parser =LtpParser()
+ parser =LtpKirkParser()
results, sections =parser.parse(dst)
sections['duration'] = int(endtime-starttime)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase):
# LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh'])
- @OEHasPackage(["ltp"])
+ @OEHasPackage(["ltp", "python3-kirk"])
deftest_ltp_help(self):
- (status, output) =self.target.run('/opt/ltp/runltp --help')+ (status, output) =self.target.run('kirk --help')msg = 'Failed to get ltp help. Output: %s' % outputself.assertEqual(status, 0, msg=msg)meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCasefrom oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu
meta/lib/oeqa/runtime/cases/ltp_stress.py +1
-1-from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser
class LtpStressBase(OERuntimeTestCase):meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase):class LtpStressTest(LtpStressBase):
defrunltp(self, stress_group):
- cmd = '/opt/ltp/runltp-f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group,stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group,stress_group) starttime =time.time()
(status, output) =self.target.run(cmd)
endtime =time.time()meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase):
self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output
- parser =LtpParser()
- results, sections =parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))+ # Copy kirk JSON report from target
+ dst =os.path.join(self.ltptest_log_dir, "%s.json" % stress_group)
+remote_src= "/opt/ltp/results/%s.json" %stress_group
+ (status, output) =self.target.copyFrom(remote_src, dst, True)
+ if status:
+ msg = 'File could not be copied. Output: %s' % output
+self.target.logger.warning(msg)
+
+ parser =LtpKirkParser()
+ results, sections =parser.parse(dst)
runtime = int(endtime-starttime)
sections['duration'] = runtimemeta/lib/oeqa/utils/logparser.py +0
-0@@ -5,6 +5,7 @@ #
import enum
+import json
import os
import remeta/lib/oeqa/utils/logparser.py +0
-0@@ -158,6 +159,34 @@ class LtpParser:return results, section
meta/lib/oeqa/utils/logparser.py +18
-0+class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} +
meta/lib/oeqa/utils/logparser.py class LtpParser: · line 189
Quoted 10 lines
meta/lib/oeqa/utils/logparser.py +10
-0+ for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + +
FWIW we can probably just update the LtpParser class directly rather than creating a wrapper around it since runltp will be no more in the next release anyway.
Cheers,
Richard
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Sure,
I’ll try to send a new patch tomorrow with your suggestion and all the comments that I have received. We are running the testimage with ltp for qemuarm64, qemux86-64 and qemuriscv64.
Best regards,
Daniel
________________________________
From: [email protected] <[email protected]> on behalf of Richard Purdie via lists.openembedded.org <[email protected]>
Sent: Wednesday, March 25, 2026 5:36 PM
To: Daniel Turull <[email protected]>; [email protected] <[email protected]>
Cc: [email protected] <[email protected]>
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 147 lines · On Wed, 2026-03-25 at 09:40 +0100, [email protected] wrote:
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
See:
https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Flinux-test-project%2Fltp%2Fcommit%2F6efd3605dc005c3ed135b463f182174e24bdce1b&data=05%7C02%7Cdaniel.turull%40ericsson.com%7Cca4e7536591a44feb4fd08de8a8c9c1f%7C92e84cebfbfd47abbe52080c6b87953f%7C0%7C0%7C639100533691606542%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=cgqr%2FrR99Ty%2F75pasLCYawpw0qV%2FAZ4sXUDWH%2B2ka9c%3D&reserved=0<https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b>Signed-off-by Daniel Turull <[email protected]>Assisted-by Claude, Anthropic--- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb | 1 + 4 files changed, 50 insertions(+), 12 deletions(-)meta/lib/oeqa/runtime/cases/ltp.py +8
-8@@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase):@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200)@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime)@@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg)meta/lib/oeqa/runtime/cases/ltp_stress.py +12
-4@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase):@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time()@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtimemeta/lib/oeqa/utils/logparser.py +19
-0@@ -5,6 +5,7 @@ # import enum +import json import os import re@@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} +
meta/lib/oeqa/utils/logparser.py class LtpParser: · line 189
Quoted 10 lines
meta/lib/oeqa/utils/logparser.py +10
-0+ for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + +
FWIW we can probably just update the LtpParser class directly rather than creating a wrapper around it since runltp will be no more in the next release anyway.
Cheers,
Richard
RE: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
In reply to Richard PurdieJust an update.
It is taking a bit more time than expected. My original test only covered the math test suite in ltp to verify the functionality.
Some of the ltp test are failing with OOM, some related to systemd-udev. I'll be disabling them now to have a working ltp. Then look at the underlaying issues.
So far I have found 4 failing test cases. Using the default config without any changes with qemux86-64
min_free_kbytes (mm) (OOM)
pty07 (pty) (OOM)
ptem02 (pty) (OOM)
cve-2018-13405 (cve) (deadlock kernel panic)
Giving enough memory 16GB makes them go away but I'm trying to make it work with 4GB of RAM.
Best regards,
Daniel
Quoted 213 lines
-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Richard Purdie via lists.openembedded.org
Sent: Wednesday, 25 March 2026 17:36
To: Daniel Turull <[email protected]>; openembedded- [email protected]
Cc: [email protected]
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirkQuoted 8 lines · On Wed, 2026-03-25 at 09:40 +0100, [email protected] wrote:
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
See:
https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgith
ub.com%2Flinux-test-project%2Fltp%2Fcommit%2F6efd3605dc005c3ed135b463f
Quoted 1 line
182174e24bdce1b&data=05%7C02%7Cdaniel.turull%40ericsson.com%7Cca4e75 36
Quoted 1 line
591a44feb4fd08de8a8c9c1f%7C92e84cebfbfd47abbe52080c6b87953f%7C0%7C 0%7C
Quoted 1 line
639100533691606542%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRy dWUsIlY
Quoted 1 line
iOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C 0%
Quoted 1 line
7C%7C%7C&sdata=cgqr%2FrR99Ty%2F75pasLCYawpw0qV%2FAZ4sXUDWH%2B2 ka9c%3D&
Quoted 160 lines
reserved=0
Signed-off-by Daniel Turull <[email protected]>Assisted-by Claude, Anthropic---
meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++-------meta/lib/oeqa/runtime/cases/ltp_stress.py| 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29
+++++++++++++++++++++++
meta/recipes-extended/ltp/ltp_20260130.bb | 1 +4 files changed, 50 insertions(+), 12 deletions(-)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -12,7 +12,7 @@ import pprintfrom oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser
+from oeqa.utils.logparser import LtpKirkParserclass LtpTestBase(OERuntimeTestCase):
meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase):def
runltp(self, ltp_group):
# LTP appends to log files, so ensure we start with a clean log
-self.target.deleteFiles("/opt/ltp/results/", ltp_group)+self.target.deleteFiles("/opt/ltp/results/", "%s.json" % +ltp_group)- cmd = '
/opt/ltp/runltp-f %s -q -r/opt/ltp-l /opt/ltp/results/%s -I 1 -d/opt/ltp' % (ltp_group,ltp_group) + cmd = 'kirk --run-suite %s --json-report +/opt/ltp/results/%s.json -n' % (ltp_group,ltp_group)starttime =
time.time()
(status, output) =self.target.run(cmd, timeout=1200)@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase):
self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output# Copy the machine-readable test results locally so we can parse it
- dst =os.path.join(self.ltptest_log_dir, ltp_group)-remote_src= "/opt/ltp/results/%s" %ltp_group+ dst =os.path.join(self.ltptest_log_dir, "%s.json" % +ltp_group)
+remote_src= "/opt/ltp/results/%s.json" %ltp_group(status, output) =self.target.copyFrom(remote_src, dst, True)
if status:
msg = 'File could not be copied. Output: %s' % outputself.target.logger.warning(msg)- parser =
LtpParser()
+ parser =LtpKirkParser()results, sections =parser.parse(dst)sections['duration'] = int(endtime-starttime) @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase):
# LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh'])
- @OEHasPackage(["ltp"])
+ @OEHasPackage(["ltp", "python3-kirk"])
deftest_ltp_help(self):
- (status, output) =self.target.run('/opt/ltp/runltp --help')+ (status, output) =self.target.run('kirk --help')msg = 'Failed to get ltp help. Output: %s' % outputself.assertEqual(status, 0, msg=msg)meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCasefrom oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser
+from oeqa.utils.logparser import LtpKirkParserclass LtpStressBase(OERuntimeTestCase):
meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase):class LtpStressTest(LtpStressBase):
def
runltp(self, stress_group):
- cmd = '/opt/ltp/runltp-f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group,stress_group) + cmd = 'kirk --run-suite %s --json-report +/opt/ltp/results/%s.json -n' % (stress_group,stress_group) starttime =time.time()
(status, output) =self.target.run(cmd)endtime =time.time()meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase):self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output
- parser =
LtpParser()
- results, sections =parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))+ # Copy kirk JSON report from target
+ dst = os.path.join(self.ltptest_log_dir, "%s.json" %meta/lib/oeqa/runtime/cases/ltp_stress.py +10
-0+stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, +True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst)runtime = int(endtime-starttime) sections['duration'] = runtime diff --git
a/meta/lib/oeqa/utils/logparser.pyb/meta/lib/oeqa/utils/logparser.pyindexc479864162..a907421fab100644meta/lib/oeqa/utils/logparser.py +0
-0@@ -5,6 +5,7 @@ #import enum
+import json
import os
import remeta/lib/oeqa/utils/logparser.py +0
-0@@ -158,6 +159,34 @@ class LtpParser:return results, section
meta/lib/oeqa/utils/logparser.py +20
-0+class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] =meta/lib/oeqa/utils/logparser.py class LtpParser: · line 189
Quoted 10 lines
meta/lib/oeqa/utils/logparser.py +10
-0+self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", +{}).get("runtime", 0)) + + return results, section + +FWIW we can probably just update the LtpParser class directly rather than creating a wrapper around it since runltp will be no more in the next release anyway.
Cheers,
Richard
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 9 lines · Daniel Turull via lists.openembedded.org
Just an update.
It is taking a bit more time than expected. My original test only covered the math test suite in ltp to verify the functionality.
Some of the ltp test are failing with OOM, some related to systemd-udev. I'll be disabling them now to have a working ltp. Then look at the underlaying issues.
So far I have found 4 failing test cases. Using the default config without any changes with qemux86-64
min_free_kbytes(mm) (OOM)
This seems to be a buggy testcase for automation. Disable until its fixed. Even if the test own memory consumers sets their own OOM-score higher, there are still chances that the OOM-killer kill the wrong things.
Quoted 2 lines
pty07 (pty) (OOM)
ptem02 (pty) (OOM)
These seem to me to be related to systemd-udevd having unbounded message queue sizes to its udev workers. IMO, this is a systemd-udevd issue. Some other udev implementations starts dropping events.
We should have seen this when using runltp as well ?
Death spiral: systemd_259.5 1. Test tight loop creating devices floods systemd-udevd's workers unbounded inbox queue(s). - Single core, low mem, and long running udev rules makes the problem worse.
2. systemd-udevd grows to consume all available RAM
3. OOM killer kills everything but systemd-udevd (OOMScoreAdjust=-1000)
4. Kernel panic: "no killable processes"
Quoted 221 lines
cve-2018-13405 (cve) (deadlock kernel panic)
Giving enough memory 16GB makes them go away but I'm trying to make it work with 4GB of RAM.
Best regards,
DanielQuoted 213 lines
-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Richard Purdie via lists.openembedded.org
Sent: Wednesday, 25 March 2026 17:36
To: Daniel Turull <[email protected]>; openembedded- [email protected]
Cc: [email protected]
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirkQuoted 8 lines · On Wed, 2026-03-25 at 09:40 +0100, [email protected] wrote:
From: Daniel Turull <[email protected]>
runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests.
See:
https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgith
ub.com%2Flinux-test-project%2Fltp%2Fcommit%2F6efd3605dc005c3ed135b463f
Quoted 1 line
182174e24bdce1b&data=05%7C02%7Cdaniel.turull%40ericsson.com%7Cca4e75 36
Quoted 1 line
591a44feb4fd08de8a8c9c1f%7C92e84cebfbfd47abbe52080c6b87953f%7C0%7C 0%7C
Quoted 1 line
639100533691606542%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRy dWUsIlY
Quoted 1 line
iOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C 0%
Quoted 1 line
7C%7C%7C&sdata=cgqr%2FrR99Ty%2F75pasLCYawpw0qV%2FAZ4sXUDWH%2B2 ka9c%3D&
Quoted 160 lines
reserved=0
Signed-off-by Daniel Turull <[email protected]>Assisted-by Claude, Anthropic---
meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++-------meta/lib/oeqa/runtime/cases/ltp_stress.py| 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29
+++++++++++++++++++++++
meta/recipes-extended/ltp/ltp_20260130.bb | 1 +4 files changed, 50 insertions(+), 12 deletions(-)meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -12,7 +12,7 @@ import pprintfrom oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser
+from oeqa.utils.logparser import LtpKirkParserclass LtpTestBase(OERuntimeTestCase):
meta/lib/oeqa/runtime/cases/ltp.py +0
-0@@ -66,9 +66,9 @@ class LtpTest(LtpTestBase):def
runltp(self, ltp_group):
# LTP appends to log files, so ensure we start with a clean log
-self.target.deleteFiles("/opt/ltp/results/", ltp_group)+self.target.deleteFiles("/opt/ltp/results/", "%s.json" % +ltp_group)- cmd = '
/opt/ltp/runltp-f %s -q -r/opt/ltp-l /opt/ltp/results/%s -I 1 -d/opt/ltp' % (ltp_group,ltp_group) + cmd = 'kirk --run-suite %s --json-report +/opt/ltp/results/%s.json -n' % (ltp_group,ltp_group)starttime =
time.time()
(status, output) =self.target.run(cmd, timeout=1200)@@ -87,14 +87,14 @@ class LtpTest(LtpTestBase):
self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output# Copy the machine-readable test results locally so we can parse it
- dst =os.path.join(self.ltptest_log_dir, ltp_group)-remote_src= "/opt/ltp/results/%s" %ltp_group+ dst =os.path.join(self.ltptest_log_dir, "%s.json" % +ltp_group)
+remote_src= "/opt/ltp/results/%s.json" %ltp_group(status, output) =self.target.copyFrom(remote_src, dst, True)
if status:
msg = 'File could not be copied. Output: %s' % outputself.target.logger.warning(msg)- parser =
LtpParser()
+ parser =LtpKirkParser()results, sections =parser.parse(dst)sections['duration'] = int(endtime-starttime) @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase):
# LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh'])
- @OEHasPackage(["ltp"])
+ @OEHasPackage(["ltp", "python3-kirk"])
deftest_ltp_help(self):
- (status, output) =self.target.run('/opt/ltp/runltp --help')+ (status, output) =self.target.run('kirk --help')msg = 'Failed to get ltp help. Output: %s' % outputself.assertEqual(status, 0, msg=msg)meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCasefrom oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser
+from oeqa.utils.logparser import LtpKirkParserclass LtpStressBase(OERuntimeTestCase):
meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase):class LtpStressTest(LtpStressBase):
def
runltp(self, stress_group):
- cmd = '/opt/ltp/runltp-f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group,stress_group) + cmd = 'kirk --run-suite %s --json-report +/opt/ltp/results/%s.json -n' % (stress_group,stress_group) starttime =time.time()
(status, output) =self.target.run(cmd)endtime =time.time()meta/lib/oeqa/runtime/cases/ltp_stress.py +0
-0@@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase):self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output
- parser =
LtpParser()
- results, sections =parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))+ # Copy kirk JSON report from target
+ dst = os.path.join(self.ltptest_log_dir, "%s.json" %meta/lib/oeqa/runtime/cases/ltp_stress.py +10
-0+stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, +True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst)runtime = int(endtime-starttime) sections['duration'] = runtime diff --git
a/meta/lib/oeqa/utils/logparser.pyb/meta/lib/oeqa/utils/logparser.pyindexc479864162..a907421fab100644meta/lib/oeqa/utils/logparser.py +0
-0@@ -5,6 +5,7 @@ #import enum
+import json
import os
import remeta/lib/oeqa/utils/logparser.py +0
-0@@ -158,6 +159,34 @@ class LtpParser:return results, section
meta/lib/oeqa/utils/logparser.py +20
-0+class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] =meta/lib/oeqa/utils/logparser.py class LtpParser: · line 189
Quoted 10 lines
meta/lib/oeqa/utils/logparser.py +10
-0+self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", +{}).get("runtime", 0)) + + return results, section + +FWIW we can probably just update the LtpParser class directly rather than creating a wrapper around it since runltp will be no more in the next release anyway.
Cheers,
Richard
Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Quoted 33 lines · David Nyström
Quoted 5 lines · Daniel Turull via lists.openembedded.org
Just an update.
It is taking a bit more time than expected. My original test only covered the math test suite in ltp to verify the functionality.
Quoted 8 lines
Some of the ltp test are failing with OOM, some related to systemd- udev. I'll be disabling them now to have a working ltp. Then look at the underlaying issues.
So far I have found 4 failing test cases. Using the default config without any changes with qemux86-64
min_free_kbytes(mm) (OOM)This seems to be a buggy testcase for automation. Disable until its fixed.
Even if the test own memory consumers sets their own OOM-score higher,
there are still chances that the OOM-killer kill the wrong things.Quoted 2 lines
pty07 (pty) (OOM)
ptem02 (pty) (OOM)These seem to me to be related to systemd-udevd having unbounded message
queue sizes to its udev workers. IMO, this is a systemd-udevd issue. Some other udev implementations starts dropping events.We should have seen this when using runltp as well ?
Quoted 9 lines
Death spiral: systemd_259.5
1. Test tight loop creating devices floods systemd-udevd's workers unbounded inbox queue(s).
- Single core, low mem, and long running udev rules makes the problem worse.
2. systemd-udevd grows to consume all available RAM
3. OOM killer kills everything but systemd-udevd (OOMScoreAdjust=- 1000)
4. Kernel panic: "no killable processes"
Are the systemd developers aware of this?
You're correct that the ltp testing as currently run does sometimes seem to have issues and not all tests pass. It is something we've been wanting to look into and resolve but simply haven't had time.
Finding an issue like the systemd one does hint that there may be value in some of the testing.
Ideally we'd identify and disable the problematic tests, then we would know any failures were new and potential issues. The new version of the patch series from Daniel does move us forward with that, thanks!
Cheers,
Richard
RE: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk
Hi,
David open an issue with them now.
https://github.com/systemd/systemd/issues/41372
I used AI to generate a fix for it this afternoon and it seems to be working but I hold a bit with it to fully verify it. There is memory leak in the systemd-udev when a lot of devices are created in a short time.
Daniel
Quoted 71 lines
-----Original Message-----
From: Richard Purdie <[email protected]>
Sent: Friday, 27 March 2026 16:41
To: David Nyström <[email protected]>; Daniel Turull <[email protected]>
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirkQuoted 32 lines · David Nyström
Quoted 5 lines · Daniel Turull via lists.openembedded.org
Just an update.
It is taking a bit more time than expected. My original test only covered the math test suite in ltp to verify the functionality.
Quoted 8 lines
Some of the ltp test are failing with OOM, some related to systemd- udev. I'll be disabling them now to have a working ltp. Then look at the underlaying issues.
So far I have found 4 failing test cases. Using the default config without any changes with qemux86-64
min_free_kbytes(mm) (OOM)This seems to be a buggy testcase for automation. Disable until its fixed.
Even if the test own memory consumers sets their own OOM-score higher, there are still chances that the OOM-killer kill the wrong things.Quoted 2 lines
pty07 (pty) (OOM)
ptem02 (pty) (OOM)These seem to me to be related to systemd-udevd having unbounded message queue sizes to its udev workers. IMO, this is a systemd-udevd issue.
Some other udev implementations starts dropping events.We should have seen this when using runltp as well ?
Quoted 9 lines
Death spiral: systemd_259.5
1. Test tight loop creating devices floods systemd-udevd's workers unbounded inbox queue(s).
- Single core, low mem, and long running udev rules makes the problem worse.
2. systemd-udevd grows to consume all available RAM 3. OOM killer kills everything but systemd-udevd (OOMScoreAdjust=-
1000)
4. Kernel panic: "no killable processes"Are the systemd developers aware of this?
You're correct that the ltp testing as currently run does sometimes seem to have issues and not all tests pass. It is something we've been wanting to look into and resolve but simply haven't had time.
Finding an issue like the systemd one does hint that there may be value in some of the testing.
Ideally we'd identify and disable the problematic tests, then we would know any failures were new and potential issues. The new version of the patch series from Daniel does move us forward with that, thanks!
Cheers,
Richard
Quoted 1 line · Daniel Turull via lists.openembedded.org
+SRC_URI = "
git://github.com/linux-test-project/kirk;protocol=https;branch=master"
We probably should take kirk from pypi via pypi.bbclass?
https://pypi.org/project/kirk/
Alex
Hi,
I'll update the recipe to use pypi instead.
Daniel
Quoted 27 lines
-----Original Message-----
From: Alexander Kanavin <[email protected]>
Sent: Wednesday, 25 March 2026 13:35
To: Daniel Turull <[email protected]>
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [OE-core] [PATCH 1/2] python3-kirk: Add version 4.0.0[You don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
Quoted 1 line · Daniel Turull via lists.openembedded.org
+SRC_URI = "
git://github.com/linux-test-project/kirk;protocol=https;branch=master"
We probably should take kirk from pypi via pypi.bbclass?
https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpypi.org%
2Fproject%2Fkirk%2F&data=05%7C02%7Cdaniel.turull%40ericsson.com%7C498a 872696cc4f8be26a08de8a6aeb53%7C92e84cebfbfd47abbe52080c6b87953f%7C 0%7C0%7C639100388978726482%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1 hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIj oyfQ%3D%3D%7C60000%7C%7C%7C&sdata=lXs9Cx0YxirCnYstgQRJUJ3j%2FrYvgz C2LkZ0OgTbsao%3D&reserved=0Alex