Skip to contentSkip to conversation

kunit: convert test results to JSON

PATCH In review Heidi Fahim 2 participants 2 messages Mar 25, 2020 – Mar 30, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 kunit: convert test results to JSON

Add a --json flag, which when specified when kunit_tool is run, calls method get_json_result. This is a method within kunit_json.py that formats KUnit results into a dict conforming to the following KernelCI API test_group spec:
https://api.kernelci.org/schema-test-group.html#post. The user can specify a filename as the value to json in order to store the JSON results under linux/.
Tested within kunit_tool_test.py in a new test case called KUnitJsonTest.

---
 tools/testing/kunit/kunit.py                  |  24 ++++++-
 tools/testing/kunit/kunit_json.py             |  63 ++++++++++++++++++
 tools/testing/kunit/kunit_tool_test.py        |  33 +++++++++
 .../kunit/test_data/test_pound_sign.log       | Bin 0 -> 1656 bytes
 4 files changed, 118 insertions(+), 2 deletions(-)
 create mode 100644 tools/testing/kunit/kunit_json.py
tools/testing/kunit/kunit.py +22 -2
index 7dca74774dd2..ce8f8e5e0ccb 100755
@@ -17,6 +17,7 @@ from collections import namedtuple
 from enum import Enum, auto
 
 import kunit_config
+import kunit_json
 import kunit_kernel
 import kunit_parser
 
@@ -24,10 +25,11 @@ KunitResult = namedtuple('KunitResult', ['status','result'])
 
 KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',
 					   'build_dir', 'defconfig',
-					   'alltests', 'make_options'])
+					   'alltests', 'make_options', 'json'])
 
 KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
 
+
 class KunitStatus(Enum):
 	SUCCESS = auto()
 	CONFIG_FAILURE = auto()
@@ -70,6 +72,7 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree,
 	kunit_output = linux.run_kernel(
 		timeout=None if request.alltests else request.timeout,
 		build_dir=request.build_dir)
+
 	if request.raw_output:
 		raw_output = kunit_parser.raw_output(kunit_output)
 		isolated = list(kunit_parser.isolate_kunit_output(raw_output))
@@ -86,6 +89,15 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree,
 				build_end - build_start,
 				test_end - test_start))
 
+	if request.json:
+		json_obj = kunit_json.get_json_result(
+					test_result=test_result,
+					def_config='kunit_defconfig',
+					build_dir=request.build_dir,
+					json_path=request.json)
+		if request.json == 'stdout':
+			print(json_obj)
+
 	if test_result.status != kunit_parser.TestStatus.SUCCESS:
 		return KunitResult(KunitStatus.TEST_FAILURE, test_result)
 	else:
@@ -130,6 +142,13 @@ def main(argv, linux=None):
 				help='X=Y make option, can be repeated.',
 				action='append')
 
+	run_parser.add_argument('--json',
+				nargs='?',
+				help='Stores test results in a JSON, and either '
+				'prints to stdout or saves to file if a '
+				'filename is specified',
+				const='stdout')
+
 	cli_args = parser.parse_args(argv)
 
 	if cli_args.subcommand == 'run':
@@ -155,7 +174,8 @@ def main(argv, linux=None):
 				       cli_args.build_dir,
 				       cli_args.defconfig,
 				       cli_args.alltests,
-				       cli_args.make_options)
+				       cli_args.make_options,
+				       cli_args.json)
 		result = run_tests(linux, request)
 		if result.status != KunitStatus.SUCCESS:
 			sys.exit(1)
tools/testing/kunit/kunit_json.py new file +63 -0

63 lines not on this page · raw message

tools/testing/kunit/kunit_tool_test.py +33 -0
index ce47e87b633a..94e8a295d466 100755
@@ -11,11 +11,13 @@ from unittest import mock
 
 import tempfile, shutil # Handling test_tmpdir
 
+import json
 import os
 
 import kunit_config
 import kunit_parser
 import kunit_kernel
+import kunit_json
 import kunit
 
 test_tmpdir = ''
@@ -219,6 +221,37 @@ class KUnitParserTest(unittest.TestCase):
 			result = kunit_parser.parse_run_tests(file.readlines())
 		self.assertEqual('kunit-resource-test', result.suites[0].name)
 
+class KUnitJsonTest(unittest.TestCase):
+
+	def _json_for(self, log_file):
+		with(open(get_absolute_path(log_file))) as file:
+			test_result = kunit_parser.parse_run_tests(file)
+			json_obj = kunit_json.get_json_result(
+				test_result=test_result,
+				def_config='kunit_defconfig',
+				build_dir=None,
+				json_path='stdout')
+		return json.loads(json_obj)
+
+	def test_failed_test_json(self):
+		result = self._json_for(
+			'test_data/test_is_test_passed-failure.log')
+		self.assertEqual(
+			{'name': 'example_simple_test', 'status': 'FAIL'},
+			result["sub_groups"][1]["test_cases"][0])
+
+	def test_crashed_test_json(self):
+		result = self._json_for(
+			'test_data/test_is_test_passed-crash.log')
+		self.assertEqual(
+			{'name': 'example_simple_test', 'status': 'ERROR'},
+			result["sub_groups"][1]["test_cases"][0])
+
+	def test_no_tests_json(self):
+		result = self._json_for(
+			'test_data/test_is_test_passed-no_tests_run.log')
+		self.assertEqual(0, len(result['sub_groups']))
+
 class StrContains(str):
 	def __eq__(self, other):
 		return self in other
tools/testing/kunit/test_data/test_pound_sign.log binary +0 -0
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..28ffa5ba03bfa81ea02ea9d38e7de7acf3dd9e5d 100644
GIT binary patch
Signature · 1 line
2.25.1.696.g5e7596f4ac-goog
Reviewed-by on v1 1/1
#1 lore raw
Quoted 11 lines · Heidi Fahim

Add a --json flag, which when specified when kunit_tool is run, calls method get_json_result. This is a method within kunit_json.py that formats KUnit results into a dict conforming to the following KernelCI API test_group spec:
https://api.kernelci.org/schema-test-group.html#post. The user can specify a filename as the value to json in order to store the JSON results under linux/.
Tested within kunit_tool_test.py in a new test case called KUnitJsonTest.

Quoted 1 line