resulttool: Improvements to allow integration to the autobuilder
PATCH 1/2 resulttool: Improvements to allow integration to the autobuilder
This is a combined patch of the various tweaks and improvements I made to resulttool:
* Avoid subprocess.run() as its a python 3.6 feature and we have autobuilder workers with 3.5.
* Avoid python keywords as variable names
* Simplify dict accesses using .get()
* Rename resultsutils -> resultutils to match the resultstool -> resulttool rename
* Formalised the handling of "file_name" to "TESTSERIES" which the code will now add into the json configuration data if its not present, based on the directory name.
* When we don't have failed test cases, print something saying so instead of an empty table
* Tweak the table headers in the report to be more readable (reference "Test Series" instead if file_id and ID instead of results_id)
* Improve/simplify the max string length handling
* Merge the counts and percentage data into one table in the report since printing two reports of the same data confuses the user
* Removed the confusing header in the regression report
* Show matches, then regressions, then unmatched runs in the regression report, also remove chatting unneeded output
* Try harder to "pair" up matching configurations to reduce noise in the regressions report
* Abstracted the "mapping" table concept used to pairing in the regression code to general code in resultutils
* Created multiple mappings for results analysis, results storage and 'flattening' results data in a merge
* Simplify the merge command to take a source and a destination, letting the destination be a directory or a file, removing the need for an output directory parameter
* Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression mappings
* Have the store command place the testresults files in a layout from the mapping, making commits into the git repo for results storage more useful for simple comparison purposes
* Set the oe-git-archive tag format appropriately for oeqa results storage (and simplify the commit messages closer to their defaults)
* Fix oe-git-archive to use the commit/branch data from the results file
* Cleaned up the command option help to match other changes
* Follow the model of git branch/tag processing used by oe-build-perf-report and use that to read the data using git show to avoid branch change
* Add ptest summary to the report command
* Update the tests to match the above changes
--- .../oeqa/selftest/cases/resulttooltests.py | 106 +++--- scripts/lib/resulttool/manualexecution.py | 8 +- scripts/lib/resulttool/merge.py | 69 ++-- scripts/lib/resulttool/regression.py | 328 +++++++++--------- scripts/lib/resulttool/report.py | 157 +++++---- scripts/lib/resulttool/resultsutils.py | 67 ---- scripts/lib/resulttool/resultutils.py | 127 +++++++ scripts/lib/resulttool/store.py | 136 ++++---- .../template/test_report_full_text.txt | 33 +- scripts/resulttool | 7 +- 10 files changed, 530 insertions(+), 508 deletions(-) delete mode 100644 scripts/lib/resulttool/resultsutils.py create mode 100644 scripts/lib/resulttool/resultutils.py
meta/lib/oeqa/selftest/cases/resulttooltests.py +48 -58
@@ -4,13 +4,46 @@ basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../') lib_path = basepath + '/scripts/lib' sys.path = sys.path + [lib_path] from resulttool.report import ResultsTextReport -from resulttool.regression import ResultsRegressionSelector, ResultsRegression -from resulttool.merge import ResultsMerge -from resulttool.store import ResultsGitStore -from resulttool.resultsutils import checkout_git_dir +from resulttool import regression as regression +from resulttool import resultutils as resultutils from oeqa.selftest.case import OESelftestTestCase class ResultToolTests(OESelftestTestCase): + base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"}, + 'result': {}}, + 'base_result2': {'configuration': {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86-64"}, + 'result': {}}} + target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"}, + 'result': {}}, + 'target_result2': {'configuration': {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"}, + 'result': {}}, + 'target_result3': {'configuration': {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86-64"}, + 'result': {}}} def test_report_can_aggregate_test_result(self): result_data = {'result': {'test1': {'status': 'PASSED'},
@@ -25,23 +58,12 @@ class ResultToolTests(OESelftestTestCase): self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped']) def test_regression_can_get_regression_base_target_pair(self): - base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "oeselftest", - "HOST": "centos-7"}}, - 'base_result2': {'configuration': {"TEST_TYPE": "oeselftest", - "HOST": "centos-7", - "MACHINE": "qemux86-64"}}} - target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "oeselftest", - "HOST": "centos-7"}}, - 'target_result2': {'configuration': {"TEST_TYPE": "oeselftest", - "HOST": "centos-7", - "MACHINE": "qemux86"}}, - 'target_result3': {'configuration': {"TEST_TYPE": "oeselftest", - "HOST": "centos-7", - "MACHINE": "qemux86-64"}}} - regression = ResultsRegressionSelector() - pair = regression.get_regression_base_target_pair(self.logger, base_results_data, target_results_data) - self.assertTrue('target_result1' in pair['base_result1'], msg="Pair not correct:%s" % pair['base_result1']) - self.assertTrue('target_result3' in pair['base_result2'], msg="Pair not correct:%s" % pair['base_result2']) + + results = {} + resultutils.append_resultsdata(results, ResultToolTests.base_results_data) + resultutils.append_resultsdata(results, ResultToolTests.target_results_data) + self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results) + self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results) def test_regrresion_can_get_regression_result(self): base_result_data = {'result': {'test1': {'status': 'PASSED'},
@@ -54,8 +76,7 @@ class ResultToolTests(OESelftestTestCase): 'test3': {'status': 'PASSED'}, 'test4': {'status': 'ERROR'}, 'test5': {'status': 'SKIPPED'}}} - regression = ResultsRegression() - result = regression.get_regression_result(self.logger, base_result_data, target_result_data) + result, text = regression.compare_result(self.logger, "BaseTestRunName", "TargetTestRunName", base_result_data, target_result_data) self.assertTrue(result['test2']['base'] == 'PASSED', msg="regression not correct:%s" % result['test2']['base']) self.assertTrue(result['test2']['target'] == 'FAILED',
@@ -66,39 +87,8 @@ class ResultToolTests(OESelftestTestCase): msg="regression not correct:%s" % result['test3']['target']) def test_merge_can_merged_results(self): - base_results_data = {'base_result1': {}, - 'base_result2': {}} - target_results_data = {'target_result1': {}, - 'target_result2': {}, - 'target_result3': {}} - - merge = ResultsMerge() - results = merge.merge_results(base_results_data, target_results_data) - self.assertTrue(len(results.keys()) == 5, msg="merge not correct:%s" % len(results.keys())) - - def test_store_can_store_to_new_git_repository(self): - basepath = os.path.abspath(os.path.dirname(__file__) + '/../../') - source_dir = basepath + '/files/testresults' - git_branch = 'qa-cycle-2.7' - store = ResultsGitStore() - output_dir = store.store_to_new(self.logger, source_dir, git_branch) - self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to new git repository failed:%s" % - output_dir) - store._remove_temporary_workspace_dir(output_dir) + results = {} + resultutils.append_resultsdata(results, ResultToolTests.base_results_data, configmap=resultutils.flatten_map) + resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) + self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) - def test_store_can_store_to_existing(self): - basepath = os.path.abspath(os.path.dirname(__file__) + '/../../') - source_dir = basepath + '/files/testresults' - git_branch = 'qa-cycle-2.6' - store = ResultsGitStore() - output_dir = store.store_to_new(self.logger, source_dir, git_branch) - self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to new git repository failed:%s" % - output_dir) - git_branch = 'qa-cycle-2.7' - output_dir = store.store_to_existing_with_new_branch(self.logger, source_dir, output_dir, git_branch) - self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to existing git repository failed:%s" % - output_dir) - output_dir = store.store_to_existing(self.logger, source_dir, output_dir, git_branch) - self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to existing git repository failed:%s" % - output_dir) - store._remove_temporary_workspace_dir(output_dir)
scripts/lib/resulttool/manualexecution.py +6 -2
@@ -18,7 +18,11 @@ import sys import datetime import re from oeqa.core.runner import OETestResultJSONHelper -from resulttool.resultsutils import load_json_file + +def load_json_file(file): + with open(file, "r") as f: + return json.load(f) + class ManualTestRunner(object): def __init__(self):
@@ -134,4 +138,4 @@ def register_commands(subparsers): description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/', group='manualexecution') parser_build.set_defaults(func=manualexecution) - parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.') + parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
scripts/lib/resulttool/merge.py +20 -49
87 lines not on this page · raw message
scripts/lib/resulttool/regression.py +156 -172
359 lines not on this page · raw message
scripts/lib/resulttool/report.py +89 -68
195 lines not on this page · raw message
scripts/lib/resulttool/resultsutils.py deleted +0 -67
68 lines not on this page · raw message
scripts/lib/resulttool/resultutils.py new file +127 -0
127 lines not on this page · raw message
scripts/lib/resulttool/store.py +59 -77
165 lines not on this page · raw message
scripts/lib/resulttool/template/test_report_full_text.txt +21 -12
@@ -1,35 +1,44 @@ ============================================================================================================== -Test Report (Count of passed, failed, skipped group by file_dir, result_id) +Test Result Status Summary (Counts/Percentages sorted by testseries, ID) ============================================================================================================== -------------------------------------------------------------------------------------------------------------- -{{ 'file_dir'.ljust(max_len_dir) }} | {{ 'result_id'.ljust(max_len_result_id) }} | {{ 'passed'.ljust(10) }} | {{ 'failed'.ljust(10) }} | {{ 'skipped'.ljust(10) }} +{{ 'Test Series'.ljust(maxlen['testseries']) }} | {{ 'ID'.ljust(maxlen['result_id']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} -------------------------------------------------------------------------------------------------------------- -{% for report in test_count_reports |sort(attribute='test_file_dir_result_id') %} -{{ report.file_dir.ljust(max_len_dir) }} | {{ report.result_id.ljust(max_len_result_id) }} | {{ (report.passed|string).ljust(10) }} | {{ (report.failed|string).ljust(10) }} | {{ (report.skipped|string).ljust(10) }} +{% for report in reportvalues |sort(attribute='sort') %} +{{ report.testseries.ljust(maxlen['testseries']) }} | {{ report.result_id.ljust(maxlen['result_id']) }} | {{ (report.passed|string).ljust(maxlen['passed']) }} | {{ (report.failed|string).ljust(maxlen['failed']) }} | {{ (report.skipped|string).ljust(maxlen['skipped']) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- +{% if haveptest %} ============================================================================================================== -Test Report (Percent of passed, failed, skipped group by file_dir, result_id) +PTest Result Summary ============================================================================================================== -------------------------------------------------------------------------------------------------------------- -{{ 'file_dir'.ljust(max_len_dir) }} | {{ 'result_id'.ljust(max_len_result_id) }} | {{ 'passed_%'.ljust(10) }} | {{ 'failed_%'.ljust(10) }} | {{ 'skipped_%'.ljust(10) }} +{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} -------------------------------------------------------------------------------------------------------------- -{% for report in test_percent_reports |sort(attribute='test_file_dir_result_id') %} -{{ report.file_dir.ljust(max_len_dir) }} | {{ report.result_id.ljust(max_len_result_id) }} | {{ (report.passed|string).ljust(10) }} | {{ (report.failed|string).ljust(10) }} | {{ (report.skipped|string).ljust(10) }} +{% for ptest in ptests %} +{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[ptest]['duration']|string) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- +{% else %} +There was no ptest data +{% endif %} + ============================================================================================================== -Test Report (Failed test cases group by file_dir, result_id) +Failed test cases (sorted by testseries, ID) ============================================================================================================== +{% if havefailed %} -------------------------------------------------------------------------------------------------------------- -{% for report in test_count_reports |sort(attribute='test_file_dir_result_id') %} +{% for report in reportvalues |sort(attribute='sort') %} {% if report.failed_testcases %} -file_dir | result_id : {{ report.file_dir }} | {{ report.result_id }} +testseries | result_id : {{ report.testseries }} | {{ report.result_id }} {% for testcase in report.failed_testcases %} {{ testcase }} {% endfor %} {% endif %} {% endfor %} --------------------------------------------------------------------------------------------------------------- +-------------------------------------------------------------------------------------------------------------- +{% else %} +There were no test failures +{% endif %}
scripts/resulttool +4 -3
@@ -1,11 +1,12 @@ #!/usr/bin/env python3 # -# test results tool - tool for testresults.json (merge test results, regression analysis) +# test results tool - tool for manipulating OEQA test result json files +# (merge results, summarise results, regression analysis, generate manual test results file) # # To look for help information. # $ resulttool # -# To store test result from oeqa automated tests, execute the below +# To store test results from oeqa automated tests, execute the below # $ resulttool store <source_dir> <git_branch> # # To merge test results, execute the below
@@ -58,7 +59,7 @@ def _validate_user_input_arguments(args): return True def main(): - parser = argparse_oe.ArgumentParser(description="OpenEmbedded test results tool.", + parser = argparse_oe.ArgumentParser(description="OEQA test result manipulation tool.", epilog="Use %(prog)s <subcommand> --help to get help on a specific command") parser.add_argument('-d', '--debug', help='enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='print only errors', action='store_true')
Signature · 1 line
2.20.1
PATCH 2/2 oe-build-perf-report/gitarchive: Move common useful functions to library
These functions can be reused by the resulttool code so move to the common function library for this purpose.
--- meta/lib/oeqa/utils/gitarchive.py | 71 ++++++++++++++++++++++++ scripts/oe-build-perf-report | 90 ++++--------------------------- 2 files changed, 81 insertions(+), 80 deletions(-)
meta/lib/oeqa/utils/gitarchive.py +71 -0
@@ -17,6 +17,8 @@ import os import re import sys +from operator import attrgetter +from collections import namedtuple from oeqa.utils.git import GitRepo, GitError class ArchiveError(Exception):
@@ -171,3 +173,72 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms log.info("Pushing data to remote") data_repo.run_cmd(cmd) +# Container class for tester revisions +TestedRev = namedtuple('TestedRev', 'commit commit_number tags') + +def get_test_runs(log, repo, tag_name, **kwargs): + """Get a sorted list of test runs, matching given pattern""" + # First, get field names from the tag name pattern + field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] + undef_fields = [f for f in field_names if f not in kwargs.keys()] + + # Fields for formatting tag name pattern + str_fields = dict([(f, '*') for f in field_names]) + str_fields.update(kwargs) + + # Get a list of all matching tags + tag_pattern = tag_name.format(**str_fields) + tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() + log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) + + # Parse undefined fields from tag names + str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) + str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' + str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' + str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' + str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' + # escape parenthesis in fields in order to not messa up the regexp + fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) + str_fields.update(fixed_fields) + tag_re = re.compile(tag_name.format(**str_fields)) + + # Parse fields from tags + revs = [] + for tag in tags: + m = tag_re.match(tag) + groups = m.groupdict() + revs.append([groups[f] for f in undef_fields] + [tag]) + + # Return field names and a sorted list of revs + return undef_fields, sorted(revs) + +def get_test_revs(log, repo, tag_name, **kwargs): + """Get list of all tested revisions""" + fields, runs = get_test_runs(log, repo, tag_name, **kwargs) + + revs = {} + commit_i = fields.index('commit') + commit_num_i = fields.index('commit_number') + for run in runs: + commit = run[commit_i] + commit_num = run[commit_num_i] + tag = run[-1] + if not commit in revs: + revs[commit] = TestedRev(commit, commit_num, [tag]) + else: + assert commit_num == revs[commit].commit_number, "Commit numbers do not match" + revs[commit].tags.append(tag) + + # Return in sorted table + revs = sorted(revs.values(), key=attrgetter('commit_number')) + log.debug("Found %d tested revisions:\n %s", len(revs), + "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) + return revs + +def rev_find(revs, attr, val): + """Search from a list of TestedRev""" + for i, rev in enumerate(revs): + if getattr(rev, attr) == val: + return i + raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) +
scripts/oe-build-perf-report +10 -80
133 lines not on this page · raw message
Signature · 1 line
2.20.1