Resulttool minor enhancements
cover Resulttool minor enhancements
Resulttool minor enhancements
- Enable report to use regression_map
- Enable report to output raw test results
- Enable report to print total test result statistic
- Enable store to add extra test environment config data
Yeoh Ee Peng (4):
scripts/resulttool/report: Enable report to use regression_map scripts/resulttool/report: Enable output raw test results scripts/resulttool/report: Add total statistic to test result. resulttool/store.py: Enable add extra test environment data
scripts/lib/resulttool/report.py | 35 ++++++++++++++++++---- scripts/lib/resulttool/resultutils.py | 4 +-- scripts/lib/resulttool/store.py | 5 +++- .../resulttool/template/test_report_full_text.txt | 3 +- 4 files changed, 38 insertions(+), 9 deletions(-)
Signature · 1 line
2.7.4
PATCH 1/4 scripts/resulttool/report: Enable report to use regression_map
By default, report will use the store_map to generate the key to reference each result set. In some situation when using store_map with multiple set of tests sharing similar test configurations, the report will only showing partial result set for results that having identical result_id (use of multiconfig to run tests where it generate identical result_id).
Enable report to have the option to use the regression_map (optional) instead of the default store_map, where it will take larger set of configurations to generate the key to reference each result set, this will prevent the report from only showing partial result set.
--- scripts/lib/resulttool/report.py | 16 +++++++++++----- scripts/lib/resulttool/resultutils.py | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-)
scripts/lib/resulttool/report.py +11 -5
@@ -207,8 +207,11 @@ class ResultsTextReport(object): maxlen=maxlen) print(output) - def view_test_report(self, logger, source_dir, branch, commit, tag): + def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map): test_count_reports = [] + configmap = resultutils.store_map + if use_regression_map: + configmap = resultutils.regression_map if commit: if tag: logger.warning("Ignoring --tag as --commit was specified")
@@ -216,12 +219,12 @@ class ResultsTextReport(object): repo = GitRepo(source_dir) revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=branch) rev_index = gitarchive.rev_find(revs, 'commit', commit) - testresults = resultutils.git_get_result(repo, revs[rev_index][2]) + testresults = resultutils.git_get_result(repo, revs[rev_index][2], configmap=configmap) elif tag: repo = GitRepo(source_dir) - testresults = resultutils.git_get_result(repo, [tag]) + testresults = resultutils.git_get_result(repo, [tag], configmap=configmap) else: - testresults = resultutils.load_resultsdata(source_dir) + testresults = resultutils.load_resultsdata(source_dir, configmap=configmap) for testsuite in testresults: for resultid in testresults[testsuite]: skip = False
@@ -248,7 +251,7 @@ class ResultsTextReport(object): def report(args, logger): report = ResultsTextReport() - report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag) + report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag, args.use_regression_map) return 0 def register_commands(subparsers):
@@ -263,3 +266,6 @@ def register_commands(subparsers): parser_build.add_argument('--commit', help="Revision to report") parser_build.add_argument('-t', '--tag', default='', help='source_dir is a git repository, report on the tag specified from that repository') + parser_build.add_argument('-m', '--use_regression_map', action='store_true', + help='instead of the default "store_map", use the "regression_map" for report') +
scripts/lib/resulttool/resultutils.py +2 -2
@@ -177,7 +177,7 @@ def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, p with open(dst.replace(fn, "ptest-%s.log" % i), "w+") as f: f.write(sectionlog) -def git_get_result(repo, tags): +def git_get_result(repo, tags, configmap=store_map): git_objs = [] for tag in tags: files = repo.run_cmd(['ls-tree', "--name-only", "-r", tag]).splitlines()
@@ -200,7 +200,7 @@ def git_get_result(repo, tags): # Optimize by reading all data with one git command results = {} for obj in parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--'])): - append_resultsdata(results, obj) + append_resultsdata(results, obj, configmap=configmap) return results
Signature · 1 line
2.7.4
PATCH 2/4 scripts/resulttool/report: Enable output raw test results
In reply to Yeoh Ee PengIn case of debugging, report user need to acccess the raw test result. Instead of going back to source file/directory/URL to manually pull out the raw result, provide alternative way to let report showing raw test results by providing the result id (optional).
--- scripts/lib/resulttool/report.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)
scripts/lib/resulttool/report.py +16 -2
@@ -207,7 +207,7 @@ class ResultsTextReport(object): maxlen=maxlen) print(output) - def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map): + def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map, raw_test): test_count_reports = [] configmap = resultutils.store_map if use_regression_map:
@@ -225,6 +225,17 @@ class ResultsTextReport(object): testresults = resultutils.git_get_result(repo, [tag], configmap=configmap) else: testresults = resultutils.load_resultsdata(source_dir, configmap=configmap) + if raw_test: + raw_results = {} + for testsuite in testresults: + result = testresults[testsuite].get(raw_test, {}) + if result: + raw_results[testsuite] = result + if raw_results: + print(json.dumps(raw_results, sort_keys=True, indent=4)) + else: + print('Could not find raw test result for %s' % raw_test) + return 0 for testsuite in testresults: for resultid in testresults[testsuite]: skip = False
@@ -251,7 +262,8 @@ class ResultsTextReport(object): def report(args, logger): report = ResultsTextReport() - report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag, args.use_regression_map) + report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag, args.use_regression_map, + args.raw_test_only) return 0 def register_commands(subparsers):
@@ -268,4 +280,6 @@ def register_commands(subparsers): help='source_dir is a git repository, report on the tag specified from that repository') parser_build.add_argument('-m', '--use_regression_map', action='store_true', help='instead of the default "store_map", use the "regression_map" for report') + parser_build.add_argument('-r', '--raw_test_only', default='', + help='output raw test result only for the user provided test result id')
Signature · 1 line
2.7.4
PATCH 3/4 scripts/resulttool/report: Add total statistic to test result.
In reply to Yeoh Ee PengAdd total passed, failed, and skipped statistic to test result.
--- scripts/lib/resulttool/report.py | 5 +++++ scripts/lib/resulttool/template/test_report_full_text.txt | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-)
scripts/lib/resulttool/report.py +5 -0
@@ -186,6 +186,10 @@ class ResultsTextReport(object): havefailed = True if line['machine'] not in machines: machines.append(line['machine']) + reporttotalvalues = {} + for k in cols: + reporttotalvalues[k] = '%s' % sum([line[k] for line in test_count_reports]) + reporttotalvalues['count'] = '%s' % len(test_count_reports) for (machine, report) in self.ptests.items(): for ptest in self.ptests[machine]: if len(ptest) > maxlen['ptest']:
@@ -199,6 +203,7 @@ class ResultsTextReport(object): if len(ltpposixtest) > maxlen['ltpposixtest']: maxlen['ltpposixtest'] = len(ltpposixtest) output = template.render(reportvalues=reportvalues, + reporttotalvalues=reporttotalvalues, havefailed=havefailed, machines=machines, ptests=self.ptests,
scripts/lib/resulttool/template/test_report_full_text.txt +2 -1
@@ -8,7 +8,8 @@ Test Result Status Summary (Counts/Percentages sorted by testseries, ID) {{ report.testseries.ljust(maxlen['testseries']) }} | {{ report.result_id.ljust(maxlen['result_id']) }} | {{ (report.passed|string).ljust(maxlen['passed']) }} | {{ (report.failed|string).ljust(maxlen['failed']) }} | {{ (report.skipped|string).ljust(maxlen['skipped']) }} {% endfor %} -------------------------------------------------------------------------------------------------------------- - +{{ 'Total'.ljust(maxlen['testseries']) }} | {{ reporttotalvalues['count'].ljust(maxlen['result_id']) }} | {{ reporttotalvalues['passed'].ljust(maxlen['passed']) }} | {{ reporttotalvalues['failed'].ljust(maxlen['failed']) }} | {{ reporttotalvalues['skipped'].ljust(maxlen['skipped']) }} +-------------------------------------------------------------------------------------------------------------- {% for machine in machines %} {% if ptests[machine] %}
Signature · 1 line
2.7.4
PATCH 4/4 resulttool/store.py: Enable add extra test environment data
In reply to Yeoh Ee PengEnable the option to add extra test environment data to the configuration of each test result (as optional).
Example of optional test environment data include:
- custom packages included for runtime test
- detail machine specification used as target
- detail host environment used for bitbake
--- scripts/lib/resulttool/store.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
scripts/lib/resulttool/store.py +4 -1
@@ -24,6 +24,8 @@ def store(args, logger): configvars = resultutils.extra_configvars.copy() if args.executed_by: configvars['EXECUTED_BY'] = args.executed_by + if args.extra_test_env: + configvars['EXTRA_TEST_ENV'] = args.extra_test_env results = {} logger.info('Reading files from %s' % args.source) if resultutils.is_url(args.source) or os.path.isfile(args.source):
@@ -98,4 +100,5 @@ def register_commands(subparsers): help='don\'t error if no results to store are found') parser_build.add_argument('-x', '--executed-by', default='', help='add executed-by configuration to each result file') - + parser_build.add_argument('-t', '--extra-test-env', default='', + help='add extra test environment data to each result file configuration')
Signature · 1 line
2.7.4