rewrite parser.py file using fuego common function
PATCH 3/18 rewrite parser.py file using fuego common function
rewrite parser.py of Functional.curl using fuego common parer function under script directory
--- engine/tests/Functional.curl/parser.py | 178 ++------------------------------- 1 file changed, 11 insertions(+), 167 deletions(-)
engine/tests/Functional.curl/parser.py +11 -167
@@ -1,174 +1,18 @@ #!/usr/bin/python -# parser.py for a functional test -# can be run after a test, or during one +# See common.py for description of command-line arguments -import os, re, sys, time, collections -import xmltodict -import json -import argparse +import os, sys -fuego_rw = os.environ.get("FUEGO_RW", "/fuego-rw") -fuego_core = os.environ.get('FUEGO_CORE', '/fuego-core') -jenkins_home = os.environ.get('JENKINS_HOME', '/var/lib/jenkins') +sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser') +import common as plib -def error_out(msg): - print("Error: %s" % msg) - sys.exit(1) +measurements = {} -def main(): - # support invocation onthe command line - parser = argparse.ArgumentParser(description="Parse the results from a Fuego test") - parser.add_argument("-b", "--board", type=str, help="board name") - parser.add_argument("-t", "--test", type=str, help="test name") - parser.add_argument("-j", "--job", type=str, help="job name") - parser.add_argument("-n", "--num", type=int, help="job number") - parser.add_argument("-s", "--start", type=str, help="start time") - parser.add_argument("-r", "--result", type=str, help="result") - args = parser.parse_args() +regex_string = '^TEST-(\d+) (.*)$' +matches = plib.parse_log(regex_string) - job_name = args.job - node_name = args.board - test_name = args.test - number = args.num +if matches: + for m in matches: + measurements['default.test' + m[0]] = 'PASS' if m[1] == 'OK' else 'FAIL' - print("parsing data for test: %s, num=%s, node=%s, job=%s" % (test_name, number, node_name, job_name)) - - if not job_name: - try: - job_name = os.environ["JOB_NAME"] - except: - print("Warning: Missing job name (or $JOB_NAME)") - - if not test_name: - try: - test_name = os.environ['TESTDIR'] - except: - error_out("Missing test name (or $TESTDIR)") - - if not node_name: - try: - node_name = os.environ.get('NODE_NAME') - except: - pass - - ret_val = 0 - - cur_dict = collections.OrderedDict() - cur_dict['report'] = collections.OrderedDict() - cur_dict['report']['name'] = test_name - - if not number: - if job_name: - # Get build number - number_file = "%s/jobs/%s/nextBuildNumber" % (jenkins_home, job_name) - file_hd = open(number_file, 'r') - number_line = file_hd.read() - file_hd.close - number = int(number_line.strip()) - 1 - number = int(os.environ.get('BUILD_NUMBER', number)) - else: - number = int(os.environ["BUILD_NUMBER"]) - - # check run.json file for the test meta-data - logdir = "%s/logs/%s/%s.%s.%s" % (fuego_rw, test_name, node_name, number, number) - - start_time = args.start - result = args.result - if start_time and result: - cur_dict["report"]["starttime"] = start_time - cur_dict["report"]["result"] = result - else: - result = None - #print "start_time=%s" % start_time - #print "result=%s" % result - - run_path = logdir+"/run.json" - run_data = None - print("Looing for %s" % run_path) - - if not result and os.path.isfile(run_path): - run_fd = open(run_path, "r") - run_data = json.load(run_fd) - start_sec = run_data['start_time'] / 1000 - cur_dict['report']['starttime'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start_sec)) - cur_dict['report']['result'] = run_data["result"] - console_log = logdir + "/consolelog.txt" - - if not result and not run_data: - if not job_name: - error_out("Missing both run.json file and job_name") - - # Get start time and result of the testset - build_xml = "%s/jobs/%s/builds/%d/build.xml" % (jenkins_home, job_name, number) - build_file = open(build_xml, 'rb') - build_raw_values = build_file.read() - build_file.close() - build_dict = xmltodict.parse(build_raw_values) - - start_sec = int(build_dict['build']['startTime']) / 1000 - cur_dict['report']['starttime'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_sec)) - cur_dict['report']['result'] = str(build_dict['build']['result']) - - - # get end time from console log timestamp - # FIXTHIS - doesn't work for ftc jobs, since there's no job name for this - console_log = "%s/jobs/%s/builds/%d/log" % (jenkins_home, job_name, number) - cur_dict['report']['endtime'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getmtime(console_log))) - - # Get meta-data for the test - cur_dict['report']['items'] = collections.OrderedDict({'item': []}) - log_file = open(console_log, 'r') - log_raw_lines = log_file.readlines() - log_file.close() - - # this is specific to AGL console log output - for line in log_raw_lines: - if line.startswith("++++ BOARD_VER"): - cur_dict['report']['board_version'] = line.split("=")[1].strip() - if line.startswith("++++ DEVICE_TEST"): - cur_dict['report']['device'] = line.split("=")[1].strip() - if line.startswith("++++ DEVICE_FS"): - cur_dict['report']['filesystem'] = line.split("=")[1].strip() - if line.startswith("+++ report "): - tmp_cmd = line.split(";")[0] - cur_dict['report']['test_dir'] = tmp_cmd.split()[3] - tmp_cmd = line.split(";")[1].strip() - cur_dict['report']['command_line'] = tmp_cmd.split("'")[0] - break - - test_log = logdir + "/testlog.txt" - cur_search_pat = re.compile("^(TEST-[\d]+) (.*)$", re.MULTILINE) - cur_file = open(test_log, 'r') - pat_result = cur_search_pat.findall(cur_file.read()) - cur_file.close() - - if pat_result: - for i in range(0, len(pat_result)): - ditem = collections.OrderedDict() - cur_dict['report']['items']['item'].append(ditem) - ditem['name'] = "%s" % pat_result[i][0] - if ((pat_result[i][1]).strip() == "OK"): - ditem['result'] = "PASS" - else: - ditem['result'] = "FAIL" - ret_val = 1 - if (ret_val == 1): - cur_dict['report']['result'] = "FAILURE" - print cur_dict - - xcontent = xmltodict.unparse(cur_dict, pretty=True) - - xmlfile_path = "%s/test_result.xml" % (logdir) - cur_file = open(xmlfile_path, 'wb') - cur_file.write(xcontent) - cur_file.close() - - jsonfile_path = logdir + "/fres.json" - outfile = open(jsonfile_path, 'wb') - jcontent = json.dump(cur_dict, outfile, sort_keys=True, indent=4, ensure_ascii=False) - outfile.close() - - sys.exit(ret_val) - -if __name__=="__main__": - main() +sys.exit(plib.process(measurements))
Signature · 1 line
2.9.3.windows.3
Quoted 220 lines
-----Original Message-----
From: [email protected] [mailto:fuego- [email protected]] On Behalf Of Song Cai
Sent: Friday, October 13, 2017 3:06 AM
To: [email protected]
Subject: [Fuego] [PATCH 03/18] rewriteparser.pyfile using fuego common functionrewrite
parser.pyof Functional.curl using fuego common parer function under script directorySigned-off-by Song Cai <[email protected]>--- engine/tests/Functional.curl/parser.py | 178 ++------------------------------- 1 file changed, 11 insertions(+), 167 deletions(-)engine/tests/Functional.curl/parser.py +10
-27@@ -1,174 +1,18 @@ #!/usr/bin/python -# parser.py for a functional test -# can be run after a test, or during one +# See common.py for description of command-line arguments -import os, re, sys, time, collections -import xmltodict -import json -import argparse +import os, sys -fuego_rw = os.environ.get("FUEGO_RW", "/fuego-rw") -fuego_core = os.environ.get('FUEGO_CORE', '/fuego-core') -jenkins_home = os.environ.get('JENKINS_HOME', '/var/lib/jenkins') +sys.path.insert(0, os.environ['FUEGO_CORE'] + '/engine/scripts/parser') +import common as plib -def error_out(msg): - print("Error: %s" % msg) - sys.exit(1) +measurements = {} -def main(): - # support invocation onthe command line - parser = argparse.ArgumentParser(description="Parse the results from a Fuego test") - parser.add_argument("-b", "--board", type=str, help="board name") - parser.add_argument("-t", "--test", type=str, help="test name") - parser.add_argument("-j", "--job", type=str, help="job name") - parser.add_argument("-n", "--num", type=int, help="job number") - parser.add_argument("-s", "--start", type=str, help="start time") - parser.add_argument("-r", "--result", type=str, help="result") - args = parser.parse_args() +regex_string = '^TEST-(\d+) (.*)$' +matches = plib.parse_log(regex_string) - job_name = args.job - node_name = args.board - test_name = args.test - number = args.num +if matches: + for m in matches: + measurements['default.test' + m[0]] = 'PASS' if m[1] == 'OK' else 'FAIL' - print("parsing data for test: %s, num=%s, node=%s, job=%s" %(
test_name, number,node_name,job_name))engine/tests/Functional.curl/parser.py +0
-29- - if not job_name: - try: - job_name = os.environ["JOB_NAME"] - except: - print("Warning: Missing job name (or $JOB_NAME)") - - if not test_name: - try: - test_name = os.environ['TESTDIR'] - except: - error_out("Missing test name (or $TESTDIR)") - - if not node_name: - try: - node_name = os.environ.get('NODE_NAME') - except: - pass - - ret_val = 0 - - cur_dict = collections.OrderedDict() - cur_dict['report'] = collections.OrderedDict() - cur_dict['report']['name'] = test_name - - if not number: - if job_name: - # Get build number - number_file = "%s/jobs/%s/nextBuildNumber" % (jenkins_home,
job_name)engine/tests/Functional.curl/parser.py +0
-10- file_hd = open(number_file, 'r') - number_line = file_hd.read() - file_hd.close - number = int(number_line.strip()) - 1 - number = int(os.environ.get('BUILD_NUMBER', number)) - else: - number = int(os.environ["BUILD_NUMBER"]) - - # check run.json file for the test meta-data - logdir = "%s/logs/%s/%s.%s.%s" % (fuego_rw, test_name, node_name,number, number)
engine/tests/Functional.curl/parser.py +0
-20- - start_time = args.start - result = args.result - if start_time and result: - cur_dict["report"]["starttime"] = start_time - cur_dict["report"]["result"] = result - else: - result = None - #print "start_time=%s" % start_time - #print "result=%s" % result - - run_path = logdir+"/run.json" - run_data = None - print("Looing for %s" % run_path) - - if not result and os.path.isfile(run_path): - run_fd = open(run_path, "r") - run_data = json.load(run_fd) - start_sec = run_data['start_time'] / 1000 - cur_dict['report']['starttime'] = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(start_sec))engine/tests/Functional.curl/parser.py +0
-9- cur_dict['report']['result'] = run_data["result"] - console_log = logdir + "/consolelog.txt" - - if not result and not run_data: - if not job_name: - error_out("Missing both run.json file and job_name") - - # Get start time and result of the testset - build_xml = "%s/jobs/%s/builds/%d/build.xml" % (jenkins_home,
job_name, number)engine/tests/Functional.curl/parser.py +0
-7- build_file = open(build_xml, 'rb') - build_raw_values = build_file.read() - build_file.close() - build_dict = xmltodict.parse(build_raw_values) - - start_sec = int(build_dict['build']['startTime']) / 1000 - cur_dict['report']['starttime'] = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(start_sec))engine/tests/Functional.curl/parser.py +0
-6- cur_dict['report']['result'] = str(build_dict['build']['result']) - - - # get end time from console log timestamp - # FIXTHIS - doesn't work for ftc jobs, since there's no job name for this - console_log = "%s/jobs/%s/builds/%d/log" % (jenkins_home, job_name,number)
- cur_dict['report']['endtime'] = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(os.path.getmtime(console_log)))engine/tests/Functional.curl/parser.py +0
-52- - # Get meta-data for the test - cur_dict['report']['items'] = collections.OrderedDict({'item': []}) - log_file = open(console_log, 'r') - log_raw_lines = log_file.readlines() - log_file.close() - - # this is specific to AGL console log output - for line in log_raw_lines: - if line.startswith("++++ BOARD_VER"): - cur_dict['report']['board_version'] = line.split("=")[1].strip() - if line.startswith("++++ DEVICE_TEST"): - cur_dict['report']['device'] = line.split("=")[1].strip() - if line.startswith("++++ DEVICE_FS"): - cur_dict['report']['filesystem'] = line.split("=")[1].strip() - if line.startswith("+++ report "): - tmp_cmd = line.split(";")[0] - cur_dict['report']['test_dir'] = tmp_cmd.split()[3] - tmp_cmd = line.split(";")[1].strip() - cur_dict['report']['command_line'] = tmp_cmd.split("'")[0] - break - - test_log = logdir + "/testlog.txt" - cur_search_pat = re.compile("^(TEST-[\d]+) (.*)$", re.MULTILINE) - cur_file = open(test_log, 'r') - pat_result = cur_search_pat.findall(cur_file.read()) - cur_file.close() - - if pat_result: - for i in range(0, len(pat_result)): - ditem = collections.OrderedDict() - cur_dict['report']['items']['item'].append(ditem) - ditem['name'] = "%s" % pat_result[i][0] - if ((pat_result[i][1]).strip() == "OK"): - ditem['result'] = "PASS" - else: - ditem['result'] = "FAIL" - ret_val = 1 - if (ret_val == 1): - cur_dict['report']['result'] = "FAILURE" - print cur_dict - - xcontent = xmltodict.unparse(cur_dict, pretty=True) - - xmlfile_path = "%s/test_result.xml" % (logdir) - cur_file = open(xmlfile_path, 'wb') - cur_file.write(xcontent) - cur_file.close() - - jsonfile_path = logdir + "/fres.json" - outfile = open(jsonfile_path, 'wb') - jcontent = json.dump(cur_dict, outfile, sort_keys=True, indent=4,ensure_ascii=False)
engine/tests/Functional.curl/parser.py +1
-7- outfile.close() - - sys.exit(ret_val) - -if __name__=="__main__": - main() +sys.exit(plib.process(measurements)) --
Quoted 1 line
2.9.3.windows.3
Very nice. Thanks. Applied.
One of my boards doesn't have 'curl' by default.
Can you please add an "is_on_target" for the curl program, for this test?
Thanks,
-- tim