0/6] Integration error handling and enhacements
This changes can be also reviewed at,
http://git.yoctoproject.org/cgit/cgit.cgi/auto-upgrade-helper/log/?h=boot_images_v3
Aníbal Limón (6):
recipe/git.py: Add missing import of re module. upgradehelper.py: Add support for preserve logs in workdir. upgradehelper.py: Add support for specify what image test. testimage/ptest: Add require of ptest-pkgs in EXTRA_IMAGE_FEATURES.
testimage: Add handling of IntegrationError
upgradehelper: Remove deprecated setting drop_previous_commits.
README | 7 ++- modules/errors.py | 9 ++++ modules/recipe/git.py | 1 + modules/testimage.py | 80 ++++++++++++++++++++++-------- upgradehelper.py | 135 +++++++++++++++++++++++++++++++------------------- 5 files changed, 156 insertions(+), 76 deletions(-)
Signature · 1 line
2.1.4
[[AUH] 4/6] testimage/ptest: Add require of ptest-pkgs in EXTRA_IMAGE_FEATURES.
Instead of add manually the ptest packages use IMAGE_FEATURE to install ptest packages this save time validating if ptest pkg exist.
--- README | 3 ++- modules/testimage.py | 5 ++--- upgradehelper.py | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-)
README +2 -1
@@ -108,7 +108,8 @@ file: --------------- snip --------------- INHERIT += "testimage" -EXTRA_IMAGE_FEATURES = "debug-tweaks package-management" +DISTRO_FEATURES_append = " ptest" +EXTRA_IMAGE_FEATURES = "debug-tweaks package-management ptest-pkgs" # testimage/ptest only work with rpm PACKAGE_CLASSES = "package_rpm" --------------- snip ---------------
modules/testimage.py +2 -3
@@ -60,11 +60,10 @@ class TestImage(): # for provide access to the target if ptest: pkgs_out.append("dropbear") + pkgs_out.append("ptest-runner") for c in pkgs: pkgs_out.append(c['PN']) - if ptest: - pkgs_out.append("%s-ptest" % c['PN']) return ' '.join(pkgs_out)
@@ -122,7 +121,7 @@ class TestImage(): ptest_pkgs = self._get_ptest_pkgs() os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \ - self._get_pkgs_to_install(ptest_pkgs, True) + self._get_pkgs_to_install(ptest_pkgs, ptest=True) I( " building core-image-minimal for %s ..." % machine) self.bb.complete("core-image-minimal", machine)
upgradehelper.py +5 -0
@@ -242,6 +242,11 @@ class Updater(object): " please add to conf/local.conf.") exit(1) + if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']: + E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\ + " please add to conf/local.conf.") + exit(1) + if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]: E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\ " please add to conf/local.conf.")
Signature · 1 line
2.1.4
Re: [[AUH] 4/6] testimage/ptest: Add require of ptest-pkgs in EXTRA_IMAGE_FEATURES.
Quoted 63 lines · On Tue, 01 Dec 2015 16:03:39 Aníbal Limón wrote:
Instead of add manually the ptest packages use
IMAGE_FEATUREto install ptest packages this save time validating if ptest pkg exist.Signed-off-by Aníbal Limón <[email protected]>--- README | 3 ++- modules/testimage.py | 5 ++--- upgradehelper.py | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-)README +2
-1@@ -108,7 +108,8 @@ file: --------------- snip --------------- INHERIT += "testimage" -EXTRA_IMAGE_FEATURES = "debug-tweaks package-management" +DISTRO_FEATURES_append = " ptest" +EXTRA_IMAGE_FEATURES = "debug-tweaks package-management ptest-pkgs" # testimage/ptest only work with rpm PACKAGE_CLASSES = "package_rpm" --------------- snip ---------------modules/testimage.py +2
-3@@ -60,11 +60,10 @@ class TestImage(): # for provide access to the target if ptest: pkgs_out.append("dropbear") + pkgs_out.append("ptest-runner") for c in pkgs: pkgs_out.append(c['PN']) - if ptest: - pkgs_out.append("%s-ptest" % c['PN']) return ' '.join(pkgs_out)@@ -122,7 +121,7 @@ class TestImage(): ptest_pkgs = self._get_ptest_pkgs() os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \ - self._get_pkgs_to_install(ptest_pkgs, True) + self._get_pkgs_to_install(ptest_pkgs, ptest=True) I( " building core-image-minimal for %s ..." % machine) self.bb.complete("core-image-minimal", machine)upgradehelper.py +1
-0@@ -242,6 +242,11 @@ class Updater(object): " please add to conf/local.conf.") exit(1) + if not "ptest-pkgs" inself.base_env['EXTRA_IMAGE_FEATURES']:
+ E("
testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\
upgradehelper.py class Updater(object): · line 249
Quoted 4 lines
upgradehelper.py +3
-0+ " please add to conf/local.conf.") + exit(1) +
Strictly speaking this isn't really the proper test - the proper test is that ptest-pkgs is in IMAGE_FEATURES *for the image being built and tested* (not necessarily the global configuration). We can deal with this at some point in the future though.
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
[[AUH] 2/6] upgradehelper.py: Add support for preserve logs in workdir.
In reply to Aníbal LimónAdd FileHandler log to store the log into workdir, this needs to move up the work directory creation.
--- upgradehelper.py | 10 ++++++++++ 1 file changed, 10 insertions(+)
upgradehelper.py +10 -0
@@ -134,6 +134,10 @@ class Updater(object): def __init__(self, auto_mode=False, send_email=False, skip_compilation=False): build_dir = get_build_dir() + self._make_dirs(build_dir) + + self._add_file_logger() + self.bb = Bitbake(build_dir) try:
@@ -160,6 +164,7 @@ class Updater(object): self.opts['buildhistory'] = self._buildhistory_is_enabled() self.opts['testimage'] = self._testimage_is_enabled() + def _make_dirs(self, build_dir): self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir): os.mkdir(self.uh_dir)
@@ -176,6 +181,11 @@ class Updater(object): self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed") os.mkdir(self.uh_recipes_failed_dir) + def _add_file_logger(self): + fh = log.FileHandler(os.path.join(self.uh_work_dir, "upgrade-helper.log")) + logger = log.getLogger() + logger.addHandler(fh) + def _get_status_msg(self, err): if err: return str(err)
Signature · 1 line
2.1.4
[[AUH] 3/6] upgradehelper.py: Add support for specify what image test.
In reply to Aníbal LimónBefore this commit core-image-sato was hardcoded for image to test with this commit image name can be specified in the upgrade-helper.conf.
--- README | 1 + modules/testimage.py | 10 +++++----- upgradehelper.py | 13 ++++++++----- 3 files changed, 14 insertions(+), 10 deletions(-)
README +1 -0
@@ -68,6 +68,7 @@ machines=qemux86 qemux86-64 qemuarm qemumips qemuppc # optional features buildhistory=no testimage=no +testimage_name=image-custom # defaults to core-image-sato --------------- snip ---------------
modules/testimage.py +5 -5
@@ -153,18 +153,18 @@ class TestImage(): f.write(line) f.write("END: PTEST for %s\n" % machine) - def sato(self, machine): + def testimage(self, machine, image): os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \ self._get_pkgs_to_install(self.pkgs_ctx) if 'TEST_SUITES' in os.environ: del os.environ['TEST_SUITES'] - I( " building core-image-sato for %s ..." % machine) - self.bb.complete("core-image-sato", machine) + I( " building %s for %s ..." % (image, machine)) + self.bb.complete(image, machine) - I( " running core-image-sato/testimage for %s ..." % machine) - self.bb.complete("core-image-sato -c testimage", machine) + I( " running %s/testimage for %s ..." % (image, machine)) + self.bb.complete("%s -c testimage" % image, machine) log_file = self._find_log("log.do_testimage", machine) shutil.copyfile(log_file,
upgradehelper.py +8 -5
@@ -68,6 +68,7 @@ help_text = """Usage examples: $ upgrade-helper.py all """ +DEFAULT_TESTIMAGE = 'core-image-sato' def parse_cmdline(): parser = argparse.ArgumentParser(description='Package Upgrade Helper',
@@ -314,8 +315,8 @@ class Updater(object): "The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \ "with the next machines %s. Attached is the log file.\n\n" - testimage_sato_info = \ - "The recipe has been tested using core-image-sato testimage and succeeded with \n" \ + testimage_info = \ + "The recipe has been tested using %s testimage and succeeded with \n" \ "the next machines %s. Attached is the log file.\n\n" \ mail_footer = \
@@ -353,7 +354,8 @@ class Updater(object): msg_body += testimage_ptest_info % machines if 'testimage' in pkg_ctx: machines = pkg_ctx['testimage'].keys() - msg_body += testimage_sato_info % machines + msg_body += testimage_info % (settings.get('testimage_name', \ + DEFAULT_TESTIMAGE), machines) msg_body += mail_footer
@@ -602,10 +604,11 @@ class Updater(object): import traceback traceback.print_exc(file=sys.stdout) + image = settings.get('testimage_name', DEFAULT_TESTIMAGE) try: - tim.sato(machine) + tim.testimage(machine, image) except Exception as e: - E(" core-image-sato/testimage on machine %s failed" % machine) + E(" %s/testimage on machine %s failed" % (image, machine)) if isinstance(e, Error): E(" %s" % e.stdout) else:
Signature · 1 line
2.1.4
[[AUH] 5/6] testimage: Add handling of IntegrationError
In reply to Aníbal LimónA recipe can compile alone when the upgrade proccess is made but when integrate it can fail due to dependencies of another recipes.
If build image fail then analyze logs to find what recipe failed if recipe is found directly on the set of recipe upgrades remove it, if not recipe found directly then search within dependencies of failed recipe.
Since IntegrationError can happen now, move the statistics update and directory symlink creation after testimage.
--- modules/errors.py | 9 ++++++ modules/testimage.py | 69 +++++++++++++++++++++++++++++++--------- upgradehelper.py | 89 +++++++++++++++++++++++++++++++++++----------------- 3 files changed, 125 insertions(+), 42 deletions(-)
modules/errors.py +9 -0
@@ -91,3 +91,12 @@ class EmptyEnvError(Error): def __str__(self): return "Failed(get_env)" + +class IntegrationError(Error): + def __init__(self, stdout, pkg_ctx): + super(IntegrationError, self).__init__("Failed to build %s in testimage branch" + % pkg_ctx['PN'], stdout) + self.pkg_ctx = pkg_ctx + + def __str__(self): + return "Failed(integrate)"
modules/testimage.py +55 -14
131 lines not on this page · raw message
upgradehelper.py +61 -28
123 lines not on this page · raw message
Signature · 1 line
2.1.4
[[AUH] 6/6] upgradehelper: Remove deprecated setting drop_previous_commits.
In reply to Aníbal LimónSince recipe upgrades are done in isolate manner cleaning the repo for every recipe [1] the drop_previous_commits becomes deprecated.
Also now we have Testimage integration phase that handles/reports this kind of errors.
--- README | 3 --- upgradehelper.py | 20 -------------------- 2 files changed, 23 deletions(-)
README +0 -3
@@ -59,9 +59,6 @@ clean_sstate=yes # clean tmp directory before upgrading clean_tmp=yes -# keep previous commits or not -drop_previous_commits=yes - # machines to test build with machines=qemux86 qemux86-64 qemuarm qemumips qemuppc
upgradehelper.py +0 -20
@@ -276,26 +276,6 @@ class Updater(object): # this function will be called at the end of each recipe upgrade def pkg_upgrade_handler(self, pkg_ctx): - if self.opts['interactive'] and pkg_ctx['error'] and pkg_ctx['patch_file']: - answer = "N" - I(" %s: Do you want to keep the changes? (y/N)" % pkg_ctx['PN']) - answer = sys.stdin.readline().strip().upper() - - if answer == '' or answer == 'N': - I(" %s: Dropping changes from git ..." % pkg_ctx['PN']) - self.git.reset_hard(1) - self.git.clean_untracked() - return - - # drop last upgrade from git. It's safer this way if the upgrade has - # problems and other recipes depend on it. Give the other recipes a - # chance... - if (settings.get("drop_previous_commits", "no") == "yes" and - not pkg_ctx['error']) or (pkg_ctx['error'] and pkg_ctx['patch_file']): - I(" %s: Dropping changes from git ..." % pkg_ctx['PN']) - self.git.reset_hard(1) - self.git.clean_untracked() - mail_header = \ "Hello,\n\nYou are receiving this email because you are the maintainer\n" \ "of *%s* recipe and this is to let you know that the automatic attempt\n" \
Signature · 1 line
2.1.4
[[AUH] 1/6] recipe/git.py: Add missing import of re module.
In reply to Aníbal Limón--- modules/recipe/git.py | 1 + 1 file changed, 1 insertion(+)
modules/recipe/git.py +1 -0
@@ -23,6 +23,7 @@ # from recipe.base import Recipe +import re class GitRecipe(Recipe): def _extract_tag_from_ver(self, ver):
Signature · 1 line
2.1.4