1/4] upgradehelper: commit_changes ensure that patch_file exist
If patch_file isn't generated treat it as an error this avoid future exceptions in package_handler when try to send/save email.
--- upgradehelper.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-)
upgradehelper.py +27 -7
@@ -369,6 +369,8 @@ class Updater(object): f.write("\n%s\n" % msg_body) def commit_changes(self, pkg_ctx): + fail = False + try: pkg_ctx['patch_file'] = None
@@ -376,19 +378,32 @@ class Updater(object): I(" %s: Auto commit changes ..." % pkg_ctx['PN']) self.git.commit(pkg_ctx['recipe'].commit_msg, self.opts['author']) - I(" %s: Save patch in directory: %s." % - (pkg_ctx['PN'], pkg_ctx['workdir'])) - stdout = self.git.create_patch(pkg_ctx['workdir']) pkg_ctx['patch_file'] = stdout.strip() + + if not pkg_ctx['patch_file']: + msg = "Patch file not generated." + E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) + pkg_ctx['error'] = Error(msg, stdout) + fail = True + else: + I(" %s: Save patch in directory: %s." % + (pkg_ctx['PN'], pkg_ctx['workdir'])) except Error as e: + msg = '' + for line in e.stdout.split("\n"): if line.find("nothing to commit") == 0: - I(" %s: Nothing to commit!" % pkg_ctx['PN']) - return + msg = "Nothing to commit!" + I(" %s: %s" % (pkg_ctx['PN'], msg)) I(" %s: %s" % (pkg_ctx['PN'], e.stdout)) - raise e + + pkg_ctx['error'] = Error(msg, e.stdout) + fail = True + + if fail: + raise pkg_ctx['error'] def send_status_mail(self, statistics_summary): if "status_recipients" not in settings:
@@ -554,7 +569,12 @@ class Updater(object): pkg_ctx['error'] = e failed_pkgs_ctx.append(pkg_ctx) - self.commit_changes(pkg_ctx) + try: + self.commit_changes(pkg_ctx) + except: + if pkg_ctx in succeeded_pkgs_ctx: + succeeded_pkgs_ctx.remove(pkg_ctx) + failed_pkgs_ctx.append(pkg_ctx) if self.opts['testimage']: if len(succeeded_pkgs_ctx) > 0:
Signature · 1 line
2.1.4
[[AUH] 2/4] testimage: Move run code from upgradehelper to testimage
Modularize code in order to keep upgradehelper small so move the code from main module to testimage.
--- modules/testimage.py | 107 ++++++++++++++++++++++++++++++++++++++++++++------- upgradehelper.py | 85 ++++------------------------------------ 2 files changed, 102 insertions(+), 90 deletions(-)
modules/testimage.py +94 -13
@@ -42,10 +42,13 @@ def _pn_in_pkgs_ctx(pn, pkgs_ctx): return None class TestImage(): - def __init__(self, bb, git, uh_work_dir): + def __init__(self, bb, git, uh_work_dir, opts, *args, **kwargs): self.bb = bb self.git = git self.uh_work_dir = uh_work_dir + self.opts = opts + self.pkgs_ctx = args[0] + self.image = args[1] os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \ " TEST_SUITES CORE_IMAGE_EXTRA_INSTALL"
@@ -73,19 +76,29 @@ class TestImage(): return ' '.join(pkgs_out) def prepare_branch(self, pkgs_ctx): - self.git.checkout_branch("master") - try: - self.git.delete_branch("testimage") - self.git.delete_branch("upgrades") - except Error: - pass - self.git.reset_hard() - - self.git.create_branch("testimage") - for c in pkgs_ctx: - patch_file = os.path.join(c['workdir'], c['patch_file']) - self.git.apply_patch(patch_file) + ok = False + try: + self.git.reset_hard() + self.git.checkout_branch("master") + + try: + self.git.delete_branch("testimage") + except Error: + pass + + self.git.create_branch("testimage") + for c in pkgs_ctx: + patch_file = os.path.join(c['workdir'], c['patch_file']) + self.git.apply_patch(patch_file) + + ok = True + except Exception as e: + E(error_msg) + self._log_error(" testimage: Failed to prepare branch.") + + return ok + def _parse_ptest_log(self, log_file): ptest_results = {}
@@ -224,3 +237,71 @@ class TestImage(): for line in lf: of.write(line) of.write("END: TESTIMAGE for %s\n" % machine) + + def _log_error(self, e): + if isinstance(e, Error): + E(" %s" % e.stdout) + else: + import traceback + tb = traceback.format_exc() + E("%s" % tb) + + def _handle_error(self, e, machine): + handled = True + + if isinstance(e, IntegrationError): + pkg_ctx = e.pkg_ctx + + E(" %s on machine %s failed in integration, removing..." + % (pkg_ctx['PN'], machine)) + + with open(os.path.join(pkg_ctx['workdir'], + 'integration_error.log'), 'a+') as f: + f.write(e.stdout) + + if not pkg_ctx in self.pkgs_ctx['succeeded']: + E( "Infinite loop IntegrationError trying to " \ + "remove %s twice, see logs.", pkg_ctx['PN']) + handled = False + else: + pkg_ctx['error'] = e + self.pkgs_ctx['failed'].append(pkg_ctx) + self.pkgs_ctx['succeeded'].remove(pkg_ctx) + + if not self.prepare_branch(self.pkgs_ctx['succeeded']): + handled = False + else: + handled = False + + return handled + + def run(self): + if len(self.pkgs_ctx['succeeded']) <= 0: + I(" Testimage was enabled but any upgrade was successful.") + return + + if not self.prepare_branch(self.pkgs_ctx['succeeded']): + return + + I(" Images will test for %s." % ', '.join(self.opts['machines'])) + for machine in self.opts['machines']: + I(" Testing images for %s ..." % machine) + while True: + try: + self.ptest(self.pkgs_ctx['succeeded'], machine) + break + except Exception as e: + if not self._handle_error(e, machine): + E(" %s/testimage on machine %s failed" % (image, machine)) + self._log_error(e) + break + + while True: + try: + self.testimage(self.pkgs_ctx['succeeded'], machine, self.image) + break + except Exception as e: + if not self._handle_error(e, machine): + E(" %s/testimage on machine %s failed" % (image, machine)) + self._log_error(e) + break
upgradehelper.py +8 -77
91 lines not on this page · raw message
Signature · 1 line
2.1.4
[[AUH] 3/4] upgradehelper.py: Don't remove patches if fails to apply
In reply to Aníbal LimónInstead of remove patches when fails to apply now comment it this helps the maintainer to review why the patch failed to apply.
--- modules/recipe/base.py | 114 ++++++++++++++++++++++++++++++------------------- upgradehelper.py | 4 +- 2 files changed, 72 insertions(+), 46 deletions(-)
modules/recipe/base.py +70 -44
@@ -93,7 +93,7 @@ class Recipe(object): self.old_env = None self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver + "\n\n" - self.rm_patches_msg = "\n\nRemoved the following patch(es):\n" + self.comment_patches_msg = "\n\nCommented the following patch(es):\n" self._inherits = None
@@ -257,43 +257,65 @@ class Recipe(object): d['source_found'] = False _change(self.env, self.recipe_dir, d) - def _remove_patch_uri(self, uri): + def _comment_patch_uri(self, uri): @modify_recipe_files - def _remove(line, temp_recipe, *args, **kwargs): + def _comment(line, temp_recipe, *args, **kwargs): d = args[0] + uri = d['uri'] - if line.find(uri) == -1: - temp_recipe.write(line) + m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line) + m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line) + m3 = re.match("[\t ]*" + uri + " *\\\\", line) + m4 = re.match("([\t ]*)" + uri + " *\"", line) + + if m1 or m2 or m3 or m4: + d['commented'] = True + + if not d['start']: + d['start'] = True + + # patch on a single SRC_URI line: + if m1: + d['patches'].append(line) + d['end'] = True + # patch is on the first SRC_URI line + elif m2: + d['patches'].append(line) + temp_recipe.write(m2.group(1) + "\\\n") + # patch is in the middle + elif m3: + d['patches'].append(line) + # patch is last in list + elif m4: + d['end'] = True + d['patches'].append(line) + temp_recipe.write(m4.group(1) + "\"\n") else: - m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line) - m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line) - m3 = re.match("[\t ]*" + uri + " *\\\\", line) - m4 = re.match("([\t ]*)" + uri + " *\"", line) - - # patch on a single SRC_URI line: - if m1: - return - # patch is on the first SRC_URI line - elif m2: - temp_recipe.write(m2.group(1) + "\\\n") - # patch is in the middle - elif m3: - return - # patch is last in list - elif m4: - temp_recipe.write(m4.group(1) + "\"\n") - # nothing matched in recipe but we deleted the patch - # anyway? Then we must bail out! - else: - d['removed'] = False + temp_recipe.write(line) + + if d['start'] and (d['end'] or line.startswith('"')): + if d['patches']: + for p in d['patches']: + line = p.rstrip() + if line.endswith('\\'): + line = line[:-1] + temp_recipe.write("#%s\n" % line) + + d['start'] = False + d['end'] = False d = {} - d['removed'] = True - _remove(self.env, self.recipe_dir, d) + d['commented'] = False + + d['uri'] = uri + d['patches'] = [] + d['start'] = False + d['end'] = False + _comment(self.env, self.recipe_dir, d) - return d['removed'] + return d['commented'] - def _remove_faulty_patch(self, patch_log): + def _comment_faulty_patch(self, patch_log): patch_file = None is_reverse_applied = False
@@ -311,7 +333,7 @@ class Recipe(object): if not patch_file: return False - I(" %s: Removing patch %s ..." % (self.env['PN'], patch_file)) + I(" %s: Commenting patch %s ..." % (self.env['PN'], patch_file)) reason = None found = False dirs = [self.env['PN'] + "-" + self.env['PKGV'], self.env['PN'], "files"]
@@ -321,24 +343,26 @@ class Recipe(object): continue else: found = True + # Find out upstream status of the patch with open(patch_file_path) as patch: for line in patch: m = re.match(".*Upstream-Status:(.*)\n", line) if m: reason = m.group(1).strip().split()[0].lower() - os.remove(patch_file_path) - if not self._remove_patch_uri("file://" + patch_file): + + if not self._comment_patch_uri("file://" + patch_file): return False if not found: return False - self.rm_patches_msg += " * " + patch_file + self.comment_patches_msg += " * " + patch_file if reason: - self.rm_patches_msg += " (" + reason + ") " + self.comment_patches_msg += " (" + reason + ") " if is_reverse_applied: - self.rm_patches_msg += "+ reverse-applied" - self.rm_patches_msg += "\n" + self.comment_patches_msg += "+ reverse-applied" + self.comment_patches_msg += "\n" + return True def _is_license_issue(self, config_log):
@@ -628,7 +652,7 @@ class Recipe(object): # Undo removed patches if self.removed_patches: self.git.checkout_branch("upgrades") - self.git.delete_branch("remove_patches") + self.git.delete_branch("comment_patches") self.git.reset_hard() self.git.reset_soft(1) self.removed_patches = False
@@ -639,9 +663,9 @@ class Recipe(object): if self.removed_patches: # move temporary changes into upgrades branch self.git.checkout_branch("upgrades") - self.git.delete_branch("remove_patches") + self.git.delete_branch("comment_patches") self.git.reset_soft(1) - self.commit_msg += self.rm_patches_msg + "\n" + self.commit_msg += self.comment_patches_msg + "\n" self.removed_patches = False except Error as e: if self._is_incompatible_host(e.stdout):
@@ -659,14 +683,16 @@ class Recipe(object): failed_task = failed_recipes[self.env['PN']][0] log_file = failed_recipes[self.env['PN']][1] if failed_task == "do_patch": - # Remove one patch after the other until + + # Comment one patch after the other until # compilation works. if not self.removed_patches: self.git.commit("temporary") - self.git.create_branch("remove_patches") - self.git.checkout_branch("remove_patches") + self.git.create_branch("comment_patches") + self.git.checkout_branch("comment_patches") self.removed_patches = True - if not self._remove_faulty_patch(log_file): + + if not self._comment_faulty_patch(log_file): self._undo_temporary() raise PatchError()
upgradehelper.py +2 -2
@@ -289,8 +289,8 @@ class Updater(object): "The recipe has been successfully compiled for machines %s.\n\n" \ "Next steps:\n" \ " - apply the patch: git am %s\n" \ - " - check that required upstream patches have not been removed from the recipe,\n" \ - " if upstream patches were removed the reason is specified in the commit message.\n" \ + " - check that required upstream patches have not been commented from the recipe,\n" \ + " if upstream patches were commented the reason is specified in the commit message.\n" \ " - compile an image that contains the package\n" \ " - perform some basic sanity tests\n" \ " - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
Signature · 1 line
2.1.4
[[AUH] 4/4] utils/bitbake.py: Preserve runs in error log.
In reply to Aníbal LimónWhen AUH is trying to do upgrades every time that fails the error log is overwrite so now append the errors this provides better complete error info to the Maintainer.
--- modules/utils/bitbake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
modules/utils/bitbake.py +1 -1
@@ -68,7 +68,7 @@ class Bitbake(object): D("%s returned:\n%s" % (cmd, e.__str__())) if self.log_dir is not None and os.path.exists(self.log_dir): - with open(os.path.join(self.log_dir, BITBAKE_ERROR_LOG), "w+") as log: + with open(os.path.join(self.log_dir, BITBAKE_ERROR_LOG), "a+") as log: log.write(e.stdout) raise Error("\'" + cmd + "\' failed", e.stdout, e.stderr)
Signature · 1 line
2.1.4