Porting AUH to devtool and making it suitable for local use
cover Porting AUH to devtool and making it suitable for local use
This patchset replaces custom recipe rewriting code in AUH with calls to devtool and makes other improvements aimed at making it suitable for local, interactive use by recipe maintainers.
The following changes since commit ccf93239c7def7b475fe20f8fcd91043bc59bd61:
upgradehelper.py: Layer mode do a full cleanup of poky (2017-06-05 16:14:23 -0500)
are available in the Git repository at:
git://github.com/kanavin/auh-devtool devel
https://github.com/kanavin/auh-devtool/tree/devel
Alexander Kanavin (20):
upgradehelper.py: do not error out if testimage or buildhistory are
enabled in bitbake conf, but not in AUH conf
upgradehelper.py: always run checkpkg
upgradehelper.py: do not manipulate git branches. upgradehelper.py: support updates to a new vcs revision. upgradehelper.py: add the old recipe version in pkg_ctx Add devtool support for upgrading recipes
buildhistory.py: replace cleanall with cleansstate testimage.py: do not manipulate branches
Replace references to Aníbal Limón as the maintainer. modules/recipe: remove
upgradehelper.py: remove the package ordering code utils/git.py: print current dir when git fails testimage.py: do not call into removed code
testimage.py: add extra logging for the testimage logs retrieval upgradehelper.py: fixups to the email message template upgradehelper.py: send email even if recipe upgrade failed upgradehelper.py: add all changes before committing them upgradehelper.py: when attempting to commit changes do not discard
previous errors
upgradehelper.py: when recovering from upgrade error, do not refer to
a dict entry that may not exist
upgradehelper.py: revert commits that failed to build
README | 2 +- modules/buildhistory.py | 2 +- modules/errors.py | 7 +- modules/recipe/__init__.py | 0 modules/recipe/base.py | 724 --------------------------------------------- modules/recipe/git.py | 98 ------ modules/recipe/svn.py | 28 -- modules/steps.py | 191 ++++++------ modules/testimage.py | 46 +-- modules/utils/devtool.py | 41 +++ modules/utils/git.py | 8 +- upgradehelper.py | 254 ++++------------ 12 files changed, 208 insertions(+), 1193 deletions(-) delete mode 100644 modules/recipe/__init__.py delete mode 100644 modules/recipe/base.py delete mode 100644 modules/recipe/git.py delete mode 100644 modules/recipe/svn.py create mode 100644 modules/utils/devtool.py
Signature · 1 line
2.15.0
PATCH 1/20 upgradehelper.py: do not error out if testimage or buildhistory are enabled in bitbake conf, but not in AUH conf
From: Alexander Kanavin <[email protected]>
--- upgradehelper.py | 12 ------------ 1 file changed, 12 deletions(-)
upgradehelper.py +0 -12
@@ -240,12 +240,6 @@ class Updater(object): " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) - else: - if 'buildhistory' in self.base_env['INHERIT']: - E(" Buildhistory was INHERIT in conf/local.conf"\ - " but buildhistory=yes isn't in upgrade-helper.conf,"\ - " if you want to enable please set.") - exit(1) return enabled
@@ -280,12 +274,6 @@ class Updater(object): " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) - else: - if 'testimage' in self.base_env['INHERIT']: - E(" testimage was INHERIT in conf/local.conf"\ - " but testimage=yes isn't in upgrade-helper.conf,"\ - " if you want to enable please set.") - exit(1) return enabled
Signature · 1 line
2.15.0
PATCH 2/20 upgradehelper.py: always run checkpkg
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
There was code to skip upstream version check if it was performed recently, which was actually broken as it didn't verify that checkpkg.csv already contains the package we want to update. Let's just drop the whole logic and run checkpkg always.
--- upgradehelper.py | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-)
upgradehelper.py +2 -31
@@ -829,43 +829,14 @@ class UniverseUpdater(Updater): return True def _get_packages_to_upgrade(self, packages=None): - last_date_checked = None - last_master_commit = None - last_checkpkg_file = None - current_date = date.isoformat(date.today()) - try: - stdout = self.git.last_commit("master") - cur_master_commit = stdout - except Error: - cur_master_commit = "unknown" - - if os.path.exists(get_build_dir() + "/upgrade-helper/last_checkpkg_run"): - with open(get_build_dir() + "/upgrade-helper/last_checkpkg_run") as last_check: - line = last_check.read() - last_date_checked = line.split(',')[0] - last_master_commit = line.split(',')[1] - last_checkpkg_file = line.split(',')[2] - if not os.path.exists(last_checkpkg_file): - last_checkpkg_file = None - - if last_master_commit != cur_master_commit or last_date_checked != current_date or \ - last_checkpkg_file is None: - self._check_upstream_versions() - last_checkpkg_file = os.path.realpath(get_build_dir() + "/tmp/log/checkpkg.csv") - else: - I(" Using last checkpkg.csv file since last master commit and last" - " check date are the same ...") + self._check_upstream_versions() + last_checkpkg_file = os.path.realpath(get_build_dir() + "/tmp/log/checkpkg.csv") pkgs_list = [] for pkg in self._parse_checkpkg_file(last_checkpkg_file): if self._pkg_upgradable(pkg[0], pkg[1], pkg[2]): pkgs_list.append(pkg) - # Update last_checkpkg_run only after the version check has been completed - with open(get_build_dir() + "/upgrade-helper/last_checkpkg_run", "w+") as last_check: - last_check.write(current_date + "," + cur_master_commit + "," + - last_checkpkg_file) - return pkgs_list def _update_history(self, pn, new_ver, maintainer, upgrade_status):
Signature · 1 line
2.15.0
PATCH 3/20 upgradehelper.py: do not manipulate git branches.
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
Let's just write the commits into the current branch, and leave branch management to some other tool.
--- upgradehelper.py | 25 ------------------------- 1 file changed, 25 deletions(-)
upgradehelper.py +0 -25
@@ -695,30 +695,6 @@ class UniverseUpdater(Updater): return recipes - def _update_master(self): - if self.opts['layer_mode'] == 'yes': - I(" Sync poky master ...") - self.poky_git.reset_hard() - self.poky_git.clean_untracked() - self.poky_git.checkout_branch("master") - self.poky_git.pull() - - I(" Drop all uncommited changes (including untracked) ...") - self.git.reset_hard() - self.git.clean_untracked() - - self.git.checkout_branch("master") - try: - self.git.delete_branch("upgrades") - except Error: - pass - if self.opts['layer_mode'] == 'yes': - I(" Sync %s master ..." % self.opts['layer_name']) - else: - I(" Sync poky master ...") - self.git.pull() - self.git.create_branch("upgrades") - def _prepare(self): if settings.get("clean_sstate", "no") == "yes" and \ os.path.exists(os.path.join(get_build_dir(), "sstate-cache")):
@@ -857,7 +833,6 @@ class UniverseUpdater(Updater): self._get_status_msg(pkg_ctx['error'])) def run(self): - self._update_master() self._prepare() super(UniverseUpdater, self).run()
Signature · 1 line
2.15.0
PATCH 4/20 upgradehelper.py: support updates to a new vcs revision.
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
Devtool has support for this, and it's useful when the upstream never issues new versions; in this situation we can simply attempt to update to the latest revision.
--- upgradehelper.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
upgradehelper.py +6 -4
@@ -526,13 +526,14 @@ class Updater(object): pkgs_ctx = {} I(" ########### The list of recipes to be upgraded #############") - for p, v, m in pkgs_to_upgrade: - I(" %s, %s, %s" % (p, v, m)) + for p, v, m, r in pkgs_to_upgrade: + I(" %s, %s, %s, %s" % (p, v, m, r)) pkgs_ctx[p] = {} pkgs_ctx[p]['PN'] = p pkgs_ctx[p]['NPV'] = v pkgs_ctx[p]['MAINTAINER'] = m + pkgs_ctx[p]['NSRCREV'] = r pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir I(" ############################################################")
@@ -555,7 +556,7 @@ class Updater(object): succeeded_pkgs_ctx = [] failed_pkgs_ctx = [] attempted_pkgs = 0 - for pn, _, _ in pkgs_to_upgrade: + for pn, _, _, _ in pkgs_to_upgrade: pkg_ctx = pkgs_ctx[pn] pkg_ctx['error'] = None
@@ -738,11 +739,12 @@ class UniverseUpdater(Updater): cur_ver = row[1] next_ver = row[2] status = row[11] + revision = row[12] maintainer = row[14] no_upgrade_reason = row[15] if status == 'UPDATE' and not no_upgrade_reason: - pkgs_list.append((pn, next_ver, maintainer)) + pkgs_list.append((pn, next_ver, maintainer, revision)) else: if no_upgrade_reason: D(" Skip package %s (status = %s, current version = %s," \
Signature · 1 line
2.15.0
PATCH 5/20 upgradehelper.py: add the old recipe version in pkg_ctx
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
So that it can be later used in the commit message.
--- upgradehelper.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
upgradehelper.py +6 -5
@@ -526,12 +526,13 @@ class Updater(object): pkgs_ctx = {} I(" ########### The list of recipes to be upgraded #############") - for p, v, m, r in pkgs_to_upgrade: - I(" %s, %s, %s, %s" % (p, v, m, r)) + for p, ov, nv, m, r in pkgs_to_upgrade: + I(" %s, %s, %s, %s, %s" % (p, ov, nv, m, r)) pkgs_ctx[p] = {} pkgs_ctx[p]['PN'] = p - pkgs_ctx[p]['NPV'] = v + pkgs_ctx[p]['PV'] = ov + pkgs_ctx[p]['NPV'] = nv pkgs_ctx[p]['MAINTAINER'] = m pkgs_ctx[p]['NSRCREV'] = r
@@ -556,7 +557,7 @@ class Updater(object): succeeded_pkgs_ctx = [] failed_pkgs_ctx = [] attempted_pkgs = 0 - for pn, _, _, _ in pkgs_to_upgrade: + for pn, _, _, _, _ in pkgs_to_upgrade: pkg_ctx = pkgs_ctx[pn] pkg_ctx['error'] = None
@@ -744,7 +745,7 @@ class UniverseUpdater(Updater): no_upgrade_reason = row[15] if status == 'UPDATE' and not no_upgrade_reason: - pkgs_list.append((pn, next_ver, maintainer, revision)) + pkgs_list.append((pn, cur_ver, next_ver, maintainer, revision)) else: if no_upgrade_reason: D(" Skip package %s (status = %s, current version = %s," \
Signature · 1 line
2.15.0
PATCH 6/20 Add devtool support for upgrading recipes
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
devtool has a much better support for rebasing patches, can be used interactively when things go wrong, and we also need to avoid having two codebases for updating recipes that broadly do the same thing.
--- modules/errors.py | 7 +- modules/steps.py | 191 ++++++++++++++++++++++------------------------- modules/utils/devtool.py | 41 ++++++++++ upgradehelper.py | 30 ++++---- 4 files changed, 151 insertions(+), 118 deletions(-) create mode 100644 modules/utils/devtool.py
modules/errors.py +5 -2
@@ -33,8 +33,11 @@ class Error(Exception): class MaintainerError(Error): """ Class for group error that can be sent to Maintainer's """ - def __init__(self, message=None, stdout=None, stderr=None): - super(MaintainerError, self).__init__(message, stdout, stderr) + +class DevtoolError(Error): + """ Class for devtool errors """ + def __str__(self): + return "Failed (devtool error)" class FetchError(Error): def __init__(self):
modules/steps.py +90 -101
233 lines not on this page · raw message
modules/utils/devtool.py new file +41 -0
41 lines not on this page · raw message
upgradehelper.py +15 -15
@@ -51,6 +51,7 @@ sys.path.insert(1, os.path.join(os.path.abspath( from errors import * from utils.git import Git +from utils.devtool import Devtool from utils.bitbake import * from utils.emailhandler import Email
@@ -132,6 +133,7 @@ class Updater(object): build_dir = get_build_dir() self.bb = Bitbake(build_dir) + self.devtool = Devtool() try: self.base_env = self.bb.env()
@@ -386,25 +388,23 @@ class Updater(object): def commit_changes(self, pkg_ctx): fail = False - try: pkg_ctx['patch_file'] = None - if 'recipe' in pkg_ctx: - I(" %s: Auto commit changes ..." % pkg_ctx['PN']) - self.git.commit(pkg_ctx['recipe'].commit_msg, self.opts['author']) + I(" %s: Auto commit changes ..." % pkg_ctx['PN']) + self.git.commit(pkg_ctx['commit_msg'], self.opts['author']) - stdout = self.git.create_patch(pkg_ctx['workdir']) - pkg_ctx['patch_file'] = stdout.strip() + stdout = self.git.create_patch(pkg_ctx['workdir']) + pkg_ctx['patch_file'] = stdout.strip() - if not pkg_ctx['patch_file']: - msg = "Patch file not generated." - E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) - pkg_ctx['error'] = Error(msg, stdout) - fail = True - else: - I(" %s: Save patch in directory: %s." % - (pkg_ctx['PN'], pkg_ctx['workdir'])) + if not pkg_ctx['patch_file']: + msg = "Patch file not generated." + E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) + pkg_ctx['error'] = Error(msg, stdout) + fail = True + else: + I(" %s: Save patch in directory: %s." % + (pkg_ctx['PN'], pkg_ctx['workdir'])) except Error as e: msg = ''
@@ -568,7 +568,7 @@ class Updater(object): for step, msg in upgrade_steps: if msg is not None: I(" %s: %s" % (pkg_ctx['PN'], msg)) - step(self.bb, self.git, self.opts, pkg_ctx) + step(self.devtool, self.bb, self.git, self.opts, pkg_ctx) succeeded_pkgs_ctx.append(pkg_ctx) I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN'])
Signature · 1 line
2.15.0
PATCH 7/20 buildhistory.py: replace cleanall with cleansstate
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
It should not be really necessary, and wastes time as the upstream source needs to be re-fetched.
--- modules/buildhistory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
modules/buildhistory.py +1 -1
@@ -49,7 +49,7 @@ class BuildHistory(object): os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir def init(self, machines): - self.bb.cleanall(self.pn) + self.bb.cleansstate(self.pn) for machine in machines: self.bb.complete(self.pn, machine) self.revs.append(self.git.last_commit("master"))
Signature · 1 line
2.15.0
PATCH 8/20 testimage.py: do not manipulate branches
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
AUH simply creates a series of commits now, so we can create and run testimage directly on the branch we're on.
--- modules/testimage.py | 32 -------------------------------- 1 file changed, 32 deletions(-)
modules/testimage.py +0 -32
@@ -75,33 +75,6 @@ class TestImage(): return ' '.join(pkgs_out) - def prepare_branch(self, pkgs_ctx): - ok = False - - try: - self.git.reset_hard() - self.git.checkout_branch("master") - - try: - self.git.delete_branch("testimage") - except Error: - pass - - self.git.create_branch("testimage") - for c in pkgs_ctx: - patch_file = os.path.join(c['workdir'], c['patch_file']) - try: - self.git.apply_patch(patch_file) - except Exception as e: - c['integration_error'] = e - self.git.abort_patch() - - ok = True - except Exception as e: - self._log_error(" testimage: Failed to prepare branch.") - - return ok - def _parse_ptest_log(self, log_file): ptest_results = {}
@@ -284,8 +257,6 @@ class TestImage(): self.pkgs_ctx.remove(pkg_ctx) - if not self.prepare_branch(self.pkgs_ctx): - handled = False else: handled = False
@@ -296,9 +267,6 @@ class TestImage(): I(" Testimage was enabled but any upgrade was successful.") return - if not self.prepare_branch(self.pkgs_ctx): - return - I(" Images will test for %s." % ', '.join(self.opts['machines'])) for machine in self.opts['machines']: I(" Testing images for %s ..." % machine)
Signature · 1 line
2.15.0
PATCH 9/20 Replace references to Aníbal Limón as the maintainer.
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- README | 2 +- upgradehelper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
README +1 -1
@@ -163,7 +163,7 @@ The latest version of the code can always be found here: Contributions are welcome. Please send patches / pull requests to [email protected] with '[auh]' in the subject also CC the -current maintainer: Aníbal Limón <[email protected]>. +current maintainer: Alex Kanavin <[email protected]>. License -------
upgradehelper.py +1 -1
@@ -321,7 +321,7 @@ class Updater(object): mail_footer = \ "Attached are the patch, license diff (if change) and bitbake log.\n" \ - "Any problem please contact Anibal Limon <[email protected]>.\n\n" \ + "Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \ "Regards,\nThe Upgrade Helper" if pkg_ctx['MAINTAINER'] in maintainer_override:
Signature · 1 line
2.15.0
PATCH 11/20 upgradehelper.py: remove the package ordering code
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
It was broken and commented out, and not really necessary.
--- upgradehelper.py | 80 -------------------------------------------------------- 1 file changed, 80 deletions(-)
upgradehelper.py +0 -80
@@ -439,87 +439,7 @@ class Updater(object): else: W("No recipes attempted, not sending status mail!") - def _order_pkgs_to_upgrade(self, pkgs_to_upgrade): - def _get_pn_dep_dic(pn_list, dependency_file): - import re - - pn_dep_dic = {} - - with open(dependency_file) as dep: - data = dep.read() - dep.close() - - for line in data.split('\n'): - m = re.search('^"(.*)" -> "(.*)"$', line) - if not m: - continue - - pn = m.group(1) - pn_dep = m.group(2) - if pn == pn_dep: - continue - - if pn in pn_list: - if pn_dep in pn_list: - if pn in pn_dep_dic.keys(): - pn_dep_dic[pn].append(pn_dep) - else: - pn_dep_dic[pn] = [pn_dep] - elif not pn in pn_dep_dic.keys(): - pn_dep_dic[pn] = [] - - return pn_dep_dic - - def _dep_resolve(graph, node, resolved, seen): - seen.append(node) - - for edge in graph[node]: - if edge not in resolved: - if edge in seen: - raise RuntimeError("Packages %s and %s have " \ - "a circular dependency." \ - % (node, edge)) - _dep_resolve(graph, edge, resolved, seen) - - resolved.append(node) - - - pn_list = [] - for pn, new_ver, maintainer in pkgs_to_upgrade: - pn_list.append(pn) - - try: - self.bb.dependency_graph(' '.join(pn_list)) - except Error as e: - multiple_providers = False - for l in e.stdout.split('\n'): - if l.find("ERROR: Multiple .bb files are due to be built which each provide") == 0: - multiple_providers = True - if not multiple_providers: - raise e - - dependency_file = os.path.join(get_build_dir(), "pn-depends.dot") - - pkgs_to_upgrade_ordered = [] - pn_list_ordered = [] - - pn_dep_dic = _get_pn_dep_dic(pn_list, dependency_file) - if pn_dep_dic: - root = "__root_node__" - pn_dep_dic[root] = pn_dep_dic.keys() - _dep_resolve(pn_dep_dic, root, pn_list_ordered, []) - pn_list_ordered.remove(root) - - for pn_ordered in pn_list_ordered: - for pn, new_ver, maintainer in pkgs_to_upgrade: - if pn == pn_ordered: - pkgs_to_upgrade_ordered.append([pn, new_ver, maintainer]) - - return pkgs_to_upgrade_ordered - def run(self, package_list=None): - #pkgs_to_upgrade = self._order_pkgs_to_upgrade( - # self._get_packages_to_upgrade(package_list)) pkgs_to_upgrade = self._get_packages_to_upgrade(package_list) total_pkgs = len(pkgs_to_upgrade)
Signature · 1 line
2.15.0
PATCH 12/20 utils/git.py: print current dir when git fails
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- modules/utils/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
modules/utils/git.py +1 -1
@@ -40,7 +40,7 @@ class Git(object): try: stdout, stderr = bb.process.run(cmd) except bb.process.ExecutionError as e: - D("%s returned:\n%s" % (cmd, e.__str__())) + D("%s executed from %s returned:\n%s" % (cmd, self.repo_dir, e.__str__())) raise Error("The following git command failed: " + operation, e.stdout, e.stderr)
Signature · 1 line
2.15.0
PATCH 13/20 testimage.py: do not call into removed code
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This was using the removed recipe parsing code to determine if a package supports ptest; it should be replaced with using bitbake API but for now we can simply add all update packages to the list.
--- modules/testimage.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-)
modules/testimage.py +2 -10
@@ -53,15 +53,6 @@ class TestImage(): os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \ " TEST_SUITES CORE_IMAGE_EXTRA_INSTALL" - def _get_ptest_pkgs(self, pkgs_ctx): - pkgs = [] - - for c in pkgs_ctx: - if "ptest" in c['recipe'].get_inherits(): - pkgs.append(c) - - return pkgs - def _get_pkgs_to_install(self, pkgs, ptest=False): pkgs_out = []
@@ -149,7 +140,8 @@ class TestImage(): def ptest(self, pkgs_ctx, machine): image = 'core-image-minimal' - ptest_pkgs = self._get_ptest_pkgs(pkgs_ctx) + # should use bitbake API here to trim down the list to only the recipes that inherit ptest + ptest_pkgs = pkgs_ctx os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \ self._get_pkgs_to_install(ptest_pkgs, ptest=True)
Signature · 1 line
2.15.0
PATCH 10/20 modules/recipe: remove
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
The hand-crafted recipe updating/rewriting has been replaced by calling devtool.
--- modules/recipe/__init__.py | 0 modules/recipe/base.py | 724 --------------------------------------------- modules/recipe/git.py | 98 ------ modules/recipe/svn.py | 28 -- 4 files changed, 850 deletions(-) delete mode 100644 modules/recipe/__init__.py delete mode 100644 modules/recipe/base.py delete mode 100644 modules/recipe/git.py delete mode 100644 modules/recipe/svn.py
modules/recipe/__init__.py deleted +0 -0
modules/recipe/base.py deleted +0 -724
724 lines not on this page · raw message
modules/recipe/git.py deleted +0 -98
98 lines not on this page · raw message
modules/recipe/svn.py deleted +0 -28
@@ -1,28 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sw=4 et: -# -# Copyright (c) 2013 - 2014 Intel Corporation -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# AUTHORS -# Laurentiu Palcu <[email protected]> -# Marius Avram <[email protected]> -# - -from recipe.base import Recipe - -class SvnRecipe(Recipe): - pass
Signature · 1 line
2.15.0
PATCH 14/20 testimage.py: add extra logging for the testimage logs retrieval
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This helped with diagnozing issues described in
https://bugzilla.yoctoproject.org/show_bug.cgi?id=12396
--- modules/testimage.py | 2 ++ 1 file changed, 2 insertions(+)
modules/testimage.py +2 -0
@@ -98,8 +98,10 @@ class TestImage(): if name in files: result.append(os.path.join(root, name)) + D("Found logs named %s for machine %s: %s" %(name, machine, result)) for ptest_log in result: if machine in ptest_log: + D("Picked log: %s" %(ptest_log)) return ptest_log def _get_failed_recipe(self, log):
Signature · 1 line
2.15.0
PATCH 15/20 upgradehelper.py: fixups to the email message template
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- upgradehelper.py | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-)
upgradehelper.py +27 -16
@@ -294,19 +294,20 @@ class Updater(object): "to upgrade the recipe to *%s* has %s.\n\n" license_change_info = \ - "*LICENSE CHANGED* please review the %s file and update the LICENSE\n" \ - "variable in the recipe if is needed.\n\n" + "*LICENSE CHANGED* please review the %s file, update the LICENSE\n" \ + "variable in the recipe and summarize the changes in the commit message.\n\n" next_steps_info = \ - "The recipe has been successfully compiled for machines %s.\n\n" \ "Next steps:\n" \ " - apply the patch: git am %s\n" \ - " - check that required upstream patches have not been commented from the recipe,\n" \ - " if upstream patches were commented the reason is specified in the commit message.\n" \ + " - check the changes to upstream patches and summarize them in the commit message,\n" \ " - compile an image that contains the package\n" \ " - perform some basic sanity tests\n" \ " - amend the patch and sign it off: git commit -s --reset-author --amend\n" \ - " - send it to the list\n\n" \ + " - send it to the appropriate mailing list\n\n" \ + "Alternatively, if you believe the recipe should not be upgraded at this time,\n" \ + "you can fill RECIPE_NO_UPDATE_REASON in respective recipe file so that\n" \ + "automatic upgrades would no longer be attempted.\n\n" testimage_integration_error = \ "The recipe *FAILED* in testimage integration. Attached is the log file.\n\n"
@@ -320,7 +321,7 @@ class Updater(object): "the next machines %s. Attached is the log file.\n\n" \ mail_footer = \ - "Attached are the patch, license diff (if change) and bitbake log.\n" \ + "Please review the attached files for further information and build/update failures.\n" \ "Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \ "Regards,\nThe Upgrade Helper"
@@ -333,20 +334,30 @@ class Updater(object): if "status_recipients" in settings: cc_addr = settings["status_recipients"].split() - subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + pkg_ctx['NPV'] + newversion = pkg_ctx['NPV'] if not pkg_ctx['NPV'].endswith("new-commits-available") else pkg_ctx['NSRCREV'] + subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + newversion if not pkg_ctx['error']: subject += " SUCCEEDED" else: subject += " FAILED" - msg_body = mail_header % (pkg_ctx['PN'], pkg_ctx['NPV'], + msg_body = mail_header % (pkg_ctx['PN'], newversion, self._get_status_msg(pkg_ctx['error'])) - if 'recipe' in pkg_ctx: - license_diff_fn = pkg_ctx['recipe'].get_license_diff_file_name() - if license_diff_fn: - msg_body += license_change_info % license_diff_fn - if not pkg_ctx['error']: - msg_body += next_steps_info % (', '.join(self.opts['machines']), - os.path.basename(pkg_ctx['patch_file'])) + + if pkg_ctx['error'] is not None: + msg_body += """Detailed error information: + +%s +%s +%s + +""" %(pkg_ctx['error'].message if pkg_ctx['error'].message else "", pkg_ctx['error'].stdout if pkg_ctx['error'].stdout else "" , pkg_ctx['error'].stderr if pkg_ctx['error'].stderr else "") + + if 'license_diff_fn' in pkg_ctx: + license_diff_fn = pkg_ctx['license_diff_fn'] + msg_body += license_change_info % license_diff_fn + + if 'patch_file' in pkg_ctx and pkg_ctx['patch_file'] != None: + msg_body += next_steps_info % (os.path.basename(pkg_ctx['patch_file'])) if self.opts['testimage']: if 'integration_error' in pkg_ctx:
Signature · 1 line
2.15.0
PATCH 16/20 upgradehelper.py: send email even if recipe upgrade failed
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- upgradehelper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
upgradehelper.py +1 -2
@@ -380,8 +380,7 @@ class Updater(object): if os.path.isfile(attachment_fullpath): attachments.append(attachment_fullpath) - # Only send email to Maintainer when recipe upgrade succeed. - if self.opts['send_email'] and not pkg_ctx['error']: + if self.opts['send_email']: self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr) # Preserve email for review purposes. email_file = os.path.join(pkg_ctx['workdir'],
Signature · 1 line
2.15.0
PATCH 17/20 upgradehelper.py: add all changes before committing them
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- modules/utils/git.py | 3 +++ upgradehelper.py | 1 + 2 files changed, 4 insertions(+)
modules/utils/git.py +3 -0
@@ -52,6 +52,9 @@ class Git(object): def stash(self): return self._cmd("stash") + def add(self, src): + return self._cmd("add " + src) + def commit(self, commit_message, author=None): if author is None: return self._cmd("commit -a -s -m \"" + commit_message + "\"")
upgradehelper.py +1 -0
@@ -402,6 +402,7 @@ class Updater(object): pkg_ctx['patch_file'] = None I(" %s: Auto commit changes ..." % pkg_ctx['PN']) + self.git.add(pkg_ctx['recipe_dir']) self.git.commit(pkg_ctx['commit_msg'], self.opts['author']) stdout = self.git.create_patch(pkg_ctx['workdir'])
Signature · 1 line
2.15.0
PATCH 18/20 upgradehelper.py: when attempting to commit changes do not discard previous errors
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- upgradehelper.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
upgradehelper.py +2 -9
@@ -397,7 +397,6 @@ class Updater(object): f.write("\n%s\n" % msg_body) def commit_changes(self, pkg_ctx): - fail = False try: pkg_ctx['patch_file'] = None
@@ -411,8 +410,7 @@ class Updater(object): if not pkg_ctx['patch_file']: msg = "Patch file not generated." E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) - pkg_ctx['error'] = Error(msg, stdout) - fail = True + raise Error(msg, stdout) else: I(" %s: Save patch in directory: %s." % (pkg_ctx['PN'], pkg_ctx['workdir']))
@@ -425,12 +423,7 @@ class Updater(object): I(" %s: %s" % (pkg_ctx['PN'], msg)) I(" %s: %s" % (pkg_ctx['PN'], e.stdout)) - - pkg_ctx['error'] = Error(msg, e.stdout) - fail = True - - if fail: - raise pkg_ctx['error'] + raise e def send_status_mail(self, statistics_summary): if "status_recipients" not in settings:
Signature · 1 line
2.15.0
PATCH 19/20 upgradehelper.py: when recovering from upgrade error, do not refer to a dict entry that may not exist
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- upgradehelper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
upgradehelper.py +1 -1
@@ -510,7 +510,7 @@ class Updater(object): E(" %s: %s" % (pkg_ctx['PN'], e.message)) - if os.listdir(pkg_ctx['workdir']): + if 'workdir' in pkg_ctx and os.listdir(pkg_ctx['workdir']): E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" % (pkg_ctx['PN'], pkg_ctx['workdir']))
Signature · 1 line
2.15.0
PATCH 20/20 upgradehelper.py: revert commits that failed to build
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This helps avoid 'cascading build failures' where one failed update holds up everything else.
--- modules/utils/git.py | 3 +++ upgradehelper.py | 3 +++ 2 files changed, 6 insertions(+)
modules/utils/git.py +3 -0
@@ -61,6 +61,9 @@ class Git(object): else: return self._cmd("commit -a --author=\"" + author + "\" -m \"" + commit_message + "\"") + def revert(self, commit): + return self._cmd("revert --no-edit " + commit) + def create_patch(self, out_dir): return self._cmd("format-patch -M10 -1 -o " + out_dir)
upgradehelper.py +3 -0
@@ -414,6 +414,9 @@ class Updater(object): else: I(" %s: Save patch in directory: %s." % (pkg_ctx['PN'], pkg_ctx['workdir'])) + if pkg_ctx['error'] is not None: + I("Due to build errors, the commit will also be reverted to avoid cascading upgrade failures.") + self.git.revert("HEAD") except Error as e: msg = ''
Signature · 1 line
2.15.0