Add support for buildhistory and minor fixes
v2 cover Add support for buildhistory and minor fixes
This v2 fixes comments by Paul Eggleton in patch 01 and patch 05 of this patchset.
This changes can be found at,
http://git.yoctoproject.org/cgit/cgit.cgi/auto-upgrade-helper/log/?h=alimon/devel
Aníbal Limón (11):
upgradehelper.py: Only run with one recipe or all. upgradehelper.py: Validate if upgrade is needed upgradehelper.py: Only send emails when patch file exist bitbake.py: Improve performance on env() call upgradehelper.py: Add own step for loading environment upgradehelper.py: step _detect_repo clarify why don't load git upgradehelper.py: UniverseUpdate don't abort if recipe checkpkg fails. bitbake.py: Rename bitbake_log.txt to bitbake_error_log upgradehelper.py: Remove unused references to Buildhistory class upgradehelper.py: Move upstream versioning code to UniverseUpdater
upgradehelper: Add support for generate buildhistory recipe diff's
bitbake.py | 52 ++------ buildhistory.py | 75 ++++++++++++ errors.py | 6 + upgradehelper.py | 359 ++++++++++++++++++++++++++++++------------------------- 4 files changed, 290 insertions(+), 202 deletions(-) create mode 100644 buildhistory.py
Signature · 1 line
1.9.1
PATCH v2 1/11 upgradehelper.py: Only run with one recipe or all.
When try to upgrading a recipe is an interactive process it needs specify the version to upgrade, for simplify the upgrade process only support one recipe at time.
Now supported modes:
Batch mode for upgrade universe recipes to upstream version detected,
$ ./upgradehelper.py all
Interactive mode for upgrade one recipe to user provided version,
$ ./upgradehelper.py recipe --to_version VERSION
--- upgradehelper.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-)
upgradehelper.py +10 -24
@@ -65,7 +65,7 @@ def parse_cmdline(): parser = argparse.ArgumentParser(description='Package Upgrade Helper', formatter_class=argparse.RawTextHelpFormatter, epilog=help_text) - parser.add_argument("recipe", nargs="+", help="recipe to be upgraded") + parser.add_argument("recipe", help="recipe to be upgraded") parser.add_argument("-t", "--to_version", help="version to upgrade the recipe to")
@@ -507,14 +507,6 @@ class Updater(object): self.statistics.update(self.pn, self.new_ver, self.maintainer, error) - if self.interactive and attempted_pkgs < total_pkgs: - I(" %s: Proceed to next recipe? (Y/n)" % self.pn) - answer = sys.stdin.readline().strip().upper() - - if answer != 'Y' and answer != '': - I("Aborted by user!") - exit(0) - if (attempted_pkgs > 1): I("%s" % self.statistics.pkg_stats()) if self.send_email:
@@ -700,24 +692,18 @@ if __name__ == "__main__": settings, maintainer_override = parse_config_file(args.config_file) - if len(args.recipe) == 1 and args.recipe[0] == "all": + if args.recipe == "all": updater = UniverseUpdater() updater.run() - elif len(args.recipe) >= 1: - if len(args.recipe) == 1: - if not args.to_version: - E(" For upgrade only one recipe you must specify --to_version\n") - exit(1) - - if not args.maintainer and args.send_emails: - E(" For upgrade only one recipe and send email you must specify --maintainer\n") - exit(1) + else: + if not args.to_version: + E(" For upgrade only one recipe you must specify --to_version\n") + exit(1) - pkg_list = [(args.recipe[0], args.to_version, args.maintainer)] - else: - pkg_list = [] - for pkg in args.recipe: - pkg_list.append((pkg, None, None)) + if not args.maintainer and args.send_emails: + E(" For upgrade only one recipe and send email you must specify --maintainer\n") + exit(1) + pkg_list = [(args.recipe, args.to_version, args.maintainer)] updater = Updater(args.auto_mode, args.send_emails, args.skip_compilation) updater.run(pkg_list)
Signature · 1 line
1.9.1
PATCH v2 2/11 upgradehelper.py: Validate if upgrade is needed
In reply to Aníbal LimónWhen run upgradehelper only with one recipe upgrade needed validation isn't did.
[YOCTO #7947]
--- upgradehelper.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-)
upgradehelper.py +21 -3
@@ -207,6 +207,9 @@ class Updater(object): def _detect_repo(self): self._get_env() + if self.env['PV'] == self.new_ver: + raise UpgradeNotNeededError + if self.git is not None: return
@@ -496,9 +499,24 @@ class Updater(object): step() I(" %s: Upgrade SUCCESSFUL! Please test!" % self.pn) - except Error as e: - E(" %s: %s" % (self.pn, e.message)) - E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" % (self.pn, self.workdir)) + except Exception as e: + if isinstance(e, UpgradeNotNeededError): + I(" %s: %s" % (self.pn, e.message)) + elif isinstance(e, UnsupportedProtocolError): + I(" %s: %s" % (self.pn, e.message)) + else: + if not isinstance(e, Error): + import traceback + msg = "Failed(unknown error)\n" + traceback.format_exc() + e = Error(message=msg) + error = e + + E(" %s: %s" % (self.pn, e.message)) + + if os.listdir(self.workdir): + E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" + % (self.pn, self.workdir)) + error = e self._commit_changes()
Signature · 1 line
1.9.1
PATCH v2 3/11 upgradehelper.py: Only send emails when patch file exist
In reply to Aníbal LimónRemove old decision of send email based on error because only makes sense to send email when patch file was created.
--- upgradehelper.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-)
upgradehelper.py +2 -8
@@ -358,7 +358,8 @@ class Updater(object): self.git.reset_hard(1) self.git.clean_untracked() - if self.send_email: + # only send email to Maintainer when patch file exist + if self.send_email and self.patch_file: mail_header = \ "Hello,\n\nYou are receiving this email because you are the maintainer\n" \ "of *%s* recipe and this is to let you know that the automatic attempt\n" \
@@ -383,13 +384,6 @@ class Updater(object): "Any problem please contact Anibal Limon <[email protected]>.\n\n" \ "Regards,\nThe Upgrade Helper" - # only send email to Maintainer when is an error that can handle - if err and not isinstance(err, MaintainerError): - D( "%s: Don't send email to maintainer because the error was " \ - "%s and the information isn't useful, please review it." \ - % (self.pn, type(err).__name__)) - return - if self.maintainer in maintainer_override: to_addr = maintainer_override[self.maintainer] else:
Signature · 1 line
1.9.1
PATCH v2 4/11 bitbake.py: Improve performance on env() call
In reply to Aníbal LimónWhen AUH load recipe enviroment use bitbake -e call and parses the output using regex VAR=VALUE.
For improve performance discard all the comments on cmd output adding a output_filter using grep.
--- bitbake.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
bitbake.py +5 -2
@@ -42,7 +42,7 @@ class Bitbake(object): self.log_dir = None super(Bitbake, self).__init__() - def _cmd(self, recipe, options=None, env_var=None): + def _cmd(self, recipe, options=None, env_var=None, output_filter=None): cmd = "" if env_var is not None: cmd += env_var + " "
@@ -52,6 +52,9 @@ class Bitbake(object): cmd += recipe + if output_filter is not None: + cmd += ' | grep ' + output_filter + os.chdir(self.build_dir) try:
@@ -74,7 +77,7 @@ class Bitbake(object): return os.path.join(self.log_dir, "bitbake_log.txt") def env(self, recipe): - return self._cmd(recipe, "-e") + return self._cmd(recipe, "-e", output_filter="-v \"^#\"") def fetch(self, recipe): return self._cmd(recipe, "-c fetch")
Signature · 1 line
1.9.1
PATCH v2 5/11 upgradehelper.py: Add own step for loading environment
In reply to Aníbal LimónInstead of load environment in detecting git repository use own function making easy to debug.
--- upgradehelper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
upgradehelper.py +1 -3
@@ -147,6 +147,7 @@ class Updater(object): self.upgrade_steps = [ (self._create_workdir, None), + (self._get_env, "Loading environment ..."), (self._detect_repo, "Detecting git repository location ..."), (self._clean_repo, "Cleaning git repository of temporary branch ..."), (self._detect_recipe_type, None),
@@ -205,8 +206,6 @@ class Updater(object): os.remove(os.path.join(self.workdir, f)) def _detect_repo(self): - self._get_env() - if self.env['PV'] == self.new_ver: raise UpgradeNotNeededError
@@ -216,7 +215,6 @@ class Updater(object): self.git = Git(self.recipe_dir) stdout = self.git.status() - if stdout != "": if self.interactive: W(" %s: git repository has uncommited work which will be dropped! Proceed? (y/N)" % self.pn)
Signature · 1 line
1.9.1
PATCH v2 6/11 upgradehelper.py: step _detect_repo clarify why don't load git
In reply to Aníbal LimónAdd comment to clarify why don't load git when is already instanced the reason is the UniverseUpdater use poky git repository root instead of per recipe directory.
--- upgradehelper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
upgradehelper.py +4 -1
@@ -209,7 +209,8 @@ class Updater(object): if self.env['PV'] == self.new_ver: raise UpgradeNotNeededError - if self.git is not None: + # UniverseUpdater use git poky respository + if isinstance(self.git, UniverseUpdater): return self.git = Git(self.recipe_dir)
@@ -525,6 +526,8 @@ class Updater(object): class UniverseUpdater(Updater): def __init__(self): Updater.__init__(self, True, True) + + # XXX: assume that the poky directory is the first entry in the PATH self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0])) # read history file
Signature · 1 line
1.9.1
PATCH v2 7/11 upgradehelper.py: UniverseUpdate don't abort if recipe checkpkg fails.
In reply to Aníbal LimónWhen try to detect upstream version of recipes if one fails don't abort all the execution.
--- upgradehelper.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-)
upgradehelper.py +1 -4
@@ -285,10 +285,7 @@ class Updater(object): C(" \"distrodata.bbclass\" not inherited. Consider adding " "the following to your local.conf:\n\n" "INHERIT =+ \"distrodata\"\n") - else: - C(line) - - exit(1) + exit(1) def _parse_checkpkg_file(self, file_path): import csv
Signature · 1 line
1.9.1
PATCH v2 8/11 bitbake.py: Rename bitbake_log.txt to bitbake_error_log
In reply to Aníbal LimónBitbake log only is written when error appears, rename it to be consistent with information into the log.
--- bitbake.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
bitbake.py +4 -2
@@ -36,6 +36,8 @@ for path in os.environ["PATH"].split(':'): sys.path.insert(0, os.path.join(path, "../lib")) import bb +BITBAKE_ERROR_LOG = 'bitbake_error_log.txt' + class Bitbake(object): def __init__(self, build_dir): self.build_dir = build_dir
@@ -63,7 +65,7 @@ class Bitbake(object): D("%s returned:\n%s" % (cmd, e.__str__())) if self.log_dir is not None and os.path.exists(self.log_dir): - with open(os.path.join(self.log_dir, "bitbake_log.txt"), "w+") as log: + with open(os.path.join(self.log_dir, BITBAKE_ERROR_LOG), "w+") as log: log.write(e.stdout) raise Error("\'" + cmd + "\' failed", e.stdout, e.stderr)
@@ -74,7 +76,7 @@ class Bitbake(object): self.log_dir = dir def get_stdout_log(self): - return os.path.join(self.log_dir, "bitbake_log.txt") + return os.path.join(self.log_dir, BITBAKE_ERROR_LOG) def env(self, recipe): return self._cmd(recipe, "-e", output_filter="-v \"^#\"")
Signature · 1 line
1.9.1
PATCH v2 9/11 upgradehelper.py: Remove unused references to Buildhistory class
In reply to Aníbal LimónBuildhistory class isn't implemented so removed related code.
--- upgradehelper.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-)
upgradehelper.py +1 -10
@@ -43,7 +43,7 @@ from datetime import date import shutil from errors import * from git import Git -from bitbake import Bitbake, BuildHistory +from bitbake import Bitbake from emailhandler import Email from statistics import Statistics from recipe import Recipe
@@ -135,7 +135,6 @@ class Updater(object): os.mkdir(self.uh_work_dir) self.bb = Bitbake(get_build_dir()) - self.buildhistory = BuildHistory(get_build_dir()) self.git = None self.author_email = settings.get('from', '[email protected]') self.author = "Upgrade Helper <%s>" % self.author_email
@@ -266,14 +265,6 @@ class Updater(object): I(" %s: compiling for %s ..." % (self.pn, machine)) self.recipe.compile(machine) - def _review(self): - # Check build_history - if not self.skip_compilation: - I(" %s: Checking buildhistory ..." % self.pn) - self.buildhistory.set_work_dir(self.workdir) - if self.buildhistory.diff(len(self.machines)): - I(" %s: Wrote buildhistory-diff output ..." % self.pn) - def _check_upstream_versions(self, packages=[("universe", None, None)]): I(" Fetching upstream version(s) ...")
Signature · 1 line
1.9.1
PATCH v2 10/11 upgradehelper.py: Move upstream versioning code to UniverseUpdater
In reply to Aníbal LimónReorder code related to upstream versioning detection no makes sense to have in Updater because only in UniverseUpdater is used.
--- upgradehelper.py | 161 ++++++++++++++++++++++++++----------------------------- 1 file changed, 77 insertions(+), 84 deletions(-)
upgradehelper.py +77 -84
@@ -265,63 +265,12 @@ class Updater(object): I(" %s: compiling for %s ..." % (self.pn, machine)) self.recipe.compile(machine) - def _check_upstream_versions(self, packages=[("universe", None, None)]): - I(" Fetching upstream version(s) ...") - - try: - self.bb.checkpkg(" ".join([p[0] for p in packages])) - except Error as e: - for line in e.stdout.split('\n'): - if line.find("ERROR: Task do_checkpkg does not exist") == 0: - C(" \"distrodata.bbclass\" not inherited. Consider adding " - "the following to your local.conf:\n\n" - "INHERIT =+ \"distrodata\"\n") - exit(1) - - def _parse_checkpkg_file(self, file_path): - import csv - - pkgs_list = [] - - with open(file_path, "r") as f: - reader = csv.reader(f, delimiter='\t') - for row in reader: - if reader.line_num == 1: # skip header line - continue - - pn = row[0] - cur_ver = row[1] - next_ver = row[2] - status = row[11] - maintainer = row[14] - no_upgrade_reason = row[15] - - if status == 'UPDATE' and not no_upgrade_reason: - pkgs_list.append((pn, next_ver, maintainer)) - else: - if no_upgrade_reason: - D(" Skip package %s (status = %s, current version = %s," \ - " next version = %s, no upgrade reason = %s)" % - (pn, status, cur_ver, next_ver, no_upgrade_reason)) - else: - D(" Skip package %s (status = %s, current version = %s," \ - " next version = %s)" % - (pn, status, cur_ver, next_ver)) - return pkgs_list - def _get_packages_to_upgrade(self, packages=None): if packages is None: - return [] - - if len(packages) == 1: - # if user specified the version to upgrade to, just return the - # tuple intact - if packages[0][1] is not None: - return packages - - self._check_upstream_versions(packages) - - return self._parse_checkpkg_file(get_build_dir() + "/tmp/log/checkpkg.csv") + I( "Nothing to upgrade") + exit(0) + else: + return packages # this function will be called at the end of each recipe upgrade def pkg_upgrade_handler(self, err):
@@ -530,6 +479,75 @@ class UniverseUpdater(Updater): line.split(',')[3], line.split(',')[4]] + def _update_master(self): + I(" Drop all uncommited changes (including untracked) ...") + self.git.reset_hard() + self.git.clean_untracked() + + self.git.checkout_branch("master") + try: + self.git.delete_branch("upgrades") + except Error: + pass + I(" Sync master ...") + self.git.pull() + self.git.create_branch("upgrades") + + def _prepare(self): + if settings.get("clean_sstate", "no") == "yes" and \ + os.path.exists(os.path.join(get_build_dir(), "sstate-cache")): + I(" Removing sstate directory ...") + shutil.rmtree(os.path.join(get_build_dir(), "sstate-cache")) + if settings.get("clean_tmp", "no") == "yes" and \ + os.path.exists(os.path.join(get_build_dir(), "tmp")): + I(" Removing tmp directory ...") + shutil.rmtree(os.path.join(get_build_dir(), "tmp")) + + + def _check_upstream_versions(self, packages=[("universe", None, None)]): + I(" Fetching upstream version(s) ...") + + try: + self.bb.checkpkg(" ".join([p[0] for p in packages])) + except Error as e: + for line in e.stdout.split('\n'): + if line.find("ERROR: Task do_checkpkg does not exist") == 0: + C(" \"distrodata.bbclass\" not inherited. Consider adding " + "the following to your local.conf:\n\n" + "INHERIT =+ \"distrodata\"\n") + exit(1) + + def _parse_checkpkg_file(self, file_path): + import csv + + pkgs_list = [] + + with open(file_path, "r") as f: + reader = csv.reader(f, delimiter='\t') + for row in reader: + if reader.line_num == 1: # skip header line + continue + + pn = row[0] + cur_ver = row[1] + next_ver = row[2] + status = row[11] + maintainer = row[14] + no_upgrade_reason = row[15] + + if status == 'UPDATE' and not no_upgrade_reason: + pkgs_list.append((pn, next_ver, maintainer)) + else: + if no_upgrade_reason: + D(" Skip package %s (status = %s, current version = %s," \ + " next version = %s, no upgrade reason = %s)" % + (pn, status, cur_ver, next_ver, no_upgrade_reason)) + else: + D(" Skip package %s (status = %s, current version = %s," \ + " next version = %s)" % + (pn, status, cur_ver, next_ver)) + return pkgs_list + # checks if maintainer is in whitelist and that the recipe itself is not # blacklisted: python, gcc, etc. Also, check the history if the recipe # hasn't already been tried
@@ -580,30 +598,6 @@ class UniverseUpdater(Updater): return True - def update_master(self): - I(" Drop all uncommited changes (including untracked) ...") - self.git.reset_hard() - self.git.clean_untracked() - - self.git.checkout_branch("master") - try: - self.git.delete_branch("upgrades") - except Error: - pass - I(" Sync master ...") - self.git.pull() - self.git.create_branch("upgrades") - - def prepare(self): - if settings.get("clean_sstate", "no") == "yes" and \ - os.path.exists(os.path.join(get_build_dir(), "sstate-cache")): - I(" Removing sstate directory ...") - shutil.rmtree(os.path.join(get_build_dir(), "sstate-cache")) - if settings.get("clean_tmp", "no") == "yes" and \ - os.path.exists(os.path.join(get_build_dir(), "tmp")): - I(" Removing tmp directory ...") - shutil.rmtree(os.path.join(get_build_dir(), "tmp")) - def _get_packages_to_upgrade(self, packages=None): last_date_checked = None last_master_commit = None
@@ -649,7 +643,7 @@ class UniverseUpdater(Updater): return pkgs_list - def update_history(self, pn, new_ver, maintainer, upgrade_status): + def _update_history(self, pn, new_ver, maintainer, upgrade_status): with open(self.history_file + ".tmp", "w+") as tmp_file: if os.path.exists(self.history_file): with open(self.history_file) as history:
@@ -661,15 +655,14 @@ class UniverseUpdater(Updater): upgrade_status + "\n") os.rename(self.history_file + ".tmp", self.history_file) - # overriding the base method def pkg_upgrade_handler(self, err): super(UniverseUpdater, self).pkg_upgrade_handler(err) - self.update_history(self.pn, self.new_ver, self.maintainer, + self._update_history(self.pn, self.new_ver, self.maintainer, self._get_status_msg(err)) def run(self): - self.update_master() - self.prepare() + self._update_master() + self._prepare() super(UniverseUpdater, self).run() def close_child_processes(signal_id, frame):
Signature · 1 line
1.9.1
PATCH v2 11/11 upgradehelper: Add support for generate buildhistory recipe diff's
In reply to Aníbal LimónNow AUH generates buildhistory diff when recipe was upgraded successfully.
For enable this feature set into local.conf.
INHERIT += "buildhistory" BUILDHISTORY_COMMIT = "1"
Summary of the changes,
- bitbake.py: Enable environment generation without recipe, removes Buildhistory class.
- buildhistory.py: Add buildhistory class that generates initial buildhistory revision and keeps track of build revisions for generate diff when build is successful.
- upgradehelper.py: Add buildhistory steps for upgrade, add support for detect when buildhistory is enabled loading the environment without recipe.
[YOCTO #7175]
--- bitbake.py | 41 ++------------------ buildhistory.py | 75 ++++++++++++++++++++++++++++++++++++ errors.py | 6 +++ upgradehelper.py | 115 ++++++++++++++++++++++++++++++++++++++++--------------- 4 files changed, 170 insertions(+), 67 deletions(-) create mode 100644 buildhistory.py
bitbake.py +4 -37
@@ -44,7 +44,7 @@ class Bitbake(object): self.log_dir = None super(Bitbake, self).__init__() - def _cmd(self, recipe, options=None, env_var=None, output_filter=None): + def _cmd(self, recipe=None, options=None, env_var=None, output_filter=None): cmd = "" if env_var is not None: cmd += env_var + " "
@@ -52,7 +52,8 @@ class Bitbake(object): if options is not None: cmd += options + " " - cmd += recipe + if recipe is not None: + cmd += recipe if output_filter is not None: cmd += ' | grep ' + output_filter
@@ -78,7 +79,7 @@ class Bitbake(object): def get_stdout_log(self): return os.path.join(self.log_dir, BITBAKE_ERROR_LOG) - def env(self, recipe): + def env(self, recipe=None): return self._cmd(recipe, "-e", output_filter="-v \"^#\"") def fetch(self, recipe):
@@ -104,37 +105,3 @@ class Bitbake(object): def dependency_graph(self, package_list): return self._cmd(package_list, "-g") - -class BuildHistory(object): - def __init__(self, build_dir): - self.build_dir = build_dir - self.work_dir = None - - def set_work_dir(self, work_dir): - self.work_dir = work_dir - - # Return True if buildhistory-diff gives output - def diff(self, revision_steps): - os.chdir(self.build_dir) - cmd = "buildhistory-diff HEAD~" + str(revision_steps) - - try: - stdout, stderr = bb.process.run(cmd) - # Write diff output to log file if there is any - - if stdout and os.path.exists(self.work_dir): - with open(os.path.join(self.work_dir, "buildhistory.txt"), "w+") as log: - log.write(stdout) - return True - except bb.process.ExecutionError as e: - for line in e.stdout.split('\n'): - if line.find("Buildhistory directory \"buildhistory/\" does not exist") == 0: - C(" \"buildhistory.bbclass\" not inherited. Consider adding " - "the following to your local.conf:\n\n" - "INHERIT =+ \"buildhistory\"\n" - "BUILDHISTORY_COMMIT = \"1\"\n\n" - "Do not remove any other inherited class in the process (e.g. distrodata)\n") - exit(1) - - return False -
buildhistory.py new file +75 -0
75 lines not on this page · raw message
errors.py +6 -0
@@ -85,3 +85,9 @@ class UpgradeNotNeededError(Error): def __str__(self): return "Failed(up to date)" +class EmptyEnvError(Error): + def __init__(self, stdout): + super(EmptyEnvError, self).__init__("Empty environment returned", stdout) + + def __str__(self): + return "Failed(get_env)"
upgradehelper.py +85 -30
179 lines not on this page · raw message
Signature · 1 line
1.9.1