00/17] AUH code refactor and support test image.
The following patchset contains,
- A refactor of AUH code clean-up and improve source tree.
- A implementation of testimage that do ptest and sato testimages.
- New settings for publish patches and logs generated by AUH.
- Improvements on the status email.
The changes can also be found at,
http://git.yoctoproject.org/cgit/cgit.cgi/auto-upgrade-helper/log/?h=boot_images_v2
Aníbal Limón (17):
{upgradehelper, bitbake}.py: Move _get_env function to bitbake upgradehelper.py: Merge options into a dictionary upgradehelper.py: Adds own module for steps
upgradehelper: Reorder files into directories.
buildhistory: Add option for enable in upgrade-helper.conf recipe/base.py: Add is_recipe_or_include_file func recipe/base.py: Add modify_recipe_files function decorator recipe/base.py: Add support for get recipe inherits. steps.py: Merge load_dirs step into load_env. steps.py: Move clean_repo to first step. utils/git.py: Add method for apply patches into a branch. upgradehelper.py: Add settings for enable testimage.
upgradehelper: Add testimage feature. upgradehelper.py: Changed retry failure build to 30 days.
upgradehelper: Add workdir setting.
statistics: Improve email format and get_summary method.
statistics: Add support for publish_work_url setting.
README | 40 ++- bitbake.py | 107 ------- buildhistory.py | 75 ----- emailhandler.py | 108 ------- errors.py | 93 ------ git.py | 101 ------- gitrecipe.py | 94 ------ modules/buildhistory.py | 75 +++++ modules/errors.py | 93 ++++++ modules/recipe/__init__.py | 0 modules/recipe/base.py | 682 ++++++++++++++++++++++++++++++++++++++++++ modules/recipe/git.py | 94 ++++++ modules/recipe/svn.py | 28 ++ modules/statistics.py | 112 +++++++ modules/steps.py | 144 +++++++++ modules/testimage.py | 185 ++++++++++++ modules/utils/__init__.py | 0 modules/utils/bitbake.py | 124 ++++++++ modules/utils/emailhandler.py | 108 +++++++ modules/utils/git.py | 105 +++++++ recipe.py | 669 ----------------------------------------- statistics.py | 102 ------- svnrecipe.py | 28 -- upgradehelper.py | 505 +++++++++++++++---------------- 24 files changed, 2042 insertions(+), 1630 deletions(-) delete mode 100644 bitbake.py delete mode 100644 buildhistory.py delete mode 100644 emailhandler.py delete mode 100644 errors.py delete mode 100644 git.py delete mode 100644 gitrecipe.py create mode 100644 modules/buildhistory.py create mode 100644 modules/errors.py create mode 100644 modules/recipe/__init__.py create mode 100644 modules/recipe/base.py create mode 100644 modules/recipe/git.py create mode 100644 modules/recipe/svn.py create mode 100644 modules/statistics.py create mode 100644 modules/steps.py create mode 100644 modules/testimage.py create mode 100644 modules/utils/__init__.py create mode 100644 modules/utils/bitbake.py create mode 100644 modules/utils/emailhandler.py create mode 100644 modules/utils/git.py delete mode 100644 recipe.py delete mode 100644 statistics.py delete mode 100644 svnrecipe.py
Signature · 1 line
2.1.4
[[AUH] 02/17] upgradehelper.py: Merge options into a dictionary
Use one dictionary to keep together the options this enables possibility to define an interface for steps, see next commit.
--- upgradehelper.py | 83 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 42 insertions(+), 41 deletions(-)
upgradehelper.py +42 -41
@@ -126,15 +126,38 @@ def parse_config_file(config_file): class Updater(object): def __init__(self, auto_mode=False, send_email=False, skip_compilation=False): + build_dir = get_build_dir() - self.uh_dir = os.path.join(get_build_dir(), "upgrade-helper") + self.bb = Bitbake(build_dir) + + try: + self.base_env = self.bb.env() + except EmptyEnvError as e: + import traceback + E( " %s\n%s" % (e.message, traceback.format_exc())) + E( " Bitbake output:\n%s" % (e.stdout)) + exit(1) + + self.email_handler = Email(settings) + self.statistics = Statistics() + self.git = None + + self.opts = {} + self.opts['interactive'] = not auto_mode + self.opts['send_email'] = send_email + self.opts['author'] = "Upgrade Helper <%s>" % \ + settings.get('from', '[email protected]') + self.opts['machines'] = settings.get('machines', + 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() + self.opts['skip_compilation'] = skip_compilation + self.opts['buildhistory_enabled'] = self._buildhistory_is_enabled() + + self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir): os.mkdir(self.uh_dir) - self.uh_work_dir = os.path.join(self.uh_dir, "work-%s" % \ datetime.now().strftime("%Y%m%d%H%M%S")) os.mkdir(self.uh_work_dir) - self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all") os.mkdir(self.uh_recipes_all_dir) self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed")
@@ -142,16 +165,6 @@ class Updater(object): self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed") os.mkdir(self.uh_recipes_failed_dir) - self.bb = Bitbake(get_build_dir()) - self.git = None - self.author_email = settings.get('from', '[email protected]') - self.author = "Upgrade Helper <%s>" % self.author_email - self.skip_compilation = skip_compilation - self.interactive = not auto_mode - self.send_email = send_email - - self.machines = settings.get('machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() - self.upgrade_steps = [ (self._load_env, "Loading environment ..."), (self._create_workdir, None),
@@ -167,18 +180,6 @@ class Updater(object): (self._buildhistory_diff, None) ] - try: - self.base_env = self.bb.env() - except EmptyEnvError as e: - import traceback - E( " %s\n%s" % (e.message, traceback.format_exc())) - E( " Bitbake output:\n%s" % (e.stdout)) - exit(1) - self.buildhistory_enabled = self._buildhistory_is_enabled() - - self.email_handler = Email(settings) - self.statistics = Statistics() - def _get_status_msg(self, err): if err: return str(err)
@@ -199,7 +200,7 @@ class Updater(object): " BUILDHISTORY_COMMIT=1 please set.") exit(1) - if self.skip_compilation: + if self.opts['skip_compilation']: W(" Buildhistory disabled because user" \ " skip compilation!") else:
@@ -228,7 +229,7 @@ class Updater(object): stdout = self.git.status() if stdout != "": - if self.interactive: + if self.opts['interactive']: W(" %s: git repository has uncommited work which will be dropped! Proceed? (y/N)" % self.pn) answer = sys.stdin.readline().strip().upper() if answer == '' or answer != 'Y':
@@ -261,16 +262,16 @@ class Updater(object): else: raise UnsupportedProtocolError - self.recipe = recipe(self.env, self.new_ver, self.interactive, self.workdir, + self.recipe = recipe(self.env, self.new_ver, self.opts['interactive'], self.workdir, self.recipe_dir, self.bb, self.git) def _buildhistory_init(self): - if self.buildhistory_enabled == False: + if not self.opts['buildhistory_enabled']: return self.buildhistory = BuildHistory(self.bb, self.pn, self.workdir) - I(" %s: Initial buildhistory for %s ..." % (self.pn, self.machines)) - self.buildhistory.init(self.machines) + I(" %s: Initial buildhistory for %s ..." % (self.pn, self.opts['machines'])) + self.buildhistory.init(self.opts['machines']) def _unpack_original(self): self.recipe.unpack()
@@ -289,18 +290,18 @@ class Updater(object): self.recipe.fetch() def _compile(self): - if self.skip_compilation: + if self.opts['skip_compilation']: W(" %s: Compilation was skipped by user choice!") return - for machine in self.machines: + for machine in self.opts['machines']: I(" %s: compiling for %s ..." % (self.pn, machine)) self.recipe.compile(machine) - if self.buildhistory_enabled == True: + if self.opts['buildhistory_enabled']: self.buildhistory.add() def _buildhistory_diff(self): - if self.buildhistory_enabled == False: + if not self.opts['buildhistory_enabled']: return I(" %s: Checking buildhistory ..." % self.pn)
@@ -315,7 +316,7 @@ class Updater(object): # this function will be called at the end of each recipe upgrade def pkg_upgrade_handler(self, err): - if err and self.patch_file and self.interactive: + if err and self.patch_file and self.opts['interactive']: answer = "N" I(" %s: Do you want to keep the changes? (y/N)" % self.pn) answer = sys.stdin.readline().strip().upper()
@@ -380,7 +381,7 @@ class Updater(object): if license_diff_fn: msg_body += license_change_info % license_diff_fn if not err: - msg_body += next_steps_info % (', '.join(self.machines), + msg_body += next_steps_info % (', '.join(self.opts['machines']), os.path.basename(self.patch_file)) msg_body += mail_footer
@@ -393,7 +394,7 @@ class Updater(object): attachments.append(attachment_fullpath) # Only send email to Maintainer when recipe upgrade succeed. - if self.send_email and not err: + if self.opts['send_email'] and not err: self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr) # Preserve email for review purposes.
@@ -415,7 +416,7 @@ class Updater(object): self.patch_file = None if self.recipe is not None: I(" %s: Auto commit changes ..." % self.pn) - self.git.commit(self.recipe.commit_msg, self.author) + self.git.commit(self.recipe.commit_msg, self.opts['author']) I(" %s: Save patch in %s." % (self.pn, self.workdir)) stdout = self.git.create_patch(self.workdir) self.patch_file = stdout.strip()
@@ -525,7 +526,7 @@ class Updater(object): def run(self, package_list=None): I(" Building gcc runtimes ...") - for machine in self.machines: + for machine in self.opts['machines']: I(" building gcc runtime for %s" % machine) self.bb.complete("gcc-runtime", machine)
@@ -595,7 +596,7 @@ class Updater(object): I("%s" % statistics_summary) - if self.send_email: + if self.opts['send_email']: self.send_status_mail() class UniverseUpdater(Updater):
Signature · 1 line
2.1.4
[[AUH] 01/17] {upgradehelper, bitbake}.py: Move _get_env function to bitbake
In reply to Aníbal LimónRemove unnecesary access function (_get_env) to environment move logic to build dictionary to bitbake env method.
--- bitbake.py | 19 ++++++++++++++++++- upgradehelper.py | 26 ++++---------------------- 2 files changed, 22 insertions(+), 23 deletions(-)
bitbake.py +18 -1
@@ -29,6 +29,8 @@ from logging import debug as D from logging import error as E from logging import critical as C import sys +import re + from errors import * for path in os.environ["PATH"].split(':'):
@@ -80,7 +82,22 @@ class Bitbake(object): return os.path.join(self.log_dir, BITBAKE_ERROR_LOG) def env(self, recipe=None): - return self._cmd(recipe, "-e", output_filter="-v \"^#\"") + stdout = self._cmd(recipe, "-e", output_filter="-v \"^#\"") + + assignment = re.compile("^([^ \t=]*)=(.*)") + bb_env = dict() + for line in stdout.split('\n'): + m = assignment.match(line) + if m: + if m.group(1) in bb_env: + continue + + bb_env[m.group(1)] = m.group(2).strip("\"") + + if not bb_env: + raise EmptyEnvError(stdout) + + return bb_env def fetch(self, recipe): return self._cmd(recipe, "-c fetch")
upgradehelper.py +4 -22
@@ -168,7 +168,7 @@ class Updater(object): ] try: - self.base_env = self._get_env() + self.base_env = self.bb.env() except EmptyEnvError as e: import traceback E( " %s\n%s" % (e.message, traceback.format_exc()))
@@ -185,24 +185,6 @@ class Updater(object): else: return "Succeeded" - def _get_env(self, pn=None): - stdout = self.bb.env(pn) - - assignment = re.compile("^([^ \t=]*)=(.*)") - bb_env = dict() - for line in stdout.split('\n'): - m = assignment.match(line) - if m: - if m.group(1) in bb_env: - continue - - bb_env[m.group(1)] = m.group(2).strip("\"") - - if not bb_env: - raise EmptyEnvError(stdout) - - return bb_env - def _buildhistory_is_enabled(self): enabled = False
@@ -226,7 +208,7 @@ class Updater(object): return enabled def _load_env(self): - self.env = self._get_env(self.pn) + self.env = self.bb.env(self.pn) def _create_workdir(self): self.workdir = os.path.join(self.uh_recipes_all_dir, self.pn)
@@ -257,7 +239,7 @@ class Updater(object): self.git.reset_hard() self.git.clean_untracked() - self.env = self._get_env(self.pn) + self.env = self.bb.env(self.pn) def _clean_repo(self): try:
@@ -296,7 +278,7 @@ class Updater(object): def _rename(self): self.recipe.rename() - self.env = self._get_env(self.pn) + self.env = self.bb.env(self.pn) self.recipe.update_env(self.env)
Signature · 1 line
2.1.4
[[AUH] 05/17] buildhistory: Add option for enable in upgrade-helper.conf
In reply to Aníbal LimónMake consistent the enablement for this feature it need to be explicit enable into upgrade-helper.conf and also needs configuration in local.conf.
--- README | 9 +++++++-- modules/steps.py | 6 +++--- upgradehelper.py | 40 ++++++++++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 19 deletions(-)
README +7 -2
@@ -58,6 +58,10 @@ drop_previous_commits=yes # machines to test build with machines=qemux86 qemux86-64 qemuarm qemumips qemuppc + +# optional features +buildhistory=no + --------------- snip --------------- 3. Enable distrodata and supply appropriate additional metadata. For
@@ -75,8 +79,9 @@ WARNING: if you are using the default maintainers.inc file supplied emails to the default maintainers. Please be careful not to do this :) -4. Enable buildhistory by adding the following lines to your -conf/local.conf file: +4. If you want to enable buildhistory (optional) you need to enable in +upgrade-helper.conf also add the following lines to your conf/local.conf +file: --------------- snip --------------- INHERIT =+ "buildhistory"
modules/steps.py +3 -3
@@ -85,7 +85,7 @@ def detect_recipe_type(bb, git, opts, pkg_ctx): pkg_ctx['recipe_dir'], bb, git) def buildhistory_init(bb, git, opts, pkg_ctx): - if not opts['buildhistory_enabled']: + if not opts['buildhistory']: return pkg_ctx['buildhistory'] = BuildHistory(bb, pkg_ctx['PN'],
@@ -118,11 +118,11 @@ def compile(bb, git, opts, pkg_ctx): for machine in opts['machines']: I(" %s: compiling for %s ..." % (pkg_ctx['PN'], machine)) pkg_ctx['recipe'].compile(machine) - if opts['buildhistory_enabled']: + if opts['buildhistory']: pkg_ctx['buildhistory'].add() def buildhistory_diff(bb, git, opts, pkg_ctx): - if not opts['buildhistory_enabled']: + if not opts['buildhistory']: return I(" %s: Checking buildhistory ..." % pkg_ctx['PN'])
upgradehelper.py +26 -14
@@ -156,7 +156,7 @@ class Updater(object): self.opts['machines'] = settings.get('machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['skip_compilation'] = skip_compilation - self.opts['buildhistory_enabled'] = self._buildhistory_is_enabled() + self.opts['buildhistory'] = self._buildhistory_is_enabled() self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir):
@@ -180,22 +180,34 @@ class Updater(object): def _buildhistory_is_enabled(self): enabled = False - if 'buildhistory' in self.base_env['INHERIT']: - if not 'BUILDHISTORY_COMMIT' in self.base_env: - E(" Buildhistory was enabled but need"\ - " BUILDHISTORY_COMMIT=1 please set.") - exit(1) + if settings.get("buildhistory", "no") == "yes": + if 'buildhistory' in self.base_env['INHERIT']: + if not 'BUILDHISTORY_COMMIT' in self.base_env: + E(" Buildhistory was INHERIT in conf/local.conf"\ + " but need BUILDHISTORY_COMMIT=1 please set.") + exit(1) - if not self.base_env['BUILDHISTORY_COMMIT'] == '1': - E(" Buildhistory was enabled but need"\ - " BUILDHISTORY_COMMIT=1 please set.") - exit(1) + if not self.base_env['BUILDHISTORY_COMMIT'] == '1': + E(" Buildhistory was INHERIT in conf/local.conf"\ + " but need BUILDHISTORY_COMMIT=1 please set.") + exit(1) - if self.opts['skip_compilation']: - W(" Buildhistory disabled because user" \ - " skip compilation!") + if self.opts['skip_compilation']: + W(" Buildhistory disabled because user" \ + " skip compilation!") + else: + enabled = True else: - enabled = True + E(" Buildhistory was enabled in upgrade-helper.conf"\ + " but isn't INHERIT in conf/local.conf, if you want"\ + " to enable please set.") + exit(1) + else: + if 'buildhistory' in self.base_env['INHERIT']: + E(" Buildhistory was INHERIT in conf/local.conf"\ + " but buildhistory=yes isn't in upgrade-helper.conf,"\ + " if you want to enable please set.") + exit(1) return enabled
Signature · 1 line
2.1.4
[[AUH] 03/17] upgradehelper.py: Adds own module for steps
In reply to Aníbal LimónIsolate steps into own module in order to provide better structure of the code.
In order to do that the step interfaces was defined to, step(bb, git, opts, pkg_ctx)
- bb: Bitbake helper instance.
- git: Git helper instance.
- opts: Upgrade helper options.
- pkg_ctx: Package context per upgrade, it stores all information related to one upgrade.
--- steps.py | 142 ++++++++++++++++++++++++++++++ upgradehelper.py | 259 +++++++++++++++++-------------------------------------- 2 files changed, 221 insertions(+), 180 deletions(-) create mode 100644 steps.py
steps.py new file +142 -0
@@ -0,0 +1,142 @@ +#!/usr/bin/env python +# vim: set ts=4 sw=4 et: +# +# Copyright (c) 2013 - 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +import os +import sys + +from logging import debug as D +from logging import info as I +from logging import warning as W +from logging import error as E +from logging import critical as C + +from errors import * +from buildhistory import BuildHistory +from recipe import Recipe +from gitrecipe import GitRecipe +from svnrecipe import SvnRecipe + +def load_env(bb, git, opts, pkg_ctx): + stdout = git.status() + if stdout != "": + if opts['interactive']: + W(" %s: git repository has uncommited work which will be dropped!" \ + " Proceed? (y/N)" % pkg_ctx['PN']) + answer = sys.stdin.readline().strip().upper() + if answer == '' or answer != 'Y': + I(" %s: User abort!" % pkg_ctx['PN']) + exit(1) + + I(" %s: Dropping uncommited work!" % pkg_ctx['PN']) + git.reset_hard() + git.clean_untracked() + + pkg_ctx['env'] = bb.env(pkg_ctx['PN']) + if pkg_ctx['env']['PV'] == pkg_ctx['NPV']: + raise UpgradeNotNeededError + +def load_dirs(bb, git, opts, pkg_ctx): + pkg_ctx['workdir'] = os.path.join(pkg_ctx['base_dir'], pkg_ctx['PN']) + os.mkdir(pkg_ctx['workdir']) + + pkg_ctx['recipe_dir'] = os.path.dirname(pkg_ctx['env']['FILE']) + +def clean_repo(bb, git, opts, pkg_ctx): + try: + git.checkout_branch("upgrades") + except Error: + git.create_branch("upgrades") + + try: + git.delete_branch("remove_patches") + except: + pass + +def detect_recipe_type(bb, git, opts, pkg_ctx): + if pkg_ctx['env']['SRC_URI'].find("ftp://") != -1 or \ + pkg_ctx['env']['SRC_URI'].find("http://") != -1 or \ + pkg_ctx['env']['SRC_URI'].find("https://") != -1: + recipe = Recipe + elif pkg_ctx['env']['SRC_URI'].find("git://") != -1: + recipe = GitRecipe + else: + raise UnsupportedProtocolError + + pkg_ctx['recipe'] = recipe(pkg_ctx['env'], pkg_ctx['NPV'], + opts['interactive'], pkg_ctx['workdir'], + pkg_ctx['recipe_dir'], bb, git) + +def buildhistory_init(bb, git, opts, pkg_ctx): + if not opts['buildhistory_enabled']: + return + + pkg_ctx['buildhistory'] = BuildHistory(bb, pkg_ctx['PN'], + pkg_ctx['workdir']) + I(" %s: Initial buildhistory for %s ..." % (pkg_ctx['PN'], + opts['machines'])) + pkg_ctx['buildhistory'].init(opts['machines']) + +def unpack_original(bb, git, opts, pkg_ctx): + pkg_ctx['recipe'].unpack() + +def rename(bb, git, opts, pkg_ctx): + pkg_ctx['recipe'].rename() + + pkg_ctx['env'] = bb.env(pkg_ctx['PN']) + + pkg_ctx['recipe'].update_env(pkg_ctx['env']) + +def cleanall(bb, git, opts, pkg_ctx): + pkg_ctx['recipe'].cleanall() + +def fetch(bb, git, opts, pkg_ctx): + pkg_ctx['recipe'].fetch() + +def compile(bb, git, opts, pkg_ctx): + if opts['skip_compilation']: + W(" %s: Compilation was skipped by user choice!") + return + + for machine in opts['machines']: + I(" %s: compiling for %s ..." % (pkg_ctx['PN'], machine)) + pkg_ctx['recipe'].compile(machine) + if opts['buildhistory_enabled']: + pkg_ctx['buildhistory'].add() + +def buildhistory_diff(bb, git, opts, pkg_ctx): + if not opts['buildhistory_enabled']: + return + + I(" %s: Checking buildhistory ..." % pkg_ctx['PN']) + pkg_ctx['buildhistory'].diff() + +upgrade_steps = [ + (load_env, "Loading environment ..."), + (load_dirs, None), + (clean_repo, "Cleaning git repository of temporary branch ..."), + (detect_recipe_type, None), + (buildhistory_init, None), + (unpack_original, "Fetch & unpack original version ..."), + (rename, "Renaming recipes, reset PR (if exists) ..."), + (cleanall, "Clean all ..."), + (fetch, "Fetch new version (old checksums) ..."), + (compile, None), + (buildhistory_diff, None) +]
upgradehelper.py +79 -180
416 lines not on this page · raw message
Signature · 1 line
2.1.4
[[AUH] 06/17] recipe/base.py: Add is_recipe_or_include_file func
In reply to Aníbal LimónInstead of have duplicate code add this new function for make code easy to read.
--- modules/recipe/base.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-)
modules/recipe/base.py +17 -18
@@ -33,6 +33,18 @@ from logging import warning as W from errors import * from utils.bitbake import * +def is_recipe_or_include_file(full_path_f, f): + is_file = os.path.isfile(full_path_f) + + is_recipe = f.find(self.env['PN']) == 0 and \ + f.find(self.env['PKGV']) != -1 and \ + f.find(".bb") != -1 + + is_include = f.find(self.env['PN']) == 0 and \ + f.find(".inc") != -1 + + return is_file and (is_recipe or is_include) + class Recipe(object): def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git): self.env = env
@@ -63,6 +75,7 @@ class Recipe(object): super(Recipe, self).__init__() + def update_env(self, env): self.env = env
@@ -80,10 +93,7 @@ class Recipe(object): # change PR before renaming for f in os.listdir(self.recipe_dir): full_path_f = os.path.join(self.recipe_dir, f) - if os.path.isfile(full_path_f) and \ - ((f.find(self.env['PN']) == 0 and f.find(self.env['PKGV']) != -1 and - f.find(".bb") != -1) or - (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)): + if is_recipe_or_include_file(full_path_f, f): with open(full_path_f + ".tmp", "w+") as temp_recipe: with open(full_path_f) as recipe: for line in recipe:
@@ -130,12 +140,7 @@ class Recipe(object): for f in os.listdir(self.recipe_dir): full_path_f = os.path.join(self.recipe_dir, f) - if os.path.isfile(full_path_f) and \ - ((f.find(self.env['PN']) == 0 and - f.find(self.env['PKGV']) != -1 and - f.find(".bb") != -1) or - (f.find(self.env['PN']) == 0 and - f.find(".inc") != -1)): + if is_recipe_or_include_file(full_path_f, f): with open(full_path_f + ".tmp", "w+") as temp_recipe: with open(full_path_f) as recipe: for line in recipe:
@@ -178,10 +183,7 @@ class Recipe(object): # files... Go through the recipes/inc files until we find them for f in os.listdir(self.recipe_dir): full_path_f = os.path.join(self.recipe_dir, f) - if os.path.isfile(full_path_f) and \ - ((f.find(self.env['PN']) == 0 and f.find(self.env['PKGV']) != -1 and - f.find(".bb") != -1) or - (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)): + if is_recipe_or_include_file(full_path_f, f): with open(full_path_f + ".tmp", "w+") as temp_recipe: with open(full_path_f) as recipe: for line in recipe:
@@ -218,10 +220,7 @@ class Recipe(object): # Will change the extension of the archive from the SRC_URI for f in os.listdir(self.recipe_dir): full_path_f = os.path.join(self.recipe_dir, f) - if os.path.isfile(full_path_f) and \ - ((f.find(self.env['PN']) == 0 and f.find(self.env['PKGV']) != -1 and - f.find(".bb") != -1) or - (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)): + if is_recipe_or_include_file(full_path_f, f): with open(full_path_f + ".tmp", "w+") as temp_recipe: with open(full_path_f) as recipe: source_found = False
Signature · 1 line
2.1.4
[[AUH] 07/17] recipe/base.py: Add modify_recipe_files function decorator
In reply to Aníbal LimónRemove duplicate code adding modify_recipe_files decorator, this function interate over recipe_dir and found bb and includes files to make modifications.
Modifications to recipe bb and include files are made by function passed to modify_recipe_files.
--- modules/recipe/base.py | 293 +++++++++++++++++++++++-------------------------- 1 file changed, 140 insertions(+), 153 deletions(-)
modules/recipe/base.py +140 -153
@@ -33,18 +33,30 @@ from logging import warning as W from errors import * from utils.bitbake import * -def is_recipe_or_include_file(full_path_f, f): +def is_recipe_or_include_file(env, full_path_f, f): is_file = os.path.isfile(full_path_f) - is_recipe = f.find(self.env['PN']) == 0 and \ - f.find(self.env['PKGV']) != -1 and \ + is_recipe = f.find(env['PN']) == 0 and \ + f.find(env['PKGV']) != -1 and \ f.find(".bb") != -1 - is_include = f.find(self.env['PN']) == 0 and \ + is_include = f.find(env['PN']) == 0 and \ f.find(".inc") != -1 return is_file and (is_recipe or is_include) +def modify_recipe_files(func): + def modify(env, recipe_dir): + for f in os.listdir(recipe_dir): + full_path_f = os.path.join(recipe_dir, f) + if is_recipe_or_include_file(env, full_path_f, f): + with open(full_path_f + ".tmp", "w+") as temp_recipe: + with open(full_path_f) as recipe: + for line in recipe: + func(line, temp_recipe) + os.rename(full_path_f + ".tmp", full_path_f) + return modify + class Recipe(object): def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git): self.env = env
@@ -90,18 +102,12 @@ class Recipe(object): self.git.mv(src_dir, dest_dir) def rename(self): - # change PR before renaming - for f in os.listdir(self.recipe_dir): - full_path_f = os.path.join(self.recipe_dir, f) - if is_recipe_or_include_file(full_path_f, f): - with open(full_path_f + ".tmp", "w+") as temp_recipe: - with open(full_path_f) as recipe: - for line in recipe: - if line.startswith("PR=") or line.startswith("PR ="): - continue - else: - temp_recipe.write(line) - os.rename(full_path_f + ".tmp", full_path_f) + # clean PR before renaming + @modify_recipe_files + def _clean_pr(line, temp_recipe): + if not (line.startswith("PR=") or line.startswith("PR =")): + temp_recipe.write(line) + _clean_pr(self.env, self.recipe_dir) # rename recipes (not directories) for path in os.listdir(self.recipe_dir):
@@ -121,8 +127,6 @@ class Recipe(object): # since we did some renaming, backup the current environment self.old_env = self.env - # start formatting the commit message - def create_diff_file(self, file, old_md5, new_md5): old_file = os.path.join(self.old_env['S'], file) new_file = os.path.join(self.env['S'], file)
@@ -138,20 +142,6 @@ class Recipe(object): f.write("old checksum = %s\n" % old_md5) f.write("new_checksum = %s\n" % new_md5) - for f in os.listdir(self.recipe_dir): - full_path_f = os.path.join(self.recipe_dir, f) - if is_recipe_or_include_file(full_path_f, f): - with open(full_path_f + ".tmp", "w+") as temp_recipe: - with open(full_path_f) as recipe: - for line in recipe: - m = re.match("(.*)" + old_md5 + "(.*)", line) - if m is not None: - temp_recipe.write(m.group(1) + new_md5 + m.group(2) + "\n") - else: - temp_recipe.write(line) - - os.rename(full_path_f + ".tmp", full_path_f) - def _change_recipe_checksums(self, fetch_log): sums = {}
@@ -178,27 +168,23 @@ class Recipe(object): if len(sums) == 0: raise FetchError() - I(" %s: Update recipe checksums ..." % self.env['PN']) # checksums are usually in the main recipe but they can also be in inc # files... Go through the recipes/inc files until we find them - for f in os.listdir(self.recipe_dir): - full_path_f = os.path.join(self.recipe_dir, f) - if is_recipe_or_include_file(full_path_f, f): - with open(full_path_f + ".tmp", "w+") as temp_recipe: - with open(full_path_f) as recipe: - for line in recipe: - for name in sums: - m1 = re.match("^SRC_URI\["+ name + "md5sum\].*", line) - m2 = re.match("^SRC_URI\["+ name + "sha256sum\].*", line) - if m1: - temp_recipe.write(sums[name]["md5sum"]) - elif m2: - temp_recipe.write(sums[name]["sha256sum"]) - else: - temp_recipe.write(line) + @modify_recipe_files + def _update_recipe_checksums(line, temp_recipe): + for name in sums: + m1 = re.match("^SRC_URI\["+ name + "md5sum\].*", line) + m2 = re.match("^SRC_URI\["+ name + "sha256sum\].*", line) + if m1: + temp_recipe.write(sums[name]["md5sum"]) + elif m2: + temp_recipe.write(sums[name]["sha256sum"]) + else: + temp_recipe.write(line) + + I(" %s: Update recipe checksums ..." % self.env['PN']) + _update_recipe_checksums(self.env, self.recipe_dir) - os.rename(full_path_f + ".tmp", full_path_f) - self.checksums_changed = True def _is_uri_failure(self, fetch_log):
@@ -218,71 +204,63 @@ class Recipe(object): def _change_source_suffix(self, new_suffix): # Will change the extension of the archive from the SRC_URI - for f in os.listdir(self.recipe_dir): - full_path_f = os.path.join(self.recipe_dir, f) - if is_recipe_or_include_file(full_path_f, f): - with open(full_path_f + ".tmp", "w+") as temp_recipe: - with open(full_path_f) as recipe: - source_found = False - for line in recipe: - # source on first line - m1 = re.match("^SRC_URI.*\${PV}\.(.*)[\" \\\\].*", line) - # SRC_URI alone on the first line - m2 = re.match("^SRC_URI.*", line) - # source on second line - m3 = re.match(".*\${PV}\.(.*)[\" \\\\].*", line) - if m1: - old_suffix = m1.group(1) - line = line.replace(old_suffix, new_suffix+" ") - if m2 and not m1: - source_found = True - if m3 and source_found: - old_suffix = m3.group(1) - line = line.replace(old_suffix, new_suffix+" ") - source_found = False - - temp_recipe.write(line) - os.rename(full_path_f + ".tmp", full_path_f) + + source_found = False + @modify_recipe_files + def _change(line, temp_recipe): + # source on first line + m1 = re.match("^SRC_URI.*\${PV}\.(.*)[\" \\\\].*", line) + # SRC_URI alone on the first line + m2 = re.match("^SRC_URI.*", line) + # source on second line + m3 = re.match(".*\${PV}\.(.*)[\" \\\\].*", line) + if m1: + old_suffix = m1.group(1) + line = line.replace(old_suffix, new_suffix+" ") + if m2 and not m1: + source_found = True + if m3 and source_found: + old_suffix = m3.group(1) + line = line.replace(old_suffix, new_suffix+" ") + source_found = False + + temp_recipe.write(line) + + _change(self.env, self.recipe_dir) def _remove_patch_uri(self, uri): - recipe_files = [ - os.path.join(self.recipe_dir, self.env['PN'] + ".inc"), - self.env['FILE']] - - for recipe_filename in recipe_files: - if os.path.isfile(recipe_filename): - with open(recipe_filename + ".tmp", "w+") as temp_recipe: - with open(recipe_filename) as recipe: - for line in recipe: - if line.find(uri) == -1: - temp_recipe.write(line) - continue - - m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line) - m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line) - m3 = re.match("[\t ]*" + uri + " *\\\\", line) - m4 = re.match("([\t ]*)" + uri + " *\"", line) - - # patch on a single SRC_URI line: - if m1: - continue - # patch is on the first SRC_URI line - elif m2: - temp_recipe.write(m2.group(1) + "\\\n") - # patch is in the middle - elif m3: - continue - # patch is last in list - elif m4: - temp_recipe.write(m4.group(1) + "\"\n") - # nothing matched in recipe but we deleted the patch - # anyway? Then we must bail out! - else: - return False - - os.rename(recipe_filename + ".tmp", recipe_filename) + removed = True - return True + @modify_recipe_files + def _remove(line, temp_recipe): + if line.find(uri) == -1: + temp_recipe.write(line) + else: + m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line) + m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line) + m3 = re.match("[\t ]*" + uri + " *\\\\", line) + m4 = re.match("([\t ]*)" + uri + " *\"", line) + + # patch on a single SRC_URI line: + if m1: + return + # patch is on the first SRC_URI line + elif m2: + temp_recipe.write(m2.group(1) + "\\\n") + # patch is in the middle + elif m3: + return + # patch is last in list + elif m4: + temp_recipe.write(m4.group(1) + "\"\n") + # nothing matched in recipe but we deleted the patch + # anyway? Then we must bail out! + else: + removed = False + + _remove(self.env, self.recipe_dir) + + return removed def _remove_faulty_patch(self, patch_log): patch_file = None
@@ -343,6 +321,14 @@ class Recipe(object): return False def _license_issue_handled(self, config_log): + @modify_recipe_files + def _update_license_checksum(line, temp_recipe): + m = re.match("(.*)" + old_md5 + "(.*)", line) + if m is not None: + temp_recipe.write(m.group(1) + new_md5 + m.group(2) + "\n") + else: + temp_recipe.write(line) + license_file = None with open(config_log) as log: for line in log:
@@ -368,8 +354,11 @@ class Recipe(object): new_md5 = m_new.group(1) if license_file is not None: + _update_license_checksum(self.env, self.recipe_dir) + self.create_diff_file(license_file, old_md5, new_md5) self.license_diff_file = os.path.join(self.workdir, os.path.basename(license_file + ".diff")) + if self.interactive: W(" %s: license checksum failed for file %s. The recipe has" "been updated! View diff? (Y/n)" % (self.env['PN'], license_file))
@@ -492,48 +481,46 @@ class Recipe(object): replacement = "${" + prefixes[largest_prefix] + "}" files[i] = files[i].replace(largest_prefix, replacement) - recipe_files = [ - os.path.join(self.recipe_dir, self.env['PN'] + ".inc"), - self.env['FILE']] - # Append the new files - for recipe_filename in recipe_files: - if os.path.isfile(recipe_filename): - with open(recipe_filename + ".tmp", "w+") as temp_recipe: - with open(recipe_filename) as recipe: - files_clause = False - for line in recipe: - if re.match("^FILES_\${PN}[ +=].*", line): - files_clause = True - temp_recipe.write(line) - continue - # Get front spacing - if files_clause: - front_spacing = re.sub("[^ \t]", "", line) - # Append once the last line has of FILES has been reached - if re.match(".*\".*", line) and files_clause: - files_clause = False - line = line.replace("\"", "") - line = line.rstrip() - front_spacing = re.sub("[^ \t]", "", line) - # Do not write an empty line - if line.strip(): - temp_recipe.write(line + " \\\n") - # Add spacing in case there was none - if len(front_spacing) == 0: - front_spacing = " " * 8 - # Write to file - for i in range(len(files)-1): - line = front_spacing + files[i] + " \\\n" - temp_recipe.write(line) - - line = front_spacing + files[len(files) - 1] + "\"\n" - temp_recipe.write(line) - continue - - temp_recipe.write(line) - - os.rename(recipe_filename + ".tmp", recipe_filename) + files_clause = False + @modify_recipe_files + def _append_new_files(line, temp_file): + if re.match("^FILES_\${PN}[ +=].*", line): + files_clause = True + temp_recipe.write(line) + return + + # Get front spacing + if files_clause: + front_spacing = re.sub("[^ \t]", "", line) + + # Append once the last line has of FILES has been reached + if re.match(".*\".*", line) and files_clause: + files_clause = False + line = line.replace("\"", "") + line = line.rstrip() + front_spacing = re.sub("[^ \t]", "", line) + + # Do not write an empty line + if line.strip(): + temp_recipe.write(line + " \\\n") + + # Add spacing in case there was none + if len(front_spacing) == 0: + front_spacing = " " * 8 + + # Write to file + for i in range(len(files) - 1): + line = front_spacing + files[i] + " \\\n" + temp_recipe.write(line) + + line = front_spacing + files[len(files) - 1] + "\"\n" + temp_recipe.write(line) + return + + temp_recipe.write(line) + + _append_new_files(self.env, self.recipe_dir) def unpack(self): self.bb.unpack(self.env['PN'])
Signature · 1 line
2.1.4
[[AUH] 04/17] upgradehelper: Reorder files into directories.
In reply to Aníbal LimónReorder files into modules directory to provide a better source tree.
Changed was made for update the import lines and include modules directory into search path.
--- bitbake.py | 124 -------- buildhistory.py | 75 ----- emailhandler.py | 108 ------- errors.py | 93 ------ git.py | 101 ------- gitrecipe.py | 94 ------ modules/buildhistory.py | 75 +++++ modules/errors.py | 93 ++++++ modules/recipe/__init__.py | 0 modules/recipe/base.py | 671 ++++++++++++++++++++++++++++++++++++++++++ modules/recipe/git.py | 94 ++++++ modules/recipe/svn.py | 28 ++ modules/statistics.py | 102 +++++++ modules/steps.py | 143 +++++++++ modules/utils/__init__.py | 0 modules/utils/bitbake.py | 124 ++++++++ modules/utils/emailhandler.py | 108 +++++++ modules/utils/git.py | 102 +++++++ recipe.py | 669 ----------------------------------------- statistics.py | 102 ------- steps.py | 142 --------- svnrecipe.py | 28 -- upgradehelper.py | 11 +- 23 files changed, 1547 insertions(+), 1540 deletions(-) delete mode 100644 bitbake.py delete mode 100644 buildhistory.py delete mode 100644 emailhandler.py delete mode 100644 errors.py delete mode 100644 git.py delete mode 100644 gitrecipe.py create mode 100644 modules/buildhistory.py create mode 100644 modules/errors.py create mode 100644 modules/recipe/__init__.py create mode 100644 modules/recipe/base.py create mode 100644 modules/recipe/git.py create mode 100644 modules/recipe/svn.py create mode 100644 modules/statistics.py create mode 100644 modules/steps.py create mode 100644 modules/utils/__init__.py create mode 100644 modules/utils/bitbake.py create mode 100644 modules/utils/emailhandler.py create mode 100644 modules/utils/git.py delete mode 100644 recipe.py delete mode 100644 statistics.py delete mode 100644 steps.py delete mode 100644 svnrecipe.py
bitbake.py deleted +0 -124
@@ -1,124 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sw=4 et: -# -# Copyright (c) 2013 - 2014 Intel Corporation -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# AUTHORS -# Laurentiu Palcu <[email protected]> -# Marius Avram <[email protected]> -# - -import os -import logging as log -from logging import info as I -from logging import debug as D -from logging import error as E -from logging import critical as C -import sys -import re - -from errors import * - -for path in os.environ["PATH"].split(':'): - if os.path.exists(path) and "bitbake" in os.listdir(path): - sys.path.insert(0, os.path.join(path, "../lib")) - import bb - -BITBAKE_ERROR_LOG = 'bitbake_error_log.txt' - -class Bitbake(object): - def __init__(self, build_dir): - self.build_dir = build_dir - self.log_dir = None - super(Bitbake, self).__init__() - - def _cmd(self, recipe=None, options=None, env_var=None, output_filter=None): - cmd = "" - if env_var is not None: - cmd += env_var + " " - cmd += "bitbake " - if options is not None: - cmd += options + " " - - if recipe is not None: - cmd += recipe - - if output_filter is not None: - cmd += ' | grep ' + output_filter - - os.chdir(self.build_dir) - - try: - stdout, stderr = bb.process.run(cmd) - except bb.process.ExecutionError as e: - D("%s returned:\n%s" % (cmd, e.__str__())) - - if self.log_dir is not None and os.path.exists(self.log_dir): - with open(os.path.join(self.log_dir, BITBAKE_ERROR_LOG), "w+") as log: - log.write(e.stdout) - - raise Error("\'" + cmd + "\' failed", e.stdout, e.stderr) - - return stdout - - def set_log_dir(self, dir): - self.log_dir = dir - - def get_stdout_log(self): - return os.path.join(self.log_dir, BITBAKE_ERROR_LOG) - - def env(self, recipe=None): - stdout = self._cmd(recipe, "-e", output_filter="-v \"^#\"") - - assignment = re.compile("^([^ \t=]*)=(.*)") - bb_env = dict() - for line in stdout.split('\n'): - m = assignment.match(line) - if m: - if m.group(1) in bb_env: - continue - - bb_env[m.group(1)] = m.group(2).strip("\"") - - if not bb_env: - raise EmptyEnvError(stdout) - - return bb_env - - def fetch(self, recipe): - return self._cmd(recipe, "-c fetch") - - def unpack(self, recipe): - return self._cmd(recipe, "-c unpack") - - def checkpkg(self, recipe): - if recipe == "universe": - return self._cmd(recipe, "-c checkpkg -k") - else: - return self._cmd(recipe, "-c checkpkg") - - def cleanall(self, recipe): - return self._cmd(recipe, "-c cleanall") - - def cleansstate(self, recipe): - return self._cmd(recipe, "-c cleansstate") - - def complete(self, recipe, machine): - return self._cmd(recipe, env_var="MACHINE=" + machine) - - def dependency_graph(self, package_list): - return self._cmd(package_list, "-g")
buildhistory.py deleted +0 -75
75 lines not on this page · raw message
emailhandler.py deleted +0 -108
108 lines not on this page · raw message
errors.py deleted +0 -93
93 lines not on this page · raw message
git.py deleted +0 -101
101 lines not on this page · raw message
gitrecipe.py deleted +0 -94
94 lines not on this page · raw message
modules/buildhistory.py new file +75 -0
75 lines not on this page · raw message
modules/errors.py new file +93 -0
93 lines not on this page · raw message
modules/recipe/__init__.py new file +0 -0
modules/recipe/base.py new file +671 -0
671 lines not on this page · raw message
modules/recipe/git.py new file +94 -0
94 lines not on this page · raw message
modules/recipe/svn.py new file +28 -0
@@ -0,0 +1,28 @@ +#!/usr/bin/env python +# vim: set ts=4 sw=4 et: +# +# Copyright (c) 2013 - 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# AUTHORS +# Laurentiu Palcu <[email protected]> +# Marius Avram <[email protected]> +# + +from base import Recipe + +class SvnRecipe(Recipe): + pass
modules/statistics.py new file +102 -0
102 lines not on this page · raw message
modules/steps.py new file +143 -0
143 lines not on this page · raw message
modules/utils/__init__.py new file +0 -0
modules/utils/bitbake.py new file +124 -0
124 lines not on this page · raw message
modules/utils/emailhandler.py new file +108 -0
108 lines not on this page · raw message
modules/utils/git.py new file +102 -0
102 lines not on this page · raw message
recipe.py deleted +0 -669
669 lines not on this page · raw message
statistics.py deleted +0 -102
102 lines not on this page · raw message
steps.py deleted +0 -142
94 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.1.4
Part of the diff is not shown here. View the raw message.
[[AUH] 09/17] steps.py: Merge load_dirs step into load_env.
In reply to Aníbal Limón--- modules/steps.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
modules/steps.py +3 -6
@@ -50,15 +50,13 @@ def load_env(bb, git, opts, pkg_ctx): git.clean_untracked() pkg_ctx['env'] = bb.env(pkg_ctx['PN']) - if pkg_ctx['env']['PV'] == pkg_ctx['NPV']: - raise UpgradeNotNeededError - -def load_dirs(bb, git, opts, pkg_ctx): pkg_ctx['workdir'] = os.path.join(pkg_ctx['base_dir'], pkg_ctx['PN']) os.mkdir(pkg_ctx['workdir']) - pkg_ctx['recipe_dir'] = os.path.dirname(pkg_ctx['env']['FILE']) + if pkg_ctx['env']['PV'] == pkg_ctx['NPV']: + raise UpgradeNotNeededError + def clean_repo(bb, git, opts, pkg_ctx): try: git.checkout_branch("upgrades")
@@ -130,7 +128,6 @@ def buildhistory_diff(bb, git, opts, pkg_ctx): upgrade_steps = [ (load_env, "Loading environment ..."), - (load_dirs, None), (clean_repo, "Cleaning git repository of temporary branch ..."), (detect_recipe_type, None), (buildhistory_init, None),
Signature · 1 line
2.1.4
[[AUH] 10/17] steps.py: Move clean_repo to first step.
In reply to Aníbal LimónIn order to avoid random errors of uncleaned repository.
--- modules/steps.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-)
modules/steps.py +16 -12
@@ -34,6 +34,21 @@ from recipe.base import Recipe from recipe.git import GitRecipe from recipe.svn import SvnRecipe +def clean_repo(bb, git, opts, pkg_ctx): + git.checkout_branch("master") + + try: + git.delete_branch("remove_patches") + except: + pass + try: + git.delete_branch("upgrades") + except: + pass + + git.reset_hard() + git.create_branch("upgrades") + def load_env(bb, git, opts, pkg_ctx): stdout = git.status() if stdout != "":
@@ -57,17 +72,6 @@ def load_env(bb, git, opts, pkg_ctx): if pkg_ctx['env']['PV'] == pkg_ctx['NPV']: raise UpgradeNotNeededError -def clean_repo(bb, git, opts, pkg_ctx): - try: - git.checkout_branch("upgrades") - except Error: - git.create_branch("upgrades") - - try: - git.delete_branch("remove_patches") - except: - pass - def detect_recipe_type(bb, git, opts, pkg_ctx): if pkg_ctx['env']['SRC_URI'].find("ftp://") != -1 or \ pkg_ctx['env']['SRC_URI'].find("http://") != -1 or \
@@ -127,8 +131,8 @@ def buildhistory_diff(bb, git, opts, pkg_ctx): pkg_ctx['buildhistory'].diff() upgrade_steps = [ - (load_env, "Loading environment ..."), (clean_repo, "Cleaning git repository of temporary branch ..."), + (load_env, "Loading environment ..."), (detect_recipe_type, None), (buildhistory_init, None), (unpack_original, "Fetch & unpack original version ..."),
Signature · 1 line
2.1.4
[[AUH] 08/17] recipe/base.py: Add support for get recipe inherits.
In reply to Aníbal LimónRecipe inherits are needed to detect what recipes support certain feature.
For example: For detect ptest enable recipes and then do ptest runtime test after succesful upgrade.
--- modules/recipe/base.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
modules/recipe/base.py +25 -0
@@ -57,6 +57,16 @@ def modify_recipe_files(func): os.rename(full_path_f + ".tmp", full_path_f) return modify +def read_recipe_files(func): + def read(env, recipe_dir): + for f in os.listdir(recipe_dir): + full_path_f = os.path.join(recipe_dir, f) + if is_recipe_or_include_file(env, full_path_f, f): + with open(full_path_f) as recipe: + for line in recipe: + func(line) + return read + class Recipe(object): def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git): self.env = env
@@ -85,8 +95,23 @@ class Recipe(object): self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver + "\n\n" self.rm_patches_msg = "\n\nRemoved the following patch(es):\n" + self._inherits = None + super(Recipe, self).__init__() + def get_inherits(self): + @read_recipe_files + def _get_inherits(line): + m = re.search("^inherit (.*)$", line) + if m: + tmp = m.group(1).split() + self._inherits.extend(tmp) + + if not self._inherits: + self._inherits = [] + _get_inherits(self.env, self.recipe_dir) + + return self._inherits def update_env(self, env): self.env = env
Signature · 1 line
2.1.4
Re: [[AUH] 08/17] recipe/base.py: Add support for get recipe inherits.
Quoted 46 lines · On Wednesday 25 November 2015 18:00:37 Aníbal Limón wrote:
Recipe inherits are needed to detect what recipes support certain feature.
For example: For detect ptest enable recipes and then do ptest runtime test after succesful upgrade.
Signed-off-by Aníbal Limón <[email protected]>--- modules/recipe/base.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)modules/recipe/base.py +10
-0@@ -57,6 +57,16 @@ def modify_recipe_files(func): os.rename(full_path_f + ".tmp", full_path_f) return modify +def read_recipe_files(func): + def read(env, recipe_dir): + for f in os.listdir(recipe_dir): + full_path_f = os.path.join(recipe_dir, f) + if is_recipe_or_include_file(env, full_path_f, f): + with open(full_path_f) as recipe: + for line in recipe: + func(line) + return read + class Recipe(object): def __init__(self, env, new_ver, interactive, workdir, recipe_dir,bitbake, git): self.env = env
modules/recipe/base.py +0
-0@@ -85,8 +95,23 @@ class Recipe(object): self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver +"\n\n" self.rm_patches_msg = "\n\nRemoved the following patch(es):\n"
modules/recipe/base.py +8
-0+ self._inherits = None + super(Recipe, self).__init__() + def get_inherits(self): + @read_recipe_files + def _get_inherits(line): + m = re.search("^inherit (.*)$", line) + if m: + tmp = m.group(1).split()
modules/recipe/base.py class Recipe(object): · line 117
Quoted 10 lines
modules/recipe/base.py +7
-0+ self._inherits.extend(tmp) + + if not self._inherits: + self._inherits = [] + _get_inherits(self.env, self.recipe_dir) + + return self._inherits def update_env(self, env): self.env = env
I'm not sure I like this - you're not parsing the recipe so you'll miss things such as inc files inheriting the recipe. Would it be possible to parse the recipe properly instead e.g. with tinfoil?
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Re: [[AUH] 08/17] recipe/base.py: Add support for get recipe inherits.
Quoted 55 lines · Paul Eggleton
Quoted 54 lines · On Wednesday 25 November 2015 18:00:37 Aníbal Limón wrote:
Recipe inherits are needed to detect what recipes support certain feature.
For example: For detect ptest enable recipes and then do ptest runtime test after succesful upgrade.
Signed-off-by Aníbal Limón <[email protected]>--- modules/recipe/base.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)modules/recipe/base.py +10
-0@@ -57,6 +57,16 @@ def modify_recipe_files(func): os.rename(full_path_f + ".tmp", full_path_f) return modify +def read_recipe_files(func): + def read(env, recipe_dir): + for f in os.listdir(recipe_dir): + full_path_f = os.path.join(recipe_dir, f) + if is_recipe_or_include_file(env, full_path_f, f): + with open(full_path_f) as recipe: + for line in recipe: + func(line) + return read + class Recipe(object): def __init__(self, env, new_ver, interactive, workdir, recipe_dir,bitbake, git): self.env = env
modules/recipe/base.py +0
-0@@ -85,8 +95,23 @@ class Recipe(object): self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver +"\n\n" self.rm_patches_msg = "\n\nRemoved the following patch(es):\n"
modules/recipe/base.py +15
-0+ self._inherits = None + super(Recipe, self).__init__() + def get_inherits(self): + @read_recipe_files + def _get_inherits(line): + m = re.search("^inherit (.*)$", line) + if m: + tmp = m.group(1).split() + self._inherits.extend(tmp) + + if not self._inherits: + self._inherits = [] + _get_inherits(self.env, self.recipe_dir) + + return self._inherits
Quoted 6 lines
Quoted 2 lines
def
update_env(self, env):
self.env = envI'm not sure I like this - you're not parsing the recipe so you'll miss things such as inc files inheriting the recipe. Would it be possible to parse the recipe properly instead e.g. with tinfoil?
I agree with you but for now it works for detect recipes that have ptest enabled.
This change isn't trivial so i loaded a ticket for change AUH to use tinfoil (may be tinfoil2 :)).
Cheers,
alimon
[1] https://bugzilla.yoctoproject.org/show_bug.cgi?id=8735
Quoted 4 lines
Cheers,
Paul
[[AUH] 11/17] utils/git.py: Add method for apply patches into a branch.
In reply to Aníbal Limón--- modules/utils/git.py | 3 +++ 1 file changed, 3 insertions(+)
modules/utils/git.py +3 -0
@@ -61,6 +61,9 @@ class Git(object): def create_patch(self, out_dir): return self._cmd("format-patch -M10 -1 -o " + out_dir) + def apply_patch(self, patch_file): + return self._cmd("am %s" % patch_file) + def status(self): return self._cmd("status --porcelain")
Signature · 1 line
2.1.4
Re: [[AUH] 11/17] utils/git.py: Add method for apply patches into a branch.
Quoted 10 lines · On Wednesday 25 November 2015 18:00:40 Aníbal Limón wrote:
Signed-off-by Aníbal Limón <[email protected]>--- modules/utils/git.py | 3 +++ 1 file changed, 3 insertions(+)modules/utils/git.py +0
-0
modules/utils/git.py class Git(object): · line 68
Quoted 8 lines
modules/utils/git.py +3
-0@@ -61,6 +61,9 @@ class Git(object): def create_patch(self, out_dir): return self._cmd("format-patch -M10 -1 -o " + out_dir) + def apply_patch(self, patch_file): + return self._cmd("am %s" % patch_file) + def status(self): return self._cmd("status --porcelain")
Just a warning (which may not apply in your situation, not sure) - you may need to do extra cleanup beyond git am --abort if this fails; see:
http://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=e0b9a96002bb23e7c2c8a8c9c4d2431461fd6cb7
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Re: [[AUH] 11/17] utils/git.py: Add method for apply patches into a branch.
Good observation, i'll make this change.
alimon
Quoted 28 lines · Paul Eggleton
Quoted 10 lines · On Wednesday 25 November 2015 18:00:40 Aníbal Limón wrote:
Signed-off-by Aníbal Limón <[email protected]>--- modules/utils/git.py | 3 +++ 1 file changed, 3 insertions(+)modules/utils/git.py +0
-0modules/utils/git.py class Git(object): · line 68
Quoted 8 lines
modules/utils/git.py +3
-0@@ -61,6 +61,9 @@ class Git(object): def create_patch(self, out_dir): return self._cmd("format-patch -M10 -1 -o " + out_dir) + def apply_patch(self, patch_file): + return self._cmd("am %s" % patch_file) + def status(self): return self._cmd("status --porcelain")Just a warning (which may not apply in your situation, not sure) - you may need to do extra cleanup beyond git am --abort if this fails; see:
http://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=e0b9a96002bb23e7c2c8a8c9c4d2431461fd6cb7
Cheers,
Paul
[[AUH] 12/17] upgradehelper.py: Add settings for enable testimage.
In reply to Aníbal LimónTestimage feature will provide the ability to run testimage class over a set of succesful upgrades.
Also it will run ptest on recipes that support ptest and retrive the result for store/send to maintainer.
--- README | 25 +++++++++++++++++++++++++ upgradehelper.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 2 deletions(-)
README +25 -0
@@ -61,6 +61,7 @@ machines=qemux86 qemux86-64 qemuarm qemumips qemuppc # optional features buildhistory=no +testimage=no --------------- snip ---------------
@@ -93,6 +94,30 @@ $BUILDDIR/upgrade-helper/work/recipe/buildhistory-diff.txt. (Do not remove any other inherited class in the process, e.g. distrodata). +5. If you want to enable testimage (optional) you need to enable in +upgrade-helper.conf also add the following lines to your conf/local.conf +file: + +--------------- snip --------------- +INHERIT += "testimage" + +EXTRA_IMAGE_FEATURES = "debug-tweaks package-management" +# testimage/ptest only work with rpm +PACKAGE_CLASSES = "package_rpm" +--------------- snip --------------- + +Also if you are running in a server without X11 session, you need to start +a vncserver example: + +$ vncserver :1 +$ export DISPLAY=:1 + +If upgrade is succesful testimage/ptest results are generated into +$BUILDDIR/upgrade-helper/work/recipe/ptest_recipe.log if recipe support +ptest. + +(Do not remove any other inherited class in the process, e.g. distrodata). + Usage -----
upgradehelper.py +36 -2
@@ -157,6 +157,7 @@ class Updater(object): 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['skip_compilation'] = skip_compilation self.opts['buildhistory'] = self._buildhistory_is_enabled() + self.opts['testimage'] = self._testimage_is_enabled() self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir):
@@ -211,6 +212,41 @@ class Updater(object): return enabled + def _testimage_is_enabled(self): + enabled = False + + if settings.get("testimage", "no") == "yes": + if 'testimage' in self.base_env['INHERIT']: + if not "ptest" in self.base_env["DISTRO_FEATURES"]: + E(" testimage requires ptest in DISTRO_FEATURES please add to"\ + " conf/local.conf.") + exit(1) + + if not "package-management" in self.base_env['EXTRA_IMAGE_FEATURES']: + E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\ + " please add to conf/local.conf.") + exit(1) + + if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]: + E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\ + " please add to conf/local.conf.") + exit(1) + + enabled = True + else: + E(" testimage was enabled in upgrade-helper.conf"\ + " but isn't INHERIT in conf/local.conf, if you want"\ + " to enable please set.") + exit(1) + else: + if 'testimage' in self.base_env['INHERIT']: + E(" testimage was INHERIT in conf/local.conf"\ + " but testimage=yes isn't in upgrade-helper.conf,"\ + " if you want to enable please set.") + exit(1) + + return enabled + def _get_packages_to_upgrade(self, packages=None): if packages is None: I( "Nothing to upgrade")
@@ -497,10 +533,8 @@ class Updater(object): self.uh_recipes_failed_dir, pkg_ctx['PN'])) self.commit_changes(pkg_ctx) - self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'], pkg_ctx['MAINTAINER'], pkg_ctx['error']) - self.pkg_upgrade_handler(pkg_ctx) if attempted_pkgs > 1:
Signature · 1 line
2.1.4
[[AUH] 13/17] upgradehelper: Add testimage feature.
In reply to Aníbal LimónTestimage module implements usage of testimage class, a integration branch is created with succesful recipe upgrades and then the next tests are run for every machine configured,
- ptest: Recipes that support ptest are run and retrive the result after upgrade.
- sato: Runs core-image-sato -c testimage.
The results are stored into recipe work directory and send to the maintainer.
[YOCTO #7471]
[YOCTO #7567]
--- modules/testimage.py | 185 +++++++++++++++++++++++++++++++++++++++++++++++++++ upgradehelper.py | 82 +++++++++++++++++++++-- 2 files changed, 261 insertions(+), 6 deletions(-) create mode 100644 modules/testimage.py
modules/testimage.py new file +185 -0
@@ -0,0 +1,185 @@ +#!/usr/bin/env python +# vim: set ts=4 sw=4 et: +# +# Copyright (c) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# This module implements logic for run image tests on recipes when upgrade +# process succeed. +# + +import os +import sys +import shutil + +import logging as log +from logging import debug as D +from logging import info as I +from logging import warning as W +from logging import error as E +from logging import critical as C + +from errors import * +from utils.bitbake import * + +class TestImage(): + def __init__(self, bb, git, uh_work_dir, pkgs_ctx): + self.bb = bb + self.git = git + self.uh_work_dir = uh_work_dir + self.pkgs_ctx = pkgs_ctx + + os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \ + " TEST_SUITES CORE_IMAGE_EXTRA_INSTALL" + + def _get_ptest_pkgs(self): + pkgs = [] + + for c in self.pkgs_ctx: + if "ptest" in c['recipe'].get_inherits(): + pkgs.append(c) + + return pkgs + + def _get_pkgs_to_install(self, pkgs, ptest=False): + pkgs_out = [] + + # for provide access to the target + if ptest: + pkgs_out.append("dropbear") + + for c in pkgs: + pkgs_out.append(c['PN']) + if ptest: + pkgs_out.append("%s-ptest" % c['PN']) + + return ' '.join(pkgs_out) + + def prepare_branch(self): + self.git.checkout_branch("master") + try: + self.git.delete_branch("testimage") + self.git.delete_branch("upgrades") + except Error: + pass + self.git.reset_hard() + + self.git.create_branch("testimage") + for c in self.pkgs_ctx: + patch_file = os.path.join(c['workdir'], c['patch_file']) + self.git.apply_patch(patch_file) + + def _parse_ptest_log(self, log_file): + ptest_results = {} + + with open(log_file, "r") as f: + pn = None + processing = False + + for line in f: + if not processing: + m = re.search("^BEGIN: /usr/lib/(.*)/ptest$", line) + if m: + pn = m.group(1) + ptest_results[pn] = [] + processing = True + else: + m = re.search("^END: $", line) + if m: + pn = None + processing = False + else: + ptest_results[pn].append(line) + + return ptest_results + + def _find_log(self, name, machine): + result = [] + + base_dir = os.path.join(os.getenv('BUILDDIR'), 'tmp', 'work') + for root, dirs, files in os.walk(base_dir): + if name in files: + result.append(os.path.join(root, name)) + + for ptest_log in result: + if machine in ptest_log: + return ptest_log + + def ptest(self, machine): + ptest_pkgs = self._get_ptest_pkgs() + + os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \ + self._get_pkgs_to_install(ptest_pkgs, True) + I( " building core-image-minimal for %s ..." % machine) + self.bb.complete("core-image-minimal", machine) + + os.environ['TEST_SUITES'] = "ping ssh _ptest" + I( " running core-image-minimal/ptest for %s ..." % machine) + self.bb.complete("core-image-minimal -c testimage", machine) + + ptest_log_file = self._find_log("ptest.log", machine) + shutil.copyfile(ptest_log_file, + os.path.join(self.uh_work_dir, "ptest_%s.log" % machine)) + + ptest_result = self._parse_ptest_log(ptest_log_file) + for pn in ptest_result: + for pkg_ctx in self.pkgs_ctx: + if not pn == pkg_ctx['PN']: + continue + + if not 'ptest' in pkg_ctx: + pkg_ctx['ptest'] = {} + if not 'ptest_log' in pkg_ctx: + pkg_ctx['ptest_log'] = os.path.join(pkg_ctx['workdir'], + "ptest.log") + + pkg_ctx['ptest'][machine] = True + with open(pkg_ctx['ptest_log'], "a+") as f: + f.write("BEGIN: PTEST for %s\n" % machine) + for line in ptest_result[pn]: + f.write(line) + f.write("END: PTEST for %s\n" % machine) + + def sato(self, machine): + os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \ + self._get_pkgs_to_install(self.pkgs_ctx) + + if 'TEST_SUITES' in os.environ: + del os.environ['TEST_SUITES'] + + I( " building core-image-sato for %s ..." % machine) + self.bb.complete("core-image-sato", machine) + + I( " running core-image-sato/testimage for %s ..." % machine) + self.bb.complete("core-image-sato -c testimage", machine) + + log_file = self._find_log("log.do_testimage", machine) + shutil.copyfile(log_file, + os.path.join(self.uh_work_dir, "log_%s.do_testimage" % machine)) + for pkg_ctx in self.pkgs_ctx: + if not 'testimage' in pkg_ctx: + pkg_ctx['testimage'] = {} + if not 'testimage_log' in pkg_ctx: + pkg_ctx['testimage_log'] = os.path.join( + pkg_ctx['workdir'], "log.do_testimage") + + pkg_ctx['testimage'][machine] = True + with open(log_file, "r") as lf: + with open(pkg_ctx['testimage_log'], "a+") as of: + of.write("BEGIN: TESTIMAGE for %s\n" % machine) + for line in lf: + of.write(line) + of.write("END: TESTIMAGE for %s\n" % machine)
upgradehelper.py +76 -6
146 lines not on this page · raw message
Signature · 1 line
2.1.4
Re: [[AUH] 13/17] upgradehelper: Add testimage feature.
Quoted 7 lines · On Wednesday 25 November 2015 18:00:42 Aníbal Limón wrote:
Testimage module implements usage of testimage class, a integration branch is created with succesful recipe upgrades and then the next tests are run for every machine configured,
- ptest: Recipes that support ptest are run and retrive the result after upgrade.
- sato: Runs core-image-sato -c testimage.
Can we avoid hardcoding sato anywhere here? It would be ideal if the image could be configured or passed in instead.
Cheers,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre
Re: [[AUH] 13/17] upgradehelper: Add testimage feature.
Sure,
alimon
Quoted 15 lines · Paul Eggleton
Quoted 7 lines · On Wednesday 25 November 2015 18:00:42 Aníbal Limón wrote:
Testimage module implements usage of testimage class, a integration branch is created with succesful recipe upgrades and then the next tests are run for every machine configured,
- ptest: Recipes that support ptest are run and retrive the result after upgrade.
- sato: Runs core-image-sato -c testimage.Can we avoid hardcoding sato anywhere here? It would be ideal if the image could be configured or passed in instead.
Cheers,
Paul
[[AUH] 14/17] upgradehelper.py: Changed retry failure build to 30 days.
In reply to Aníbal LimónWhen build fails is stored into history fail that enables to don't try to build again in a period of time, so change that period to 30 days since the work patches will be publish in web server.
--- upgradehelper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
upgradehelper.py +3 -3
@@ -738,12 +738,12 @@ class UniverseUpdater(Updater): date.toordinal(date.today()) - \ date.toordinal(datetime.strptime(self.history[pn][2], '%Y-%m-%d')) # retry recipes that had fetch errors or other errors after - # more than 7 days + # more than 30 days if (self.history[pn][3] == str(FetchError()) or - self.history[pn][3] == str(Error())) and retry_delta > 7: + self.history[pn][3] == str(Error())) and retry_delta > 30: return True - D(" Skipping upgrade of %s: is in history and not 7 days passed" % pn) + D(" Skipping upgrade of %s: is in history and not 30 days passed" % pn) return False # drop native/cross/cross-canadian recipes. We deal with native
Signature · 1 line
2.1.4
[[AUH] 16/17] statistics: Improve email format and get_summary method.
In reply to Aníbal Limón--- modules/statistics.py | 24 ++++++++++++++---------- upgradehelper.py | 11 +++++------ 2 files changed, 19 insertions(+), 16 deletions(-)
modules/statistics.py +14 -10
@@ -62,16 +62,15 @@ class Statistics(object): self.total_attempted += 1 - def pkg_stats(self): - stat_msg = "\nUpgrade statistics:\n" - stat_msg += "====================================================\n" + def _pkg_stats(self): + stat_msg = "Recipe upgrade statistics:\n\n" for status in self.upgrade_stats: list_len = len(self.upgrade_stats[status]) if list_len > 0: - stat_msg += "* " + status + ": " + str(list_len) + "\n" + stat_msg += " * " + status + ": " + str(list_len) + "\n" for pkg, new_ver, maintainer in self.upgrade_stats[status]: - stat_msg += " " + pkg + ", " + new_ver + ", " + \ + stat_msg += " " + pkg + ", " + new_ver + ", " + \ maintainer + "\n" if self.total_attempted == 0:
@@ -80,8 +79,7 @@ class Statistics(object): else: percent_succeded = self.succeeded["total"] * 100.0 / self.total_attempted percent_failed = self.failed["total"] * 100.0 / self.total_attempted - stat_msg += "++++++++++++++++++++++++++++++++++++++++++++++++++++\n" - stat_msg += "TOTAL: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n\n" % \ + stat_msg += "\n TOTAL: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n\n" % \ (self.total_attempted, self.succeeded["total"], percent_succeded, self.failed["total"],
@@ -89,14 +87,20 @@ class Statistics(object): return stat_msg - def maintainer_stats(self): - stat_msg = "* Statistics per maintainer:\n" + def _maintainer_stats(self): + stat_msg = "Recipe upgrade statistics per Maintainer:\n\n" for m in self.maintainers: attempted = self.succeeded[m] + self.failed[m] - stat_msg += " %s: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n\n" % \ + stat_msg += " %s: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n" % \ (m.split("@")[0], attempted, self.succeeded[m], self.succeeded[m] * 100.0 / attempted, self.failed[m], self.failed[m] * 100.0 / attempted) return stat_msg + + def get_summary(self): + msg = self._pkg_stats() + msg += self._maintainer_stats() + + return msg
upgradehelper.py +5 -6
@@ -395,14 +395,14 @@ class Updater(object): def send_status_mail(self): if "status_recipients" not in settings: - E("Could not send status email, no recipients set!") + E(" Could not send status email, no recipients set!") return -1 to_list = settings["status_recipients"].split() subject = "[AUH] Upgrade status: " + date.isoformat(date.today()) - msg = self.statistics.pkg_stats() + self.statistics.maintainer_stats() + msg = self.statistics.get_summary() if self.statistics.total_attempted: self.email_handler.send_email(to_list, subject, msg)
@@ -610,16 +610,15 @@ class Updater(object): pkg_ctx = pkgs_ctx[pn] self.pkg_upgrade_handler(pkg_ctx) - if attempted_pkgs > 1: - statistics_summary = self.statistics.pkg_stats() + \ - self.statistics.maintainer_stats() + if attempted_pkgs > 0: + statistics_summary = self.statistics.get_summary() statistics_file = os.path.join(self.uh_work_dir, "statistics_summary") with open(statistics_file, "w+") as f: f.write(statistics_summary) - I("%s" % statistics_summary) + I(" %s" % statistics_summary) if self.opts['send_email']: self.send_status_mail()
Signature · 1 line
2.1.4
[[AUH] 17/17] statistics: Add support for publish_work_url setting.
In reply to Aníbal LimónPublish setting helps to point the url where the AUH patches and logs are saved, this url is included into AUH upgrade status email.
--- README | 3 +++ modules/statistics.py | 10 ++++++++-- upgradehelper.py | 12 ++++++------ 3 files changed, 17 insertions(+), 8 deletions(-)
README +3 -0
@@ -50,6 +50,9 @@ [email protected] # specify the directory where work (patches) will be saved (optional) #workdir= +# publish url to use in statistics summary (optional) +#publish_work_url=http://auh.somehost.com/work + # clean sstate directory before upgrading clean_sstate=yes
modules/statistics.py +8 -2
@@ -99,8 +99,14 @@ class Statistics(object): return stat_msg - def get_summary(self): - msg = self._pkg_stats() + def get_summary(self, publish_work_url, workdir): + msg = '' + + if publish_work_url: + msg += "AUH finished upgrade batch the result patches/logs can be found at:\n" \ + "%s/%s, next are the statistics:\n\n" % (publish_work_url, workdir) + + msg += self._pkg_stats() msg += self._maintainer_stats() return msg
upgradehelper.py +6 -6
@@ -393,7 +393,7 @@ class Updater(object): I(" %s: %s" % (pkg_ctx['PN'], e.stdout)) raise e - def send_status_mail(self): + def send_status_mail(self, statistics_summary): if "status_recipients" not in settings: E(" Could not send status email, no recipients set!") return -1
@@ -402,10 +402,8 @@ class Updater(object): subject = "[AUH] Upgrade status: " + date.isoformat(date.today()) - msg = self.statistics.get_summary() - if self.statistics.total_attempted: - self.email_handler.send_email(to_list, subject, msg) + self.email_handler.send_email(to_list, subject, statistics_summary) else: W("No recipes attempted, not sending status mail!")
@@ -611,7 +609,9 @@ class Updater(object): self.pkg_upgrade_handler(pkg_ctx) if attempted_pkgs > 0: - statistics_summary = self.statistics.get_summary() + statistics_summary = self.statistics.get_summary( + settings.get('publish_work_url', 'no'), + os.path.basename(self.uh_work_dir)) statistics_file = os.path.join(self.uh_work_dir, "statistics_summary")
@@ -621,7 +621,7 @@ class Updater(object): I(" %s" % statistics_summary) if self.opts['send_email']: - self.send_status_mail() + self.send_status_mail(statistics_summary) class UniverseUpdater(Updater): def __init__(self):
Signature · 1 line
2.1.4
[[AUH] 15/17] upgradehelper: Add workdir setting.
In reply to Aníbal LimónThe workdir setting is useful for store the AUH generated patches into another directory than $BULDDIR/upgrade-helper/work this will be used to publish AUH work.
--- README | 3 +++ upgradehelper.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-)
README +3 -0
@@ -47,6 +47,9 @@ [email protected] # who should get the status mail with statistics, at the end [email protected] +# specify the directory where work (patches) will be saved (optional) +#workdir= + # clean sstate directory before upgrading clean_sstate=yes
upgradehelper.py +4 -1
@@ -163,7 +163,10 @@ class Updater(object): self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir): os.mkdir(self.uh_dir) - self.uh_work_dir = os.path.join(self.uh_dir, "work-%s" % \ + uh_base_work_dir = settings.get('workdir', '') + if not uh_base_work_dir: + uh_base_work_dir = self.uh_dir + self.uh_work_dir = os.path.join(uh_base_work_dir, "%s" % \ datetime.now().strftime("%Y%m%d%H%M%S")) os.mkdir(self.uh_work_dir) self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all")
Signature · 1 line
2.1.4