Skip to contentSkip to conversation

00/17] AUH code refactor and support test image.

AUH Aníbal Limón 2 participants 24 messages Nov 26, 2015 – Nov 27, 2015

lore
Share & tools
lore
#0 lore raw

The following patchset contains,

- A refactor of AUH code clean-up and improve source tree.
- A implementation of testimage that do ptest and sato testimages.
- New settings for publish patches and logs generated by AUH.
- Improvements on the status email.

The changes can also be found at,

http://git.yoctoproject.org/cgit/cgit.cgi/auto-upgrade-helper/log/?h=boot_images_v2

Aníbal Limón (17):
{upgradehelper, bitbake}.py: Move _get_env function to bitbake upgradehelper.py: Merge options into a dictionary upgradehelper.py: Adds own module for steps
upgradehelper: Reorder files into directories.
buildhistory: Add option for enable in upgrade-helper.conf recipe/base.py: Add is_recipe_or_include_file func recipe/base.py: Add modify_recipe_files function decorator recipe/base.py: Add support for get recipe inherits. steps.py: Merge load_dirs step into load_env. steps.py: Move clean_repo to first step. utils/git.py: Add method for apply patches into a branch. upgradehelper.py: Add settings for enable testimage.
upgradehelper: Add testimage feature. upgradehelper.py: Changed retry failure build to 30 days.
upgradehelper: Add workdir setting.
statistics: Improve email format and get_summary method.
statistics: Add support for publish_work_url setting.

 README                        |  40 ++-
 bitbake.py                    | 107 -------
 buildhistory.py               |  75 -----
 emailhandler.py               | 108 -------
 errors.py                     |  93 ------
 git.py                        | 101 -------
 gitrecipe.py                  |  94 ------
 modules/buildhistory.py       |  75 +++++
 modules/errors.py             |  93 ++++++
 modules/recipe/__init__.py    |   0
 modules/recipe/base.py        | 682 ++++++++++++++++++++++++++++++++++++++++++
 modules/recipe/git.py         |  94 ++++++
 modules/recipe/svn.py         |  28 ++
 modules/statistics.py         | 112 +++++++
 modules/steps.py              | 144 +++++++++
 modules/testimage.py          | 185 ++++++++++++
 modules/utils/__init__.py     |   0
 modules/utils/bitbake.py      | 124 ++++++++
 modules/utils/emailhandler.py | 108 +++++++
 modules/utils/git.py          | 105 +++++++
 recipe.py                     | 669 -----------------------------------------
 statistics.py                 | 102 -------
 svnrecipe.py                  |  28 --
 upgradehelper.py              | 505 +++++++++++++++----------------
 24 files changed, 2042 insertions(+), 1630 deletions(-)
 delete mode 100644 bitbake.py
 delete mode 100644 buildhistory.py
 delete mode 100644 emailhandler.py
 delete mode 100644 errors.py
 delete mode 100644 git.py
 delete mode 100644 gitrecipe.py
 create mode 100644 modules/buildhistory.py
 create mode 100644 modules/errors.py
 create mode 100644 modules/recipe/__init__.py
 create mode 100644 modules/recipe/base.py
 create mode 100644 modules/recipe/git.py
 create mode 100644 modules/recipe/svn.py
 create mode 100644 modules/statistics.py
 create mode 100644 modules/steps.py
 create mode 100644 modules/testimage.py
 create mode 100644 modules/utils/__init__.py
 create mode 100644 modules/utils/bitbake.py
 create mode 100644 modules/utils/emailhandler.py
 create mode 100644 modules/utils/git.py
 delete mode 100644 recipe.py
 delete mode 100644 statistics.py
 delete mode 100644 svnrecipe.py
Signature · 1 line
2.1.4
#1 lore raw

[[AUH] 02/17] upgradehelper.py: Merge options into a dictionary

Use one dictionary to keep together the options this enables possibility to define an interface for steps, see next commit.

---
 upgradehelper.py | 83 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 42 insertions(+), 41 deletions(-)
upgradehelper.py +42 -41
index 128bc07..1c3dcbc 100755
@@ -126,15 +126,38 @@ def parse_config_file(config_file):
 
 class Updater(object):
     def __init__(self, auto_mode=False, send_email=False, skip_compilation=False):
+        build_dir = get_build_dir()
 
-        self.uh_dir = os.path.join(get_build_dir(), "upgrade-helper")
+        self.bb = Bitbake(build_dir)
+
+        try:
+            self.base_env = self.bb.env()
+        except EmptyEnvError as e:
+            import traceback
+            E( " %s\n%s" % (e.message, traceback.format_exc()))
+            E( " Bitbake output:\n%s" % (e.stdout))
+            exit(1)
+
+        self.email_handler = Email(settings)
+        self.statistics = Statistics()
+        self.git = None
+
+        self.opts = {}
+        self.opts['interactive'] = not auto_mode
+        self.opts['send_email'] = send_email
+        self.opts['author'] = "Upgrade Helper <%s>" % \
+                settings.get('from', '[email protected]')
+        self.opts['machines'] = settings.get('machines',
+                'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
+        self.opts['skip_compilation'] = skip_compilation
+        self.opts['buildhistory_enabled'] = self._buildhistory_is_enabled()
+
+        self.uh_dir = os.path.join(build_dir, "upgrade-helper")
         if not os.path.exists(self.uh_dir):
             os.mkdir(self.uh_dir)
-
         self.uh_work_dir = os.path.join(self.uh_dir, "work-%s" % \
                 datetime.now().strftime("%Y%m%d%H%M%S"))
         os.mkdir(self.uh_work_dir)
-
         self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all")
         os.mkdir(self.uh_recipes_all_dir)
         self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed")
@@ -142,16 +165,6 @@ class Updater(object):
         self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed")
         os.mkdir(self.uh_recipes_failed_dir)
 
-        self.bb = Bitbake(get_build_dir())
-        self.git = None
-        self.author_email = settings.get('from', '[email protected]')
-        self.author = "Upgrade Helper <%s>" % self.author_email
-        self.skip_compilation = skip_compilation
-        self.interactive = not auto_mode
-        self.send_email = send_email
-
-        self.machines = settings.get('machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
-
         self.upgrade_steps = [
             (self._load_env, "Loading environment ..."),
             (self._create_workdir, None),
@@ -167,18 +180,6 @@ class Updater(object):
             (self._buildhistory_diff, None)
         ]
 
-        try:
-            self.base_env = self.bb.env()
-        except EmptyEnvError as e:
-            import traceback
-            E( " %s\n%s" % (e.message, traceback.format_exc()))
-            E( " Bitbake output:\n%s" % (e.stdout))
-            exit(1)
-        self.buildhistory_enabled = self._buildhistory_is_enabled()
-
-        self.email_handler = Email(settings)
-        self.statistics = Statistics()
-
     def _get_status_msg(self, err):
         if err:
             return str(err)
@@ -199,7 +200,7 @@ class Updater(object):
                         " BUILDHISTORY_COMMIT=1 please set.")
                 exit(1)
 
-            if self.skip_compilation:
+            if self.opts['skip_compilation']:
                 W(" Buildhistory disabled because user" \
                         " skip compilation!")
             else:
@@ -228,7 +229,7 @@ class Updater(object):
 
         stdout = self.git.status()
         if stdout != "":
-            if self.interactive:
+            if self.opts['interactive']:
                 W(" %s: git repository has uncommited work which will be dropped! Proceed? (y/N)" % self.pn)
                 answer = sys.stdin.readline().strip().upper()
                 if answer == '' or answer != 'Y':
@@ -261,16 +262,16 @@ class Updater(object):
         else:
             raise UnsupportedProtocolError
 
-        self.recipe = recipe(self.env, self.new_ver, self.interactive, self.workdir,
+        self.recipe = recipe(self.env, self.new_ver, self.opts['interactive'], self.workdir,
                              self.recipe_dir, self.bb, self.git)
 
     def _buildhistory_init(self):
-        if self.buildhistory_enabled == False:
+        if not self.opts['buildhistory_enabled']:
             return
 
         self.buildhistory = BuildHistory(self.bb, self.pn, self.workdir)
-        I(" %s: Initial buildhistory for %s ..." % (self.pn, self.machines))
-        self.buildhistory.init(self.machines)
+        I(" %s: Initial buildhistory for %s ..." % (self.pn, self.opts['machines']))
+        self.buildhistory.init(self.opts['machines'])
 
     def _unpack_original(self):
         self.recipe.unpack()
@@ -289,18 +290,18 @@ class Updater(object):
         self.recipe.fetch()
 
     def _compile(self):
-        if self.skip_compilation:
+        if self.opts['skip_compilation']:
             W(" %s: Compilation was skipped by user choice!")
             return
 
-        for machine in self.machines:
+        for machine in self.opts['machines']:
             I(" %s: compiling for %s ..." % (self.pn, machine))
             self.recipe.compile(machine)
-            if self.buildhistory_enabled == True:
+            if self.opts['buildhistory_enabled']:
                 self.buildhistory.add()
 
     def _buildhistory_diff(self):
-        if self.buildhistory_enabled == False:
+        if not self.opts['buildhistory_enabled']:
             return
 
         I(" %s: Checking buildhistory ..." % self.pn)
@@ -315,7 +316,7 @@ class Updater(object):
 
     # this function will be called at the end of each recipe upgrade
     def pkg_upgrade_handler(self, err):
-        if err and self.patch_file and self.interactive:
+        if err and self.patch_file and self.opts['interactive']:
             answer = "N"
             I(" %s: Do you want to keep the changes? (y/N)" % self.pn)
             answer = sys.stdin.readline().strip().upper()
@@ -380,7 +381,7 @@ class Updater(object):
         if license_diff_fn:
             msg_body += license_change_info % license_diff_fn
         if not err:
-            msg_body += next_steps_info % (', '.join(self.machines),
+            msg_body += next_steps_info % (', '.join(self.opts['machines']),
                     os.path.basename(self.patch_file))
 
         msg_body += mail_footer
@@ -393,7 +394,7 @@ class Updater(object):
                 attachments.append(attachment_fullpath)
 
         # Only send email to Maintainer when recipe upgrade succeed.
-        if self.send_email and not err:
+        if self.opts['send_email'] and not err:
             self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr)
 
         # Preserve email for review purposes.
@@ -415,7 +416,7 @@ class Updater(object):
             self.patch_file = None
             if self.recipe is not None:
                 I(" %s: Auto commit changes ..." % self.pn)
-                self.git.commit(self.recipe.commit_msg, self.author)
+                self.git.commit(self.recipe.commit_msg, self.opts['author'])
                 I(" %s: Save patch in %s." % (self.pn, self.workdir))
                 stdout = self.git.create_patch(self.workdir)
                 self.patch_file = stdout.strip()
@@ -525,7 +526,7 @@ class Updater(object):
 
     def run(self, package_list=None):
         I(" Building gcc runtimes ...")
-        for machine in self.machines:
+        for machine in self.opts['machines']:
             I("  building gcc runtime for %s" % machine)
             self.bb.complete("gcc-runtime", machine)
 
@@ -595,7 +596,7 @@ class Updater(object):
 
             I("%s" % statistics_summary)
 
-            if self.send_email:
+            if self.opts['send_email']:
                 self.send_status_mail()
 
 class UniverseUpdater(Updater):
Signature · 1 line
2.1.4
#2 lore raw

[[AUH] 01/17] {upgradehelper, bitbake}.py: Move _get_env function to bitbake

In reply to Aníbal Limón

Remove unnecesary access function (_get_env) to environment move logic to build dictionary to bitbake env method.

---
 bitbake.py       | 19 ++++++++++++++++++-
 upgradehelper.py | 26 ++++----------------------
 2 files changed, 22 insertions(+), 23 deletions(-)
bitbake.py +18 -1
index a1587ce..cdbce2b 100644
@@ -29,6 +29,8 @@ from logging import debug as D
 from logging import error as E
 from logging import critical as C
 import sys
+import re
+
 from errors import *
 
 for path in os.environ["PATH"].split(':'):
@@ -80,7 +82,22 @@ class Bitbake(object):
         return os.path.join(self.log_dir, BITBAKE_ERROR_LOG)
 
     def env(self, recipe=None):
-        return self._cmd(recipe, "-e", output_filter="-v \"^#\"")
+        stdout = self._cmd(recipe, "-e", output_filter="-v \"^#\"")
+
+        assignment = re.compile("^([^ \t=]*)=(.*)")
+        bb_env = dict()
+        for line in stdout.split('\n'):
+            m = assignment.match(line)
+            if m:
+                if m.group(1) in bb_env:
+                    continue
+
+                bb_env[m.group(1)] = m.group(2).strip("\"")
+
+        if not bb_env:
+            raise EmptyEnvError(stdout)
+
+        return bb_env
 
     def fetch(self, recipe):
         return self._cmd(recipe, "-c fetch")
upgradehelper.py +4 -22
index 0223ac0..128bc07 100755
@@ -168,7 +168,7 @@ class Updater(object):
         ]
 
         try:
-            self.base_env = self._get_env()
+            self.base_env = self.bb.env()
         except EmptyEnvError as e:
             import traceback
             E( " %s\n%s" % (e.message, traceback.format_exc()))
@@ -185,24 +185,6 @@ class Updater(object):
         else:
             return "Succeeded"
 
-    def _get_env(self, pn=None):
-        stdout = self.bb.env(pn)
-
-        assignment = re.compile("^([^ \t=]*)=(.*)")
-        bb_env = dict()
-        for line in stdout.split('\n'):
-            m = assignment.match(line)
-            if m:
-                if m.group(1) in bb_env:
-                    continue
-
-                bb_env[m.group(1)] = m.group(2).strip("\"")
-
-        if not bb_env:
-            raise EmptyEnvError(stdout)
-
-        return bb_env
-
     def _buildhistory_is_enabled(self):
         enabled = False
 
@@ -226,7 +208,7 @@ class Updater(object):
         return enabled
 
     def _load_env(self):
-        self.env = self._get_env(self.pn)
+        self.env = self.bb.env(self.pn)
 
     def _create_workdir(self):
         self.workdir = os.path.join(self.uh_recipes_all_dir, self.pn)
@@ -257,7 +239,7 @@ class Updater(object):
             self.git.reset_hard()
             self.git.clean_untracked()
 
-            self.env = self._get_env(self.pn)
+            self.env = self.bb.env(self.pn)
 
     def _clean_repo(self):
         try:
@@ -296,7 +278,7 @@ class Updater(object):
     def _rename(self):
         self.recipe.rename()
 
-        self.env = self._get_env(self.pn)
+        self.env = self.bb.env(self.pn)
 
         self.recipe.update_env(self.env)
 
Signature · 1 line
2.1.4
#3 lore raw

[[AUH] 05/17] buildhistory: Add option for enable in upgrade-helper.conf

In reply to Aníbal Limón

Make consistent the enablement for this feature it need to be explicit enable into upgrade-helper.conf and also needs configuration in local.conf.

---
 README           |  9 +++++++--
 modules/steps.py |  6 +++---
 upgradehelper.py | 40 ++++++++++++++++++++++++++--------------
 3 files changed, 36 insertions(+), 19 deletions(-)
README +7 -2
index 44c3bf1..ee1bd1f 100644
@@ -58,6 +58,10 @@ drop_previous_commits=yes
 
 # machines to test build with
 machines=qemux86 qemux86-64 qemuarm qemumips qemuppc
+
+# optional features
+buildhistory=no
+
 --------------- snip ---------------
 
 3. Enable distrodata and supply appropriate additional metadata. For
@@ -75,8 +79,9 @@ WARNING: if you are using the default maintainers.inc file supplied
          emails to the default maintainers. Please be careful not to
          do this :)
 
-4. Enable buildhistory by adding the following lines to your
-conf/local.conf file:
+4. If you want to enable buildhistory (optional) you need to enable in
+upgrade-helper.conf also add the following lines to your conf/local.conf
+file:
 
 --------------- snip ---------------
 INHERIT =+ "buildhistory"
modules/steps.py +3 -3
index b85a0dc..4a2eee3 100644
@@ -85,7 +85,7 @@ def detect_recipe_type(bb, git, opts, pkg_ctx):
             pkg_ctx['recipe_dir'], bb, git)
 
 def buildhistory_init(bb, git, opts, pkg_ctx):
-    if not opts['buildhistory_enabled']:
+    if not opts['buildhistory']:
         return
 
     pkg_ctx['buildhistory'] = BuildHistory(bb, pkg_ctx['PN'],
@@ -118,11 +118,11 @@ def compile(bb, git, opts, pkg_ctx):
     for machine in opts['machines']:
         I(" %s: compiling for %s ..." % (pkg_ctx['PN'], machine))
         pkg_ctx['recipe'].compile(machine)
-        if opts['buildhistory_enabled']:
+        if opts['buildhistory']:
             pkg_ctx['buildhistory'].add()
 
 def buildhistory_diff(bb, git, opts, pkg_ctx):
-    if not opts['buildhistory_enabled']:
+    if not opts['buildhistory']:
         return
 
     I(" %s: Checking buildhistory ..." % pkg_ctx['PN'])
upgradehelper.py +26 -14
index e526597..c841c2a 100755
@@ -156,7 +156,7 @@ class Updater(object):
         self.opts['machines'] = settings.get('machines',
                 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
         self.opts['skip_compilation'] = skip_compilation
-        self.opts['buildhistory_enabled'] = self._buildhistory_is_enabled()
+        self.opts['buildhistory'] = self._buildhistory_is_enabled()
 
         self.uh_dir = os.path.join(build_dir, "upgrade-helper")
         if not os.path.exists(self.uh_dir):
@@ -180,22 +180,34 @@ class Updater(object):
     def _buildhistory_is_enabled(self):
         enabled = False
 
-        if 'buildhistory' in self.base_env['INHERIT']:
-            if not 'BUILDHISTORY_COMMIT' in self.base_env:
-                E(" Buildhistory was enabled but need"\
-                        " BUILDHISTORY_COMMIT=1 please set.")
-                exit(1)
+        if settings.get("buildhistory", "no") == "yes":
+            if 'buildhistory' in self.base_env['INHERIT']:
+                if not 'BUILDHISTORY_COMMIT' in self.base_env:
+                    E(" Buildhistory was INHERIT in conf/local.conf"\
+                      " but need BUILDHISTORY_COMMIT=1 please set.")
+                    exit(1)
 
-            if not self.base_env['BUILDHISTORY_COMMIT'] == '1':
-                E(" Buildhistory was enabled but need"\
-                        " BUILDHISTORY_COMMIT=1 please set.")
-                exit(1)
+                if not self.base_env['BUILDHISTORY_COMMIT'] == '1':
+                    E(" Buildhistory was INHERIT in conf/local.conf"\
+                      " but need BUILDHISTORY_COMMIT=1 please set.")
+                    exit(1)
 
-            if self.opts['skip_compilation']:
-                W(" Buildhistory disabled because user" \
-                        " skip compilation!")
+                if self.opts['skip_compilation']:
+                    W(" Buildhistory disabled because user" \
+                            " skip compilation!")
+                else:
+                    enabled = True
             else:
-                enabled = True
+                E(" Buildhistory was enabled in upgrade-helper.conf"\
+                  " but isn't INHERIT in conf/local.conf, if you want"\
+                  " to enable please set.")
+                exit(1)
+        else:
+            if 'buildhistory' in self.base_env['INHERIT']:
+                E(" Buildhistory was INHERIT in conf/local.conf"\
+                  " but buildhistory=yes isn't in upgrade-helper.conf,"\
+                  " if you want to enable please set.")
+                exit(1)
 
         return enabled
 
Signature · 1 line
2.1.4
#4 lore raw

[[AUH] 03/17] upgradehelper.py: Adds own module for steps

In reply to Aníbal Limón

Isolate steps into own module in order to provide better structure of the code.

In order to do that the step interfaces was defined to, step(bb, git, opts, pkg_ctx)

- bb: Bitbake helper instance.
- git: Git helper instance.
- opts: Upgrade helper options.
- pkg_ctx: Package context per upgrade, it stores all information related to one upgrade.

---
 steps.py         | 142 ++++++++++++++++++++++++++++++
 upgradehelper.py | 259 +++++++++++++++++--------------------------------------
 2 files changed, 221 insertions(+), 180 deletions(-)
 create mode 100644 steps.py
steps.py new file +142 -0
new file mode 100644
index 0000000..ea314da
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# vim: set ts=4 sw=4 et:
+#
+# Copyright (c) 2013 - 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+import os
+import sys
+
+from logging import debug as D
+from logging import info as I
+from logging import warning as W
+from logging import error as E
+from logging import critical as C
+
+from errors import *
+from buildhistory import BuildHistory
+from recipe import Recipe
+from gitrecipe import GitRecipe
+from svnrecipe import SvnRecipe
+
+def load_env(bb, git, opts, pkg_ctx):
+    stdout = git.status()
+    if stdout != "":
+        if opts['interactive']:
+            W(" %s: git repository has uncommited work which will be dropped!" \
+                    " Proceed? (y/N)" % pkg_ctx['PN'])
+            answer = sys.stdin.readline().strip().upper()
+            if answer == '' or answer != 'Y':
+                I(" %s: User abort!" % pkg_ctx['PN'])
+                exit(1)
+
+        I(" %s: Dropping uncommited work!" % pkg_ctx['PN'])
+        git.reset_hard()
+        git.clean_untracked()
+
+    pkg_ctx['env'] = bb.env(pkg_ctx['PN'])
+    if pkg_ctx['env']['PV'] == pkg_ctx['NPV']:
+        raise UpgradeNotNeededError
+
+def load_dirs(bb, git, opts, pkg_ctx):
+    pkg_ctx['workdir'] = os.path.join(pkg_ctx['base_dir'], pkg_ctx['PN'])
+    os.mkdir(pkg_ctx['workdir'])
+
+    pkg_ctx['recipe_dir'] = os.path.dirname(pkg_ctx['env']['FILE'])
+
+def clean_repo(bb, git, opts, pkg_ctx):
+    try:
+        git.checkout_branch("upgrades")
+    except Error:
+        git.create_branch("upgrades")
+
+    try:
+        git.delete_branch("remove_patches")
+    except:
+        pass
+
+def detect_recipe_type(bb, git, opts, pkg_ctx):
+    if pkg_ctx['env']['SRC_URI'].find("ftp://") != -1 or  \
+            pkg_ctx['env']['SRC_URI'].find("http://") != -1 or \
+            pkg_ctx['env']['SRC_URI'].find("https://") != -1:
+        recipe = Recipe
+    elif pkg_ctx['env']['SRC_URI'].find("git://") != -1:
+        recipe = GitRecipe
+    else:
+        raise UnsupportedProtocolError
+
+    pkg_ctx['recipe'] = recipe(pkg_ctx['env'], pkg_ctx['NPV'],
+            opts['interactive'], pkg_ctx['workdir'],
+            pkg_ctx['recipe_dir'], bb, git)
+
+def buildhistory_init(bb, git, opts, pkg_ctx):
+    if not opts['buildhistory_enabled']:
+        return
+
+    pkg_ctx['buildhistory'] = BuildHistory(bb, pkg_ctx['PN'],
+            pkg_ctx['workdir'])
+    I(" %s: Initial buildhistory for %s ..." % (pkg_ctx['PN'],
+            opts['machines']))
+    pkg_ctx['buildhistory'].init(opts['machines'])
+
+def unpack_original(bb, git, opts, pkg_ctx):
+    pkg_ctx['recipe'].unpack()
+
+def rename(bb, git, opts, pkg_ctx):
+    pkg_ctx['recipe'].rename()
+
+    pkg_ctx['env'] = bb.env(pkg_ctx['PN'])
+
+    pkg_ctx['recipe'].update_env(pkg_ctx['env'])
+
+def cleanall(bb, git, opts, pkg_ctx):
+    pkg_ctx['recipe'].cleanall()
+
+def fetch(bb, git, opts, pkg_ctx):
+    pkg_ctx['recipe'].fetch()
+
+def compile(bb, git, opts, pkg_ctx):
+    if opts['skip_compilation']:
+        W(" %s: Compilation was skipped by user choice!")
+        return
+
+    for machine in opts['machines']:
+        I(" %s: compiling for %s ..." % (pkg_ctx['PN'], machine))
+        pkg_ctx['recipe'].compile(machine)
+        if opts['buildhistory_enabled']:
+            pkg_ctx['buildhistory'].add()
+
+def buildhistory_diff(bb, git, opts, pkg_ctx):
+    if not opts['buildhistory_enabled']:
+        return
+
+    I(" %s: Checking buildhistory ..." % pkg_ctx['PN'])
+    pkg_ctx['buildhistory'].diff()
+
+upgrade_steps = [
+    (load_env, "Loading environment ..."),
+    (load_dirs, None),
+    (clean_repo, "Cleaning git repository of temporary branch ..."),
+    (detect_recipe_type, None),
+    (buildhistory_init, None),
+    (unpack_original, "Fetch & unpack original version ..."),
+    (rename, "Renaming recipes, reset PR (if exists) ..."),
+    (cleanall, "Clean all ..."),
+    (fetch, "Fetch new version (old checksums) ..."),
+    (compile, None),
+    (buildhistory_diff, None)
+]
upgradehelper.py +79 -180

416 lines not on this page · raw message

Signature · 1 line
2.1.4
#5 lore raw

[[AUH] 06/17] recipe/base.py: Add is_recipe_or_include_file func

In reply to Aníbal Limón

Instead of have duplicate code add this new function for make code easy to read.

---
 modules/recipe/base.py | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)
modules/recipe/base.py +17 -18
index 14aa5bb..15c5f43 100644
@@ -33,6 +33,18 @@ from logging import warning as W
 from errors import *
 from utils.bitbake import *
 
+def is_recipe_or_include_file(full_path_f, f):
+    is_file = os.path.isfile(full_path_f)
+
+    is_recipe = f.find(self.env['PN']) == 0 and \
+                f.find(self.env['PKGV']) != -1 and \
+                f.find(".bb") != -1
+
+    is_include = f.find(self.env['PN']) == 0 and \
+                 f.find(".inc") != -1
+
+    return is_file and (is_recipe or is_include)
+
 class Recipe(object):
     def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git):
         self.env = env
@@ -63,6 +75,7 @@ class Recipe(object):
 
         super(Recipe, self).__init__()
 
+
     def update_env(self, env):
         self.env = env
 
@@ -80,10 +93,7 @@ class Recipe(object):
         # change PR before renaming
         for f in os.listdir(self.recipe_dir):
             full_path_f = os.path.join(self.recipe_dir, f)
-            if os.path.isfile(full_path_f) and \
-                    ((f.find(self.env['PN']) == 0 and f.find(self.env['PKGV']) != -1 and
-                      f.find(".bb") != -1) or
-                     (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)):
+            if is_recipe_or_include_file(full_path_f, f):
                 with open(full_path_f + ".tmp", "w+") as temp_recipe:
                     with open(full_path_f) as recipe:
                         for line in recipe:
@@ -130,12 +140,7 @@ class Recipe(object):
 
         for f in os.listdir(self.recipe_dir):
             full_path_f = os.path.join(self.recipe_dir, f)
-            if os.path.isfile(full_path_f) and \
-                    ((f.find(self.env['PN']) == 0 and
-                      f.find(self.env['PKGV']) != -1 and
-                      f.find(".bb") != -1) or
-                     (f.find(self.env['PN']) == 0 and
-                      f.find(".inc") != -1)):
+            if is_recipe_or_include_file(full_path_f, f):
                 with open(full_path_f + ".tmp", "w+") as temp_recipe:
                     with open(full_path_f) as recipe:
                         for line in recipe:
@@ -178,10 +183,7 @@ class Recipe(object):
         # files... Go through the recipes/inc files until we find them
         for f in os.listdir(self.recipe_dir):
             full_path_f = os.path.join(self.recipe_dir, f)
-            if os.path.isfile(full_path_f) and \
-                    ((f.find(self.env['PN']) == 0 and f.find(self.env['PKGV']) != -1 and
-                      f.find(".bb") != -1) or
-                     (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)):
+            if is_recipe_or_include_file(full_path_f, f):
                 with open(full_path_f + ".tmp", "w+") as temp_recipe:
                     with open(full_path_f) as recipe:
                         for line in recipe:
@@ -218,10 +220,7 @@ class Recipe(object):
         # Will change the extension of the archive from the SRC_URI
         for f in os.listdir(self.recipe_dir):
             full_path_f = os.path.join(self.recipe_dir, f)
-            if os.path.isfile(full_path_f) and \
-                    ((f.find(self.env['PN']) == 0 and f.find(self.env['PKGV']) != -1 and
-                      f.find(".bb") != -1) or
-                     (f.find(self.env['PN']) == 0 and f.find(".inc") != -1)):
+            if is_recipe_or_include_file(full_path_f, f):
                 with open(full_path_f + ".tmp", "w+") as temp_recipe:
                     with open(full_path_f) as recipe:
                         source_found = False
Signature · 1 line
2.1.4
#6 lore raw

[[AUH] 07/17] recipe/base.py: Add modify_recipe_files function decorator

In reply to Aníbal Limón

Remove duplicate code adding modify_recipe_files decorator, this function interate over recipe_dir and found bb and includes files to make modifications.

Modifications to recipe bb and include files are made by function passed to modify_recipe_files.

---
 modules/recipe/base.py | 293 +++++++++++++++++++++++--------------------------
 1 file changed, 140 insertions(+), 153 deletions(-)
modules/recipe/base.py +140 -153
index 15c5f43..5c70c61 100644
@@ -33,18 +33,30 @@ from logging import warning as W
 from errors import *
 from utils.bitbake import *
 
-def is_recipe_or_include_file(full_path_f, f):
+def is_recipe_or_include_file(env, full_path_f, f):
     is_file = os.path.isfile(full_path_f)
 
-    is_recipe = f.find(self.env['PN']) == 0 and \
-                f.find(self.env['PKGV']) != -1 and \
+    is_recipe = f.find(env['PN']) == 0 and \
+                f.find(env['PKGV']) != -1 and \
                 f.find(".bb") != -1
 
-    is_include = f.find(self.env['PN']) == 0 and \
+    is_include = f.find(env['PN']) == 0 and \
                  f.find(".inc") != -1
 
     return is_file and (is_recipe or is_include)
 
+def modify_recipe_files(func):
+    def modify(env, recipe_dir):
+        for f in os.listdir(recipe_dir):
+            full_path_f = os.path.join(recipe_dir, f)
+            if is_recipe_or_include_file(env, full_path_f, f):
+                with open(full_path_f + ".tmp", "w+") as temp_recipe:
+                    with open(full_path_f) as recipe:
+                        for line in recipe:
+                            func(line, temp_recipe)
+                os.rename(full_path_f + ".tmp", full_path_f)
+    return modify
+
 class Recipe(object):
     def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git):
         self.env = env
@@ -90,18 +102,12 @@ class Recipe(object):
             self.git.mv(src_dir, dest_dir)
 
     def rename(self):
-        # change PR before renaming
-        for f in os.listdir(self.recipe_dir):
-            full_path_f = os.path.join(self.recipe_dir, f)
-            if is_recipe_or_include_file(full_path_f, f):
-                with open(full_path_f + ".tmp", "w+") as temp_recipe:
-                    with open(full_path_f) as recipe:
-                        for line in recipe:
-                            if line.startswith("PR=") or line.startswith("PR ="):
-                                continue
-                            else:
-                                temp_recipe.write(line)
-                os.rename(full_path_f + ".tmp", full_path_f)
+        # clean PR before renaming
+        @modify_recipe_files
+        def _clean_pr(line, temp_recipe):
+            if not (line.startswith("PR=") or line.startswith("PR =")):
+                temp_recipe.write(line)
+        _clean_pr(self.env, self.recipe_dir)
 
         # rename recipes (not directories)
         for path in os.listdir(self.recipe_dir):
@@ -121,8 +127,6 @@ class Recipe(object):
         # since we did some renaming, backup the current environment
         self.old_env = self.env
 
-        # start formatting the commit message
-
     def create_diff_file(self, file, old_md5, new_md5):
         old_file = os.path.join(self.old_env['S'], file)
         new_file = os.path.join(self.env['S'], file)
@@ -138,20 +142,6 @@ class Recipe(object):
             f.write("old checksum = %s\n" % old_md5)
             f.write("new_checksum = %s\n" % new_md5)
 
-        for f in os.listdir(self.recipe_dir):
-            full_path_f = os.path.join(self.recipe_dir, f)
-            if is_recipe_or_include_file(full_path_f, f):
-                with open(full_path_f + ".tmp", "w+") as temp_recipe:
-                    with open(full_path_f) as recipe:
-                        for line in recipe:
-                            m = re.match("(.*)" + old_md5 + "(.*)", line)
-                            if m is not None:
-                                temp_recipe.write(m.group(1) + new_md5 + m.group(2) + "\n")
-                            else:
-                                temp_recipe.write(line)
-
-                os.rename(full_path_f + ".tmp", full_path_f)
-
     def _change_recipe_checksums(self, fetch_log):
         sums = {}
 
@@ -178,27 +168,23 @@ class Recipe(object):
         if len(sums) == 0:
             raise FetchError()
 
-        I(" %s: Update recipe checksums ..." % self.env['PN'])
         # checksums are usually in the main recipe but they can also be in inc
         # files... Go through the recipes/inc files until we find them
-        for f in os.listdir(self.recipe_dir):
-            full_path_f = os.path.join(self.recipe_dir, f)
-            if is_recipe_or_include_file(full_path_f, f):
-                with open(full_path_f + ".tmp", "w+") as temp_recipe:
-                    with open(full_path_f) as recipe:
-                        for line in recipe:
-                            for name in sums:
-                                m1 = re.match("^SRC_URI\["+ name + "md5sum\].*", line)
-                                m2 = re.match("^SRC_URI\["+ name + "sha256sum\].*", line)
-                                if m1:
-                                    temp_recipe.write(sums[name]["md5sum"])
-                                elif m2:
-                                    temp_recipe.write(sums[name]["sha256sum"])
-                                else:
-                                    temp_recipe.write(line)
+        @modify_recipe_files
+        def _update_recipe_checksums(line, temp_recipe):
+            for name in sums:
+                m1 = re.match("^SRC_URI\["+ name + "md5sum\].*", line)
+                m2 = re.match("^SRC_URI\["+ name + "sha256sum\].*", line)
+                if m1:
+                    temp_recipe.write(sums[name]["md5sum"])
+                elif m2:
+                    temp_recipe.write(sums[name]["sha256sum"])
+                else:
+                    temp_recipe.write(line)
+
+        I(" %s: Update recipe checksums ..." % self.env['PN'])
+        _update_recipe_checksums(self.env, self.recipe_dir)
 
-                os.rename(full_path_f + ".tmp", full_path_f)
-        
         self.checksums_changed = True
 
     def _is_uri_failure(self, fetch_log):
@@ -218,71 +204,63 @@ class Recipe(object):
 
     def _change_source_suffix(self, new_suffix):
         # Will change the extension of the archive from the SRC_URI
-        for f in os.listdir(self.recipe_dir):
-            full_path_f = os.path.join(self.recipe_dir, f)
-            if is_recipe_or_include_file(full_path_f, f):
-                with open(full_path_f + ".tmp", "w+") as temp_recipe:
-                    with open(full_path_f) as recipe:
-                        source_found = False
-                        for line in recipe:
-                            # source on first line
-                            m1 = re.match("^SRC_URI.*\${PV}\.(.*)[\" \\\\].*", line)
-                            # SRC_URI alone on the first line
-                            m2 = re.match("^SRC_URI.*", line)
-                            # source on second line
-                            m3 = re.match(".*\${PV}\.(.*)[\" \\\\].*", line)
-                            if m1:
-                                old_suffix = m1.group(1)
-                                line = line.replace(old_suffix, new_suffix+" ")
-                            if m2 and not m1:
-                                source_found = True
-                            if m3 and source_found:
-                                old_suffix = m3.group(1)
-                                line = line.replace(old_suffix, new_suffix+" ")
-                                source_found = False
-
-                            temp_recipe.write(line)
-                os.rename(full_path_f + ".tmp", full_path_f)
+
+        source_found = False
+        @modify_recipe_files
+        def _change(line, temp_recipe):
+            # source on first line
+            m1 = re.match("^SRC_URI.*\${PV}\.(.*)[\" \\\\].*", line)
+            # SRC_URI alone on the first line
+            m2 = re.match("^SRC_URI.*", line)
+            # source on second line
+            m3 = re.match(".*\${PV}\.(.*)[\" \\\\].*", line)
+            if m1:
+                old_suffix = m1.group(1)
+                line = line.replace(old_suffix, new_suffix+" ")
+            if m2 and not m1:
+                source_found = True
+            if m3 and source_found:
+                old_suffix = m3.group(1)
+                line = line.replace(old_suffix, new_suffix+" ")
+                source_found = False
+
+            temp_recipe.write(line)
+
+        _change(self.env, self.recipe_dir)
 
     def _remove_patch_uri(self, uri):
-        recipe_files = [
-            os.path.join(self.recipe_dir, self.env['PN'] + ".inc"),
-            self.env['FILE']]
-
-        for recipe_filename in recipe_files:
-            if os.path.isfile(recipe_filename):
-                with open(recipe_filename + ".tmp", "w+") as temp_recipe:
-                    with open(recipe_filename) as recipe:
-                        for line in recipe:
-                            if line.find(uri) == -1:
-                                temp_recipe.write(line)
-                                continue
-                            
-                            m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line)
-                            m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line)
-                            m3 = re.match("[\t ]*" + uri + " *\\\\", line)
-                            m4 = re.match("([\t ]*)" + uri + " *\"", line)
-
-                            # patch on a single SRC_URI line:
-                            if m1:
-                                continue
-                            # patch is on the first SRC_URI line
-                            elif m2:
-                                temp_recipe.write(m2.group(1) + "\\\n")
-                            # patch is in the middle
-                            elif m3:
-                                continue
-                            # patch is last in list
-                            elif m4:
-                                temp_recipe.write(m4.group(1) + "\"\n")
-                            # nothing matched in recipe but we deleted the patch
-                            # anyway? Then we must bail out!
-                            else:
-                                return False
-
-                os.rename(recipe_filename + ".tmp", recipe_filename)
+        removed = True
 
-        return True
+        @modify_recipe_files
+        def _remove(line, temp_recipe):
+            if line.find(uri) == -1:
+               temp_recipe.write(line)
+            else:
+               m1 = re.match("SRC_URI *\+*= *\" *" + uri + " *\"", line)
+               m2 = re.match("(SRC_URI *\+*= *\" *)" + uri + " *\\\\", line)
+               m3 = re.match("[\t ]*" + uri + " *\\\\", line)
+               m4 = re.match("([\t ]*)" + uri + " *\"", line)
+
+               # patch on a single SRC_URI line:
+               if m1:
+                   return
+               # patch is on the first SRC_URI line
+               elif m2:
+                   temp_recipe.write(m2.group(1) + "\\\n")
+               # patch is in the middle
+               elif m3:
+                   return
+               # patch is last in list
+               elif m4:
+                   temp_recipe.write(m4.group(1) + "\"\n")
+               # nothing matched in recipe but we deleted the patch
+               # anyway? Then we must bail out!
+               else:
+                   removed = False
+
+        _remove(self.env, self.recipe_dir)
+
+        return removed
 
     def _remove_faulty_patch(self, patch_log):
         patch_file = None
@@ -343,6 +321,14 @@ class Recipe(object):
         return False
 
     def _license_issue_handled(self, config_log):
+        @modify_recipe_files
+        def _update_license_checksum(line, temp_recipe):
+            m = re.match("(.*)" + old_md5 + "(.*)", line)
+            if m is not None:
+                temp_recipe.write(m.group(1) + new_md5 + m.group(2) + "\n")
+            else:
+                temp_recipe.write(line)
+
         license_file = None
         with open(config_log) as log:
             for line in log:
@@ -368,8 +354,11 @@ class Recipe(object):
                     new_md5 = m_new.group(1)
         
         if license_file is not None:
+            _update_license_checksum(self.env, self.recipe_dir)
+
             self.create_diff_file(license_file, old_md5, new_md5)
             self.license_diff_file = os.path.join(self.workdir, os.path.basename(license_file + ".diff"))
+
             if self.interactive:
                 W("  %s: license checksum failed for file %s. The recipe has"
                   "been updated! View diff? (Y/n)" % (self.env['PN'], license_file))
@@ -492,48 +481,46 @@ class Recipe(object):
                 replacement = "${" + prefixes[largest_prefix] + "}"
                 files[i] = files[i].replace(largest_prefix, replacement)
 
-        recipe_files = [
-            os.path.join(self.recipe_dir, self.env['PN'] + ".inc"),
-            self.env['FILE']]
 
-        # Append the new files
-        for recipe_filename in recipe_files:
-            if os.path.isfile(recipe_filename):
-                with open(recipe_filename + ".tmp", "w+") as temp_recipe:
-                    with open(recipe_filename) as recipe:
-                        files_clause = False
-                        for line in recipe:
-                            if re.match("^FILES_\${PN}[ +=].*", line):
-                                files_clause = True
-                                temp_recipe.write(line)
-                                continue
-                            # Get front spacing
-                            if files_clause:
-                                front_spacing = re.sub("[^ \t]", "", line)
-                            # Append once the last line has of FILES has been reached
-                            if re.match(".*\".*", line) and files_clause:
-                                files_clause = False
-                                line = line.replace("\"", "")
-                                line = line.rstrip()
-                                front_spacing = re.sub("[^ \t]", "", line)
-                                # Do not write an empty line
-                                if line.strip():
-                                    temp_recipe.write(line + " \\\n")
-                                # Add spacing in case there was none
-                                if len(front_spacing) == 0:
-                                    front_spacing = " " * 8
-                                # Write to file
-                                for i in range(len(files)-1):
-                                    line = front_spacing + files[i] + " \\\n"
-                                    temp_recipe.write(line)
-
-                                line = front_spacing + files[len(files) - 1] + "\"\n"
-                                temp_recipe.write(line)
-                                continue
-
-                            temp_recipe.write(line)
-
-                os.rename(recipe_filename + ".tmp", recipe_filename)
+        files_clause = False
+        @modify_recipe_files
+        def _append_new_files(line, temp_file):
+            if re.match("^FILES_\${PN}[ +=].*", line):
+                files_clause = True
+                temp_recipe.write(line)
+                return
+
+            # Get front spacing
+            if files_clause:
+                front_spacing = re.sub("[^ \t]", "", line)
+
+            # Append once the last line has of FILES has been reached
+            if re.match(".*\".*", line) and files_clause:
+                files_clause = False
+                line = line.replace("\"", "")
+                line = line.rstrip()
+                front_spacing = re.sub("[^ \t]", "", line)
+
+                # Do not write an empty line
+                if line.strip():
+                    temp_recipe.write(line + " \\\n")
+
+                # Add spacing in case there was none
+                if len(front_spacing) == 0:
+                    front_spacing = " " * 8
+
+                # Write to file
+                for i in range(len(files) - 1):
+                    line = front_spacing + files[i] + " \\\n"
+                    temp_recipe.write(line)
+
+                line = front_spacing + files[len(files) - 1] + "\"\n"
+                temp_recipe.write(line)
+                return
+
+            temp_recipe.write(line)
+
+        _append_new_files(self.env, self.recipe_dir)
 
     def unpack(self):
         self.bb.unpack(self.env['PN'])
Signature · 1 line
2.1.4
#7 lore raw

[[AUH] 04/17] upgradehelper: Reorder files into directories.

In reply to Aníbal Limón

Reorder files into modules directory to provide a better source tree.

Changed was made for update the import lines and include modules directory into search path.

---
 bitbake.py                    | 124 --------
 buildhistory.py               |  75 -----
 emailhandler.py               | 108 -------
 errors.py                     |  93 ------
 git.py                        | 101 -------
 gitrecipe.py                  |  94 ------
 modules/buildhistory.py       |  75 +++++
 modules/errors.py             |  93 ++++++
 modules/recipe/__init__.py    |   0
 modules/recipe/base.py        | 671 ++++++++++++++++++++++++++++++++++++++++++
 modules/recipe/git.py         |  94 ++++++
 modules/recipe/svn.py         |  28 ++
 modules/statistics.py         | 102 +++++++
 modules/steps.py              | 143 +++++++++
 modules/utils/__init__.py     |   0
 modules/utils/bitbake.py      | 124 ++++++++
 modules/utils/emailhandler.py | 108 +++++++
 modules/utils/git.py          | 102 +++++++
 recipe.py                     | 669 -----------------------------------------
 statistics.py                 | 102 -------
 steps.py                      | 142 ---------
 svnrecipe.py                  |  28 --
 upgradehelper.py              |  11 +-
 23 files changed, 1547 insertions(+), 1540 deletions(-)
 delete mode 100644 bitbake.py
 delete mode 100644 buildhistory.py
 delete mode 100644 emailhandler.py
 delete mode 100644 errors.py
 delete mode 100644 git.py
 delete mode 100644 gitrecipe.py
 create mode 100644 modules/buildhistory.py
 create mode 100644 modules/errors.py
 create mode 100644 modules/recipe/__init__.py
 create mode 100644 modules/recipe/base.py
 create mode 100644 modules/recipe/git.py
 create mode 100644 modules/recipe/svn.py
 create mode 100644 modules/statistics.py
 create mode 100644 modules/steps.py
 create mode 100644 modules/utils/__init__.py
 create mode 100644 modules/utils/bitbake.py
 create mode 100644 modules/utils/emailhandler.py
 create mode 100644 modules/utils/git.py
 delete mode 100644 recipe.py
 delete mode 100644 statistics.py
 delete mode 100644 steps.py
 delete mode 100644 svnrecipe.py
bitbake.py deleted +0 -124
deleted file mode 100644
index cdbce2b..0000000
@@ -1,124 +0,0 @@
-#!/usr/bin/env python
-# vim: set ts=4 sw=4 et:
-#
-# Copyright (c) 2013 - 2014 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#
-# AUTHORS
-# Laurentiu Palcu   <[email protected]>
-# Marius Avram      <[email protected]>
-#
-
-import os
-import logging as log
-from logging import info as I
-from logging import debug as D
-from logging import error as E
-from logging import critical as C
-import sys
-import re
-
-from errors import *
-
-for path in os.environ["PATH"].split(':'):
-    if os.path.exists(path) and "bitbake" in os.listdir(path):
-        sys.path.insert(0, os.path.join(path, "../lib"))
-        import bb
-
-BITBAKE_ERROR_LOG = 'bitbake_error_log.txt'
-
-class Bitbake(object):
-    def __init__(self, build_dir):
-        self.build_dir = build_dir
-        self.log_dir = None
-        super(Bitbake, self).__init__()
-
-    def _cmd(self, recipe=None, options=None, env_var=None, output_filter=None):
-        cmd = ""
-        if env_var is not None:
-            cmd += env_var + " "
-        cmd += "bitbake "
-        if options is not None:
-            cmd += options + " "
-
-        if recipe is not None:
-            cmd += recipe
-
-        if output_filter is not None:
-            cmd += ' |  grep ' + output_filter
-
-        os.chdir(self.build_dir)
-
-        try:
-            stdout, stderr = bb.process.run(cmd)
-        except bb.process.ExecutionError as e:
-            D("%s returned:\n%s" % (cmd, e.__str__()))
-
-            if self.log_dir is not None and os.path.exists(self.log_dir):
-                with open(os.path.join(self.log_dir, BITBAKE_ERROR_LOG), "w+") as log:
-                    log.write(e.stdout)
-
-            raise Error("\'" + cmd + "\' failed", e.stdout, e.stderr)
-
-        return stdout
-
-    def set_log_dir(self, dir):
-        self.log_dir = dir
-
-    def get_stdout_log(self):
-        return os.path.join(self.log_dir, BITBAKE_ERROR_LOG)
-
-    def env(self, recipe=None):
-        stdout = self._cmd(recipe, "-e", output_filter="-v \"^#\"")
-
-        assignment = re.compile("^([^ \t=]*)=(.*)")
-        bb_env = dict()
-        for line in stdout.split('\n'):
-            m = assignment.match(line)
-            if m:
-                if m.group(1) in bb_env:
-                    continue
-
-                bb_env[m.group(1)] = m.group(2).strip("\"")
-
-        if not bb_env:
-            raise EmptyEnvError(stdout)
-
-        return bb_env
-
-    def fetch(self, recipe):
-        return self._cmd(recipe, "-c fetch")
-
-    def unpack(self, recipe):
-        return self._cmd(recipe, "-c unpack")
-
-    def checkpkg(self, recipe):
-        if recipe == "universe":
-            return self._cmd(recipe, "-c checkpkg -k")
-        else:
-            return self._cmd(recipe, "-c checkpkg")
-
-    def cleanall(self, recipe):
-        return self._cmd(recipe, "-c cleanall")
-
-    def cleansstate(self, recipe):
-        return self._cmd(recipe, "-c cleansstate")
-
-    def complete(self, recipe, machine):
-        return self._cmd(recipe, env_var="MACHINE=" + machine)
-
-    def dependency_graph(self, package_list):
-        return self._cmd(package_list, "-g")
buildhistory.py deleted +0 -75

75 lines not on this page · raw message

emailhandler.py deleted +0 -108

108 lines not on this page · raw message

errors.py deleted +0 -93

93 lines not on this page · raw message

git.py deleted +0 -101

101 lines not on this page · raw message

gitrecipe.py deleted +0 -94

94 lines not on this page · raw message

modules/buildhistory.py new file +75 -0

75 lines not on this page · raw message

modules/errors.py new file +93 -0

93 lines not on this page · raw message

modules/recipe/__init__.py new file +0 -0
new file mode 100644
index 0000000..e69de29
modules/recipe/base.py new file +671 -0

671 lines not on this page · raw message

modules/recipe/git.py new file +94 -0

94 lines not on this page · raw message

modules/recipe/svn.py new file +28 -0
new file mode 100644
index 0000000..6d60529
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# vim: set ts=4 sw=4 et:
+#
+# Copyright (c) 2013 - 2014 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# AUTHORS
+# Laurentiu Palcu   <[email protected]>
+# Marius Avram      <[email protected]>
+#
+
+from base import Recipe
+
+class SvnRecipe(Recipe):
+    pass
modules/statistics.py new file +102 -0

102 lines not on this page · raw message

modules/steps.py new file +143 -0

143 lines not on this page · raw message

modules/utils/__init__.py new file +0 -0
new file mode 100644
index 0000000..e69de29
modules/utils/bitbake.py new file +124 -0

124 lines not on this page · raw message

modules/utils/emailhandler.py new file +108 -0

108 lines not on this page · raw message

modules/utils/git.py new file +102 -0

102 lines not on this page · raw message

recipe.py deleted +0 -669

669 lines not on this page · raw message

statistics.py deleted +0 -102

102 lines not on this page · raw message

steps.py deleted +0 -142

94 lines not on this page · raw message

svnrecipe.py deleted +0 -28
deleted file mode 100644
index 7d8ad16..0000000

Lines of this file are not shown. View the raw message.

upgradehelper.py +7 -4
index 22365b7..e526597 100755

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.1.4

Part of the diff is not shown here. View the raw message.

#8 lore raw

[[AUH] 09/17] steps.py: Merge load_dirs step into load_env.

In reply to Aníbal Limón
---
 modules/steps.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)
modules/steps.py +3 -6
index 4a2eee3..19549cc 100644
@@ -50,15 +50,13 @@ def load_env(bb, git, opts, pkg_ctx):
         git.clean_untracked()
 
     pkg_ctx['env'] = bb.env(pkg_ctx['PN'])
-    if pkg_ctx['env']['PV'] == pkg_ctx['NPV']:
-        raise UpgradeNotNeededError
-
-def load_dirs(bb, git, opts, pkg_ctx):
     pkg_ctx['workdir'] = os.path.join(pkg_ctx['base_dir'], pkg_ctx['PN'])
     os.mkdir(pkg_ctx['workdir'])
-
     pkg_ctx['recipe_dir'] = os.path.dirname(pkg_ctx['env']['FILE'])
 
+    if pkg_ctx['env']['PV'] == pkg_ctx['NPV']:
+        raise UpgradeNotNeededError
+
 def clean_repo(bb, git, opts, pkg_ctx):
     try:
         git.checkout_branch("upgrades")
@@ -130,7 +128,6 @@ def buildhistory_diff(bb, git, opts, pkg_ctx):
 
 upgrade_steps = [
     (load_env, "Loading environment ..."),
-    (load_dirs, None),
     (clean_repo, "Cleaning git repository of temporary branch ..."),
     (detect_recipe_type, None),
     (buildhistory_init, None),
Signature · 1 line
2.1.4
#9 lore raw

[[AUH] 10/17] steps.py: Move clean_repo to first step.

In reply to Aníbal Limón

In order to avoid random errors of uncleaned repository.

---
 modules/steps.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)
modules/steps.py +16 -12
index 19549cc..2685b80 100644
@@ -34,6 +34,21 @@ from recipe.base import Recipe
 from recipe.git import GitRecipe
 from recipe.svn import SvnRecipe
 
+def clean_repo(bb, git, opts, pkg_ctx):
+    git.checkout_branch("master")
+
+    try:
+        git.delete_branch("remove_patches")
+    except:
+        pass
+    try:
+        git.delete_branch("upgrades")
+    except:
+        pass
+
+    git.reset_hard()
+    git.create_branch("upgrades")
+
 def load_env(bb, git, opts, pkg_ctx):
     stdout = git.status()
     if stdout != "":
@@ -57,17 +72,6 @@ def load_env(bb, git, opts, pkg_ctx):
     if pkg_ctx['env']['PV'] == pkg_ctx['NPV']:
         raise UpgradeNotNeededError
 
-def clean_repo(bb, git, opts, pkg_ctx):
-    try:
-        git.checkout_branch("upgrades")
-    except Error:
-        git.create_branch("upgrades")
-
-    try:
-        git.delete_branch("remove_patches")
-    except:
-        pass
-
 def detect_recipe_type(bb, git, opts, pkg_ctx):
     if pkg_ctx['env']['SRC_URI'].find("ftp://") != -1 or  \
             pkg_ctx['env']['SRC_URI'].find("http://") != -1 or \
@@ -127,8 +131,8 @@ def buildhistory_diff(bb, git, opts, pkg_ctx):
     pkg_ctx['buildhistory'].diff()
 
 upgrade_steps = [
-    (load_env, "Loading environment ..."),
     (clean_repo, "Cleaning git repository of temporary branch ..."),
+    (load_env, "Loading environment ..."),
     (detect_recipe_type, None),
     (buildhistory_init, None),
     (unpack_original, "Fetch & unpack original version ..."),
Signature · 1 line
2.1.4
#10 lore raw

[[AUH] 08/17] recipe/base.py: Add support for get recipe inherits.

In reply to Aníbal Limón

Recipe inherits are needed to detect what recipes support certain feature.

For example: For detect ptest enable recipes and then do ptest runtime test after succesful upgrade.

---
 modules/recipe/base.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
modules/recipe/base.py +25 -0
index 5c70c61..dabad2f 100644
@@ -57,6 +57,16 @@ def modify_recipe_files(func):
                 os.rename(full_path_f + ".tmp", full_path_f)
     return modify
 
+def read_recipe_files(func):
+    def read(env, recipe_dir):
+        for f in os.listdir(recipe_dir):
+            full_path_f = os.path.join(recipe_dir, f)
+            if is_recipe_or_include_file(env, full_path_f, f):
+                with open(full_path_f) as recipe:
+                    for line in recipe:
+                        func(line)
+    return read
+
 class Recipe(object):
     def __init__(self, env, new_ver, interactive, workdir, recipe_dir, bitbake, git):
         self.env = env
@@ -85,8 +95,23 @@ class Recipe(object):
         self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver + "\n\n"
         self.rm_patches_msg = "\n\nRemoved the following patch(es):\n"
 
+        self._inherits = None
+
         super(Recipe, self).__init__()
 
+    def get_inherits(self):
+        @read_recipe_files
+        def _get_inherits(line):
+            m = re.search("^inherit (.*)$", line)
+            if m:
+                tmp = m.group(1).split()
+                self._inherits.extend(tmp)
+
+        if not self._inherits:
+            self._inherits = []
+            _get_inherits(self.env, self.recipe_dir)
+
+        return self._inherits
 
     def update_env(self, env):
         self.env = env
Signature · 1 line
2.1.4
#11 lore raw

Re: [[AUH] 08/17] recipe/base.py: Add support for get recipe inherits.

Quoted 46 lines · On Wednesday 25 November 2015 18:00:37 Aníbal Limón wrote:

Recipe inherits are needed to detect what recipes support certain feature.

For example: For detect ptest enable recipes and then do ptest runtime test after succesful upgrade.

---
 modules/recipe/base.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
modules/recipe/base.py +10 -0
index 5c70c61..dabad2f 100644
@@ -57,6 +57,16 @@ def modify_recipe_files(func):
                 os.rename(full_path_f + ".tmp", full_path_f)
     return modify
 
+def read_recipe_files(func):
+    def read(env, recipe_dir):
+        for f in os.listdir(recipe_dir):
+            full_path_f = os.path.join(recipe_dir, f)
+            if is_recipe_or_include_file(env, full_path_f, f):
+                with open(full_path_f) as recipe:
+                    for line in recipe:
+                        func(line)
+    return read
+
 class Recipe(object):
     def __init__(self, env, new_ver, interactive, workdir, recipe_dir,

bitbake, git): self.env = env

modules/recipe/base.py +0 -0
@@ -85,8 +95,23 @@ class Recipe(object):
         self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver +

"\n\n" self.rm_patches_msg = "\n\nRemoved the following patch(es):\n"

modules/recipe/base.py +8 -0
+        self._inherits = None
+
         super(Recipe, self).__init__()
 
+    def get_inherits(self):
+        @read_recipe_files
+        def _get_inherits(line):
+            m = re.search("^inherit (.*)$", line)
+            if m:
+                tmp = m.group(1).split()

modules/recipe/base.py class Recipe(object): · line 117

Quoted 10 lines
modules/recipe/base.py +7 -0
+                self._inherits.extend(tmp)
+
+        if not self._inherits:
+            self._inherits = []
+            _get_inherits(self.env, self.recipe_dir)
+
+        return self._inherits
 
     def update_env(self, env):
         self.env = env

I'm not sure I like this - you're not parsing the recipe so you'll miss things such as inc files inheriting the recipe. Would it be possible to parse the recipe properly instead e.g. with tinfoil?

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#12 lore raw

Re: [[AUH] 08/17] recipe/base.py: Add support for get recipe inherits.

Quoted 55 lines · Paul Eggleton
Quoted 54 lines · On Wednesday 25 November 2015 18:00:37 Aníbal Limón wrote:

Recipe inherits are needed to detect what recipes support certain feature.

For example: For detect ptest enable recipes and then do ptest runtime test after succesful upgrade.

---
 modules/recipe/base.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
modules/recipe/base.py +10 -0
index 5c70c61..dabad2f 100644
@@ -57,6 +57,16 @@ def modify_recipe_files(func):
                 os.rename(full_path_f + ".tmp", full_path_f)
     return modify
 
+def read_recipe_files(func):
+    def read(env, recipe_dir):
+        for f in os.listdir(recipe_dir):
+            full_path_f = os.path.join(recipe_dir, f)
+            if is_recipe_or_include_file(env, full_path_f, f):
+                with open(full_path_f) as recipe:
+                    for line in recipe:
+                        func(line)
+    return read
+
 class Recipe(object):
     def __init__(self, env, new_ver, interactive, workdir, recipe_dir,

bitbake, git): self.env = env

modules/recipe/base.py +0 -0
@@ -85,8 +95,23 @@ class Recipe(object):
         self.commit_msg = self.env['PN'] + ": upgrade to " + self.new_ver +

"\n\n" self.rm_patches_msg = "\n\nRemoved the following patch(es):\n"

modules/recipe/base.py +15 -0
+        self._inherits = None
+
         super(Recipe, self).__init__()
 
+    def get_inherits(self):
+        @read_recipe_files
+        def _get_inherits(line):
+            m = re.search("^inherit (.*)$", line)
+            if m:
+                tmp = m.group(1).split()
+                self._inherits.extend(tmp)
+
+        if not self._inherits:
+            self._inherits = []
+            _get_inherits(self.env, self.recipe_dir)
+
+        return self._inherits
Quoted 6 lines
Quoted 2 lines

def update_env(self, env):
self.env = env

I'm not sure I like this - you're not parsing the recipe so you'll miss things such as inc files inheriting the recipe. Would it be possible to parse the recipe properly instead e.g. with tinfoil?

I agree with you but for now it works for detect recipes that have ptest enabled.

This change isn't trivial so i loaded a ticket for change AUH to use tinfoil (may be tinfoil2 :)).

Cheers,
alimon

[1] https://bugzilla.yoctoproject.org/show_bug.cgi?id=8735

Quoted 4 lines

Cheers,
Paul

#13 lore raw

[[AUH] 11/17] utils/git.py: Add method for apply patches into a branch.

In reply to Aníbal Limón
---
 modules/utils/git.py | 3 +++
 1 file changed, 3 insertions(+)
modules/utils/git.py +3 -0
index 48ed46d..d7f5713 100644
@@ -61,6 +61,9 @@ class Git(object):
     def create_patch(self, out_dir):
         return self._cmd("format-patch -M10 -1 -o " + out_dir)
 
+    def apply_patch(self, patch_file):
+        return self._cmd("am %s" % patch_file)
+
     def status(self):
         return self._cmd("status --porcelain")
 
Signature · 1 line
2.1.4
#14 lore raw

Re: [[AUH] 11/17] utils/git.py: Add method for apply patches into a branch.

Quoted 10 lines · On Wednesday 25 November 2015 18:00:40 Aníbal Limón wrote:
---
 modules/utils/git.py | 3 +++
 1 file changed, 3 insertions(+)
modules/utils/git.py +0 -0
index 48ed46d..d7f5713 100644

modules/utils/git.py class Git(object): · line 68

Quoted 8 lines
modules/utils/git.py +3 -0
@@ -61,6 +61,9 @@ class Git(object):
     def create_patch(self, out_dir):
         return self._cmd("format-patch -M10 -1 -o " + out_dir)
 
+    def apply_patch(self, patch_file):
+        return self._cmd("am %s" % patch_file)
+
     def status(self):
         return self._cmd("status --porcelain")

Just a warning (which may not apply in your situation, not sure) - you may need to do extra cleanup beyond git am --abort if this fails; see:

http://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=e0b9a96002bb23e7c2c8a8c9c4d2431461fd6cb7

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#15 lore raw

Re: [[AUH] 11/17] utils/git.py: Add method for apply patches into a branch.

Good observation, i'll make this change.

alimon

Quoted 28 lines · Paul Eggleton
Quoted 10 lines · On Wednesday 25 November 2015 18:00:40 Aníbal Limón wrote:
---
 modules/utils/git.py | 3 +++
 1 file changed, 3 insertions(+)
modules/utils/git.py +0 -0
index 48ed46d..d7f5713 100644

modules/utils/git.py class Git(object): · line 68

Quoted 8 lines
modules/utils/git.py +3 -0
@@ -61,6 +61,9 @@ class Git(object):
     def create_patch(self, out_dir):
         return self._cmd("format-patch -M10 -1 -o " + out_dir)
 
+    def apply_patch(self, patch_file):
+        return self._cmd("am %s" % patch_file)
+
     def status(self):
         return self._cmd("status --porcelain")

Just a warning (which may not apply in your situation, not sure) - you may need to do extra cleanup beyond git am --abort if this fails; see:

http://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=e0b9a96002bb23e7c2c8a8c9c4d2431461fd6cb7

Cheers,
Paul

#16 lore raw

[[AUH] 12/17] upgradehelper.py: Add settings for enable testimage.

In reply to Aníbal Limón

Testimage feature will provide the ability to run testimage class over a set of succesful upgrades.

Also it will run ptest on recipes that support ptest and retrive the result for store/send to maintainer.

---
 README           | 25 +++++++++++++++++++++++++
 upgradehelper.py | 38 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 2 deletions(-)
README +25 -0
index ee1bd1f..6432f97 100644
@@ -61,6 +61,7 @@ machines=qemux86 qemux86-64 qemuarm qemumips qemuppc
 
 # optional features
 buildhistory=no
+testimage=no
 
 --------------- snip ---------------
 
@@ -93,6 +94,30 @@ $BUILDDIR/upgrade-helper/work/recipe/buildhistory-diff.txt.
 
 (Do not remove any other inherited class in the process, e.g. distrodata).
 
+5. If you want to enable testimage (optional) you need to enable in
+upgrade-helper.conf also add the following lines to your conf/local.conf
+file:
+
+--------------- snip ---------------
+INHERIT += "testimage"
+
+EXTRA_IMAGE_FEATURES = "debug-tweaks package-management"
+# testimage/ptest only work with rpm
+PACKAGE_CLASSES = "package_rpm"
+--------------- snip ---------------
+
+Also if you are running in a server without X11 session, you need to start
+a vncserver example:
+
+$ vncserver :1
+$ export DISPLAY=:1
+
+If upgrade is succesful testimage/ptest results are generated into
+$BUILDDIR/upgrade-helper/work/recipe/ptest_recipe.log if recipe support
+ptest.
+
+(Do not remove any other inherited class in the process, e.g. distrodata).
+
 Usage
 -----
 
upgradehelper.py +36 -2
index c841c2a..222909b 100755
@@ -157,6 +157,7 @@ class Updater(object):
                 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
         self.opts['skip_compilation'] = skip_compilation
         self.opts['buildhistory'] = self._buildhistory_is_enabled()
+        self.opts['testimage'] = self._testimage_is_enabled()
 
         self.uh_dir = os.path.join(build_dir, "upgrade-helper")
         if not os.path.exists(self.uh_dir):
@@ -211,6 +212,41 @@ class Updater(object):
 
         return enabled
 
+    def _testimage_is_enabled(self):
+        enabled = False
+
+        if settings.get("testimage", "no") == "yes":
+            if 'testimage' in self.base_env['INHERIT']:
+                if not "ptest" in self.base_env["DISTRO_FEATURES"]:
+                    E(" testimage requires ptest in DISTRO_FEATURES please add to"\
+                      " conf/local.conf.")
+                    exit(1)
+
+                if not "package-management" in self.base_env['EXTRA_IMAGE_FEATURES']:
+                    E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\
+                      " please add to conf/local.conf.")
+                    exit(1)
+
+                if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]:
+                    E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\
+                      " please add to conf/local.conf.")
+                    exit(1)
+
+                enabled = True
+            else:
+                E(" testimage was enabled in upgrade-helper.conf"\
+                  " but isn't INHERIT in conf/local.conf, if you want"\
+                  " to enable please set.")
+                exit(1)
+        else:
+            if 'testimage' in self.base_env['INHERIT']:
+                E(" testimage was INHERIT in conf/local.conf"\
+                  " but testimage=yes isn't in upgrade-helper.conf,"\
+                  " if you want to enable please set.")
+                exit(1)
+
+        return enabled
+
     def _get_packages_to_upgrade(self, packages=None):
         if packages is None:
             I( "Nothing to upgrade")
@@ -497,10 +533,8 @@ class Updater(object):
                     self.uh_recipes_failed_dir, pkg_ctx['PN']))
 
             self.commit_changes(pkg_ctx)
-
             self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'],
                     pkg_ctx['MAINTAINER'], pkg_ctx['error'])
-
             self.pkg_upgrade_handler(pkg_ctx)
 
         if attempted_pkgs > 1:
Signature · 1 line
2.1.4
#17 lore raw

[[AUH] 13/17] upgradehelper: Add testimage feature.

In reply to Aníbal Limón

Testimage module implements usage of testimage class, a integration branch is created with succesful recipe upgrades and then the next tests are run for every machine configured,

- ptest: Recipes that support ptest are run and retrive the result after upgrade.
- sato: Runs core-image-sato -c testimage.

The results are stored into recipe work directory and send to the maintainer.

[YOCTO #7471]
[YOCTO #7567]

---
 modules/testimage.py | 185 +++++++++++++++++++++++++++++++++++++++++++++++++++
 upgradehelper.py     |  82 +++++++++++++++++++++--
 2 files changed, 261 insertions(+), 6 deletions(-)
 create mode 100644 modules/testimage.py
modules/testimage.py new file +185 -0
new file mode 100644
index 0000000..3fbbc19
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+# vim: set ts=4 sw=4 et:
+#
+# Copyright (c) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# This module implements logic for run image tests on recipes when upgrade
+# process succeed.
+#
+
+import os
+import sys
+import shutil
+
+import logging as log
+from logging import debug as D
+from logging import info as I
+from logging import warning as W
+from logging import error as E
+from logging import critical as C
+
+from errors import *
+from utils.bitbake import *
+
+class TestImage():
+    def __init__(self, bb, git, uh_work_dir, pkgs_ctx):
+        self.bb = bb
+        self.git = git
+        self.uh_work_dir = uh_work_dir
+        self.pkgs_ctx = pkgs_ctx
+
+        os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
+            " TEST_SUITES CORE_IMAGE_EXTRA_INSTALL"
+
+    def _get_ptest_pkgs(self):
+        pkgs = []
+
+        for c in self.pkgs_ctx:
+            if "ptest" in c['recipe'].get_inherits():
+                pkgs.append(c)
+
+        return pkgs
+
+    def _get_pkgs_to_install(self, pkgs, ptest=False):
+        pkgs_out = []
+
+        # for provide access to the target
+        if ptest:
+            pkgs_out.append("dropbear")
+
+        for c in pkgs:
+            pkgs_out.append(c['PN'])
+            if ptest:
+                pkgs_out.append("%s-ptest" % c['PN'])
+
+        return ' '.join(pkgs_out)
+
+    def prepare_branch(self):
+        self.git.checkout_branch("master")
+        try:
+            self.git.delete_branch("testimage")
+            self.git.delete_branch("upgrades")
+        except Error:
+            pass
+        self.git.reset_hard()
+
+        self.git.create_branch("testimage")
+        for c in self.pkgs_ctx:
+            patch_file = os.path.join(c['workdir'], c['patch_file'])
+            self.git.apply_patch(patch_file)
+
+    def _parse_ptest_log(self, log_file):
+        ptest_results = {}
+
+        with open(log_file, "r") as f:
+            pn = None
+            processing = False
+
+            for line in f:
+                if not processing:
+                    m = re.search("^BEGIN: /usr/lib/(.*)/ptest$", line)
+                    if m:
+                        pn = m.group(1)
+                        ptest_results[pn] = []
+                        processing = True
+                else:
+                    m = re.search("^END: $", line)
+                    if m:
+                        pn = None
+                        processing = False
+                    else:
+                        ptest_results[pn].append(line)
+
+        return ptest_results
+
+    def _find_log(self, name, machine):
+        result = []
+
+        base_dir = os.path.join(os.getenv('BUILDDIR'), 'tmp', 'work')
+        for root, dirs, files in os.walk(base_dir):
+            if name in files:
+                result.append(os.path.join(root, name))
+
+        for ptest_log in result:
+            if machine in ptest_log:
+                return ptest_log
+
+    def ptest(self, machine):
+        ptest_pkgs = self._get_ptest_pkgs()
+
+        os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
+            self._get_pkgs_to_install(ptest_pkgs, True)
+        I( "   building core-image-minimal for %s ..." % machine)
+        self.bb.complete("core-image-minimal", machine)
+
+        os.environ['TEST_SUITES'] = "ping ssh _ptest"
+        I( "   running core-image-minimal/ptest for %s ..." % machine)
+        self.bb.complete("core-image-minimal -c testimage", machine)
+
+        ptest_log_file = self._find_log("ptest.log", machine)
+        shutil.copyfile(ptest_log_file,
+                os.path.join(self.uh_work_dir, "ptest_%s.log" % machine))
+
+        ptest_result = self._parse_ptest_log(ptest_log_file)
+        for pn in ptest_result:
+            for pkg_ctx in self.pkgs_ctx:
+                if not pn == pkg_ctx['PN']:
+                    continue 
+
+                if not 'ptest' in pkg_ctx:
+                    pkg_ctx['ptest'] = {}
+                if not 'ptest_log' in pkg_ctx:
+                    pkg_ctx['ptest_log'] = os.path.join(pkg_ctx['workdir'],
+                        "ptest.log")
+
+                pkg_ctx['ptest'][machine] = True
+                with open(pkg_ctx['ptest_log'], "a+") as f:
+                    f.write("BEGIN: PTEST for %s\n" % machine)
+                    for line in ptest_result[pn]:
+                        f.write(line)
+                    f.write("END: PTEST for %s\n" % machine)
+
+    def sato(self, machine):
+        os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
+            self._get_pkgs_to_install(self.pkgs_ctx)
+
+        if 'TEST_SUITES' in os.environ:
+            del os.environ['TEST_SUITES']
+
+        I( "   building core-image-sato for %s ..." % machine)
+        self.bb.complete("core-image-sato", machine)
+
+        I( "   running core-image-sato/testimage for %s ..." % machine)
+        self.bb.complete("core-image-sato -c testimage", machine)
+
+        log_file = self._find_log("log.do_testimage", machine)
+        shutil.copyfile(log_file,
+                os.path.join(self.uh_work_dir, "log_%s.do_testimage" % machine))
+        for pkg_ctx in self.pkgs_ctx:
+            if not 'testimage' in pkg_ctx:
+                pkg_ctx['testimage'] = {}
+            if not 'testimage_log' in pkg_ctx:
+                pkg_ctx['testimage_log'] = os.path.join(
+                    pkg_ctx['workdir'], "log.do_testimage")
+
+            pkg_ctx['testimage'][machine] = True
+            with open(log_file, "r") as lf:
+                with open(pkg_ctx['testimage_log'], "a+") as of:
+                    of.write("BEGIN: TESTIMAGE for %s\n" % machine)
+                    for line in lf:
+                        of.write(line)
+                    of.write("END: TESTIMAGE for %s\n" % machine)
upgradehelper.py +76 -6

146 lines not on this page · raw message

Signature · 1 line
2.1.4
#18 lore raw

Re: [[AUH] 13/17] upgradehelper: Add testimage feature.

Quoted 7 lines · On Wednesday 25 November 2015 18:00:42 Aníbal Limón wrote:

Testimage module implements usage of testimage class, a integration branch is created with succesful recipe upgrades and then the next tests are run for every machine configured,

- ptest: Recipes that support ptest are run and retrive the result after upgrade.
- sato: Runs core-image-sato -c testimage.

Can we avoid hardcoding sato anywhere here? It would be ideal if the image could be configured or passed in instead.

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#19 lore raw

Re: [[AUH] 13/17] upgradehelper: Add testimage feature.

Sure,
alimon

Quoted 15 lines · Paul Eggleton
Quoted 7 lines · On Wednesday 25 November 2015 18:00:42 Aníbal Limón wrote:

Testimage module implements usage of testimage class, a integration branch is created with succesful recipe upgrades and then the next tests are run for every machine configured,

- ptest: Recipes that support ptest are run and retrive the result after upgrade.
- sato: Runs core-image-sato -c testimage.

Can we avoid hardcoding sato anywhere here? It would be ideal if the image could be configured or passed in instead.

Cheers,
Paul

#20 lore raw

[[AUH] 14/17] upgradehelper.py: Changed retry failure build to 30 days.

In reply to Aníbal Limón

When build fails is stored into history fail that enables to don't try to build again in a period of time, so change that period to 30 days since the work patches will be publish in web server.

---
 upgradehelper.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
upgradehelper.py +3 -3
index 18db9eb..6cf8dd0 100755
@@ -738,12 +738,12 @@ class UniverseUpdater(Updater):
                     date.toordinal(date.today()) - \
                     date.toordinal(datetime.strptime(self.history[pn][2], '%Y-%m-%d'))
                 # retry recipes that had fetch errors or other errors after
-                # more than 7 days
+                # more than 30 days
                 if (self.history[pn][3] == str(FetchError()) or
-                        self.history[pn][3] == str(Error())) and retry_delta > 7:
+                        self.history[pn][3] == str(Error())) and retry_delta > 30:
                     return True
 
-                D(" Skipping upgrade of %s: is in history and not 7 days passed" % pn)
+                D(" Skipping upgrade of %s: is in history and not 30 days passed" % pn)
                 return False
 
         # drop native/cross/cross-canadian recipes. We deal with native
Signature · 1 line
2.1.4
#21 lore raw

[[AUH] 16/17] statistics: Improve email format and get_summary method.

In reply to Aníbal Limón
---
 modules/statistics.py | 24 ++++++++++++++----------
 upgradehelper.py      | 11 +++++------
 2 files changed, 19 insertions(+), 16 deletions(-)
modules/statistics.py +14 -10
index 32a6748..905a4cc 100644
@@ -62,16 +62,15 @@ class Statistics(object):
 
         self.total_attempted += 1
 
-    def pkg_stats(self):
-        stat_msg = "\nUpgrade statistics:\n"
-        stat_msg += "====================================================\n"
+    def _pkg_stats(self):
+        stat_msg = "Recipe upgrade statistics:\n\n"
         for status in self.upgrade_stats:
             list_len = len(self.upgrade_stats[status])
             if list_len > 0:
-                stat_msg += "* " + status + ": " + str(list_len) + "\n"
+                stat_msg += "    * " + status + ": " + str(list_len) + "\n"
 
                 for pkg, new_ver, maintainer in self.upgrade_stats[status]:
-                    stat_msg += "    " + pkg + ", " + new_ver + ", " + \
+                    stat_msg += "        " + pkg + ", " + new_ver + ", " + \
                                 maintainer + "\n"
 
         if self.total_attempted == 0:
@@ -80,8 +79,7 @@ class Statistics(object):
         else:
             percent_succeded = self.succeeded["total"] * 100.0 / self.total_attempted
             percent_failed = self.failed["total"] * 100.0 / self.total_attempted
-        stat_msg += "++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
-        stat_msg += "TOTAL: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n\n" % \
+        stat_msg += "\n    TOTAL: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n\n" % \
                     (self.total_attempted, self.succeeded["total"],
                     percent_succeded,
                     self.failed["total"],
@@ -89,14 +87,20 @@ class Statistics(object):
 
         return stat_msg
 
-    def maintainer_stats(self):
-        stat_msg = "* Statistics per maintainer:\n"
+    def _maintainer_stats(self):
+        stat_msg = "Recipe upgrade statistics per Maintainer:\n\n"
         for m in self.maintainers:
             attempted = self.succeeded[m] + self.failed[m]
-            stat_msg += "    %s: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n\n" % \
+            stat_msg += "    %s: attempted=%d succeeded=%d(%.2f%%) failed=%d(%.2f%%)\n" % \
                         (m.split("@")[0], attempted, self.succeeded[m],
                         self.succeeded[m] * 100.0 / attempted,
                         self.failed[m],
                         self.failed[m] * 100.0 / attempted)
 
         return stat_msg
+
+    def get_summary(self):
+        msg = self._pkg_stats()
+        msg += self._maintainer_stats()
+
+        return msg
upgradehelper.py +5 -6
index f9d915a..d0e1439 100755
@@ -395,14 +395,14 @@ class Updater(object):
 
     def send_status_mail(self):
         if "status_recipients" not in settings:
-            E("Could not send status email, no recipients set!")
+            E(" Could not send status email, no recipients set!")
             return -1
 
         to_list = settings["status_recipients"].split()
 
         subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
 
-        msg = self.statistics.pkg_stats() + self.statistics.maintainer_stats()
+        msg = self.statistics.get_summary()
 
         if self.statistics.total_attempted:
             self.email_handler.send_email(to_list, subject, msg)
@@ -610,16 +610,15 @@ class Updater(object):
             pkg_ctx = pkgs_ctx[pn]
             self.pkg_upgrade_handler(pkg_ctx)
 
-        if attempted_pkgs > 1:
-            statistics_summary = self.statistics.pkg_stats() + \
-                    self.statistics.maintainer_stats()
+        if attempted_pkgs > 0:
+            statistics_summary = self.statistics.get_summary()
 
             statistics_file = os.path.join(self.uh_work_dir,
                     "statistics_summary")
             with open(statistics_file, "w+") as f:
                 f.write(statistics_summary)
 
-            I("%s" % statistics_summary)
+            I(" %s" % statistics_summary)
 
             if self.opts['send_email']:
                 self.send_status_mail()
Signature · 1 line
2.1.4
#22 lore raw

[[AUH] 17/17] statistics: Add support for publish_work_url setting.

In reply to Aníbal Limón

Publish setting helps to point the url where the AUH patches and logs are saved, this url is included into AUH upgrade status email.

---
 README                |  3 +++
 modules/statistics.py | 10 ++++++++--
 upgradehelper.py      | 12 ++++++------
 3 files changed, 17 insertions(+), 8 deletions(-)
README +3 -0
index d0cc4ae..0756fdc 100644
@@ -50,6 +50,9 @@ [email protected]
 # specify the directory where work (patches) will be saved (optional)
 #workdir=
 
+# publish url to use in statistics summary (optional)
+#publish_work_url=http://auh.somehost.com/work
+
 # clean sstate directory before upgrading
 clean_sstate=yes
 
modules/statistics.py +8 -2
index 905a4cc..db83c5b 100644
@@ -99,8 +99,14 @@ class Statistics(object):
 
         return stat_msg
 
-    def get_summary(self):
-        msg = self._pkg_stats()
+    def get_summary(self, publish_work_url, workdir):
+        msg = ''
+
+        if publish_work_url:
+            msg += "AUH finished upgrade batch the result patches/logs can be found at:\n" \
+                   "%s/%s, next are the statistics:\n\n" % (publish_work_url, workdir)
+
+        msg += self._pkg_stats()
         msg += self._maintainer_stats()
 
         return msg
upgradehelper.py +6 -6
index d0e1439..0d3f8b2 100755
@@ -393,7 +393,7 @@ class Updater(object):
             I(" %s: %s" % (pkg_ctx['PN'], e.stdout))
             raise e
 
-    def send_status_mail(self):
+    def send_status_mail(self, statistics_summary):
         if "status_recipients" not in settings:
             E(" Could not send status email, no recipients set!")
             return -1
@@ -402,10 +402,8 @@ class Updater(object):
 
         subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
 
-        msg = self.statistics.get_summary()
-
         if self.statistics.total_attempted:
-            self.email_handler.send_email(to_list, subject, msg)
+            self.email_handler.send_email(to_list, subject, statistics_summary)
         else:
             W("No recipes attempted, not sending status mail!")
 
@@ -611,7 +609,9 @@ class Updater(object):
             self.pkg_upgrade_handler(pkg_ctx)
 
         if attempted_pkgs > 0:
-            statistics_summary = self.statistics.get_summary()
+            statistics_summary = self.statistics.get_summary(
+                    settings.get('publish_work_url', 'no'),
+                    os.path.basename(self.uh_work_dir))
 
             statistics_file = os.path.join(self.uh_work_dir,
                     "statistics_summary")
@@ -621,7 +621,7 @@ class Updater(object):
             I(" %s" % statistics_summary)
 
             if self.opts['send_email']:
-                self.send_status_mail()
+                self.send_status_mail(statistics_summary)
 
 class UniverseUpdater(Updater):
     def __init__(self):
Signature · 1 line
2.1.4
#23 lore raw

[[AUH] 15/17] upgradehelper: Add workdir setting.

In reply to Aníbal Limón

The workdir setting is useful for store the AUH generated patches into another directory than $BULDDIR/upgrade-helper/work this will be used to publish AUH work.

---
 README           | 3 +++
 upgradehelper.py | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)
README +3 -0
index 6432f97..d0cc4ae 100644
@@ -47,6 +47,9 @@ [email protected]
 # who should get the status mail with statistics, at the end
 [email protected]
 
+# specify the directory where work (patches) will be saved (optional)
+#workdir=
+
 # clean sstate directory before upgrading
 clean_sstate=yes
 
upgradehelper.py +4 -1
index 6cf8dd0..f9d915a 100755
@@ -163,7 +163,10 @@ class Updater(object):
         self.uh_dir = os.path.join(build_dir, "upgrade-helper")
         if not os.path.exists(self.uh_dir):
             os.mkdir(self.uh_dir)
-        self.uh_work_dir = os.path.join(self.uh_dir, "work-%s" % \
+        uh_base_work_dir = settings.get('workdir', '')
+        if not uh_base_work_dir:
+            uh_base_work_dir = self.uh_dir
+        self.uh_work_dir = os.path.join(uh_base_work_dir, "%s" % \
                 datetime.now().strftime("%Y%m%d%H%M%S"))
         os.mkdir(self.uh_work_dir)
         self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all")
Signature · 1 line
2.1.4