Skip to contentSkip to conversation

0/6] Integration error handling and enhacements

AUH Aníbal Limón 2 participants 8 messages Dec 1, 2015 – Dec 11, 2015

lore
Share & tools
lore
#0 lore raw

This changes can be also reviewed at,

http://git.yoctoproject.org/cgit/cgit.cgi/auto-upgrade-helper/log/?h=boot_images_v3

Aníbal Limón (6):
recipe/git.py: Add missing import of re module. upgradehelper.py: Add support for preserve logs in workdir. upgradehelper.py: Add support for specify what image test. testimage/ptest: Add require of ptest-pkgs in EXTRA_IMAGE_FEATURES.
testimage: Add handling of IntegrationError
upgradehelper: Remove deprecated setting drop_previous_commits.

 README                |   7 ++-
 modules/errors.py     |   9 ++++
 modules/recipe/git.py |   1 +
 modules/testimage.py  |  80 ++++++++++++++++++++++--------
 upgradehelper.py      | 135 +++++++++++++++++++++++++++++++-------------------
 5 files changed, 156 insertions(+), 76 deletions(-)
Signature · 1 line
2.1.4
#1 lore raw

[[AUH] 4/6] testimage/ptest: Add require of ptest-pkgs in EXTRA_IMAGE_FEATURES.

Instead of add manually the ptest packages use IMAGE_FEATURE to install ptest packages this save time validating if ptest pkg exist.

---
 README               | 3 ++-
 modules/testimage.py | 5 ++---
 upgradehelper.py     | 5 +++++
 3 files changed, 9 insertions(+), 4 deletions(-)
README +2 -1
index 1c8eba1..63c3b39 100644
@@ -108,7 +108,8 @@ file:
 --------------- snip ---------------
 INHERIT += "testimage"
 
-EXTRA_IMAGE_FEATURES = "debug-tweaks package-management"
+DISTRO_FEATURES_append = " ptest"
+EXTRA_IMAGE_FEATURES = "debug-tweaks package-management ptest-pkgs"
 # testimage/ptest only work with rpm
 PACKAGE_CLASSES = "package_rpm"
 --------------- snip ---------------
modules/testimage.py +2 -3
index e410111..52e2592 100644
@@ -60,11 +60,10 @@ class TestImage():
         # for provide access to the target
         if ptest:
             pkgs_out.append("dropbear")
+            pkgs_out.append("ptest-runner")
 
         for c in pkgs:
             pkgs_out.append(c['PN'])
-            if ptest:
-                pkgs_out.append("%s-ptest" % c['PN'])
 
         return ' '.join(pkgs_out)
 
@@ -122,7 +121,7 @@ class TestImage():
         ptest_pkgs = self._get_ptest_pkgs()
 
         os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
-            self._get_pkgs_to_install(ptest_pkgs, True)
+            self._get_pkgs_to_install(ptest_pkgs, ptest=True)
         I( "   building core-image-minimal for %s ..." % machine)
         self.bb.complete("core-image-minimal", machine)
 
upgradehelper.py +5 -0
index db39601..5a2b844 100755
@@ -242,6 +242,11 @@ class Updater(object):
                       " please add to conf/local.conf.")
                     exit(1)
 
+                if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']:
+                    E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\
+                      " please add to conf/local.conf.")
+                    exit(1)
+
                 if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]:
                     E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\
                       " please add to conf/local.conf.")
Signature · 1 line
2.1.4
#2 lore raw

Re: [[AUH] 4/6] testimage/ptest: Add require of ptest-pkgs in EXTRA_IMAGE_FEATURES.

Quoted 63 lines · On Tue, 01 Dec 2015 16:03:39 Aníbal Limón wrote:

Instead of add manually the ptest packages use IMAGE_FEATURE to install ptest packages this save time validating if ptest pkg exist.

---
 README               | 3 ++-
 modules/testimage.py | 5 ++---
 upgradehelper.py     | 5 +++++
 3 files changed, 9 insertions(+), 4 deletions(-)
README +2 -1
index 1c8eba1..63c3b39 100644
@@ -108,7 +108,8 @@ file:
 --------------- snip ---------------
 INHERIT += "testimage"
 
-EXTRA_IMAGE_FEATURES = "debug-tweaks package-management"
+DISTRO_FEATURES_append = " ptest"
+EXTRA_IMAGE_FEATURES = "debug-tweaks package-management ptest-pkgs"
 # testimage/ptest only work with rpm
 PACKAGE_CLASSES = "package_rpm"
 --------------- snip ---------------
modules/testimage.py +2 -3
index e410111..52e2592 100644
@@ -60,11 +60,10 @@ class TestImage():
         # for provide access to the target
         if ptest:
             pkgs_out.append("dropbear")
+            pkgs_out.append("ptest-runner")
 
         for c in pkgs:
             pkgs_out.append(c['PN'])
-            if ptest:
-                pkgs_out.append("%s-ptest" % c['PN'])
 
         return ' '.join(pkgs_out)
 
@@ -122,7 +121,7 @@ class TestImage():
         ptest_pkgs = self._get_ptest_pkgs()
 
         os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
-            self._get_pkgs_to_install(ptest_pkgs, True)
+            self._get_pkgs_to_install(ptest_pkgs, ptest=True)
         I( "   building core-image-minimal for %s ..." % machine)
         self.bb.complete("core-image-minimal", machine)
 
upgradehelper.py +1 -0
index db39601..5a2b844 100755
@@ -242,6 +242,11 @@ class Updater(object):
                       " please add to conf/local.conf.")
                     exit(1)
 
+                if not "ptest-pkgs" in

self.base_env['EXTRA_IMAGE_FEATURES']:
+ E("
testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\

upgradehelper.py class Updater(object): · line 249

Quoted 4 lines
upgradehelper.py +3 -0
+            
         " please add to conf/local.conf.")
+                    exit(1)
+

Strictly speaking this isn't really the proper test - the proper test is that ptest-pkgs is in IMAGE_FEATURES *for the image being built and tested* (not necessarily the global configuration). We can deal with this at some point in the future though.

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre
#3 lore raw

[[AUH] 2/6] upgradehelper.py: Add support for preserve logs in workdir.

In reply to Aníbal Limón

Add FileHandler log to store the log into workdir, this needs to move up the work directory creation.

---
 upgradehelper.py | 10 ++++++++++
 1 file changed, 10 insertions(+)
upgradehelper.py +10 -0
index 0d3f8b2..7dbfbeb 100755
@@ -134,6 +134,10 @@ class Updater(object):
     def __init__(self, auto_mode=False, send_email=False, skip_compilation=False):
         build_dir = get_build_dir()
 
+        self._make_dirs(build_dir)
+
+        self._add_file_logger()
+
         self.bb = Bitbake(build_dir)
 
         try:
@@ -160,6 +164,7 @@ class Updater(object):
         self.opts['buildhistory'] = self._buildhistory_is_enabled()
         self.opts['testimage'] = self._testimage_is_enabled()
 
+    def _make_dirs(self, build_dir):
         self.uh_dir = os.path.join(build_dir, "upgrade-helper")
         if not os.path.exists(self.uh_dir):
             os.mkdir(self.uh_dir)
@@ -176,6 +181,11 @@ class Updater(object):
         self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed")
         os.mkdir(self.uh_recipes_failed_dir)
 
+    def _add_file_logger(self):
+        fh = log.FileHandler(os.path.join(self.uh_work_dir, "upgrade-helper.log"))
+        logger = log.getLogger()
+        logger.addHandler(fh)
+
     def _get_status_msg(self, err):
         if err:
             return str(err)
Signature · 1 line
2.1.4
#4 lore raw

[[AUH] 3/6] upgradehelper.py: Add support for specify what image test.

In reply to Aníbal Limón

Before this commit core-image-sato was hardcoded for image to test with this commit image name can be specified in the upgrade-helper.conf.

---
 README               |  1 +
 modules/testimage.py | 10 +++++-----
 upgradehelper.py     | 13 ++++++++-----
 3 files changed, 14 insertions(+), 10 deletions(-)
README +1 -0
index 0756fdc..1c8eba1 100644
@@ -68,6 +68,7 @@ machines=qemux86 qemux86-64 qemuarm qemumips qemuppc
 # optional features
 buildhistory=no
 testimage=no
+testimage_name=image-custom # defaults to core-image-sato
 
 --------------- snip ---------------
 
modules/testimage.py +5 -5
index 3fbbc19..e410111 100644
@@ -153,18 +153,18 @@ class TestImage():
                         f.write(line)
                     f.write("END: PTEST for %s\n" % machine)
 
-    def sato(self, machine):
+    def testimage(self, machine, image):
         os.environ['CORE_IMAGE_EXTRA_INSTALL'] = \
             self._get_pkgs_to_install(self.pkgs_ctx)
 
         if 'TEST_SUITES' in os.environ:
             del os.environ['TEST_SUITES']
 
-        I( "   building core-image-sato for %s ..." % machine)
-        self.bb.complete("core-image-sato", machine)
+        I( "   building %s for %s ..." % (image, machine))
+        self.bb.complete(image, machine)
 
-        I( "   running core-image-sato/testimage for %s ..." % machine)
-        self.bb.complete("core-image-sato -c testimage", machine)
+        I( "   running %s/testimage for %s ..." % (image, machine))
+        self.bb.complete("%s -c testimage" % image, machine)
 
         log_file = self._find_log("log.do_testimage", machine)
         shutil.copyfile(log_file,
upgradehelper.py +8 -5
index 7dbfbeb..db39601 100755
@@ -68,6 +68,7 @@ help_text = """Usage examples:
     $ upgrade-helper.py all
 """
 
+DEFAULT_TESTIMAGE = 'core-image-sato'
 
 def parse_cmdline():
     parser = argparse.ArgumentParser(description='Package Upgrade Helper',
@@ -314,8 +315,8 @@ class Updater(object):
             "The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \
             "with the next machines %s. Attached is the log file.\n\n"
 
-        testimage_sato_info = \
-            "The recipe has been tested using core-image-sato testimage and succeeded with \n" \
+        testimage_info = \
+            "The recipe has been tested using %s testimage and succeeded with \n" \
             "the next machines %s. Attached is the log file.\n\n" \
 
         mail_footer = \
@@ -353,7 +354,8 @@ class Updater(object):
                 msg_body += testimage_ptest_info % machines
             if 'testimage' in pkg_ctx:
                 machines = pkg_ctx['testimage'].keys()
-                msg_body += testimage_sato_info % machines
+                msg_body += testimage_info % (settings.get('testimage_name', \
+                    DEFAULT_TESTIMAGE), machines)
 
         msg_body += mail_footer
 
@@ -602,10 +604,11 @@ class Updater(object):
                             import traceback
                             traceback.print_exc(file=sys.stdout)
 
+                    image = settings.get('testimage_name', DEFAULT_TESTIMAGE)
                     try:
-                        tim.sato(machine)
+                        tim.testimage(machine, image)
                     except Exception as e:
-                        E(" core-image-sato/testimage on machine %s failed" % machine)
+                        E(" %s/testimage on machine %s failed" % (image, machine))
                         if isinstance(e, Error):
                             E(" %s" % e.stdout)
                         else:
Signature · 1 line
2.1.4
#5 lore raw

[[AUH] 5/6] testimage: Add handling of IntegrationError

In reply to Aníbal Limón

A recipe can compile alone when the upgrade proccess is made but when integrate it can fail due to dependencies of another recipes.

If build image fail then analyze logs to find what recipe failed if recipe is found directly on the set of recipe upgrades remove it, if not recipe found directly then search within dependencies of failed recipe.

Since IntegrationError can happen now, move the statistics update and directory symlink creation after testimage.

---
 modules/errors.py    |  9 ++++++
 modules/testimage.py | 69 +++++++++++++++++++++++++++++++---------
 upgradehelper.py     | 89 +++++++++++++++++++++++++++++++++++-----------------
 3 files changed, 125 insertions(+), 42 deletions(-)
modules/errors.py +9 -0
index 1504fa5..9f8f37f 100644
@@ -91,3 +91,12 @@ class EmptyEnvError(Error):
 
     def __str__(self):
         return "Failed(get_env)"
+
+class IntegrationError(Error):
+    def __init__(self, stdout, pkg_ctx):
+        super(IntegrationError, self).__init__("Failed to build %s in testimage branch"
+                % pkg_ctx['PN'], stdout)
+        self.pkg_ctx = pkg_ctx
+
+        def __str__(self):
+            return "Failed(integrate)"
modules/testimage.py +55 -14

131 lines not on this page · raw message

upgradehelper.py +61 -28

123 lines not on this page · raw message

Signature · 1 line
2.1.4
#6 lore raw

[[AUH] 6/6] upgradehelper: Remove deprecated setting drop_previous_commits.

In reply to Aníbal Limón

Since recipe upgrades are done in isolate manner cleaning the repo for every recipe [1] the drop_previous_commits becomes deprecated.

Also now we have Testimage integration phase that handles/reports this kind of errors.

[1] http://git.yoctoproject.org/cgit/cgit.cgi/auto-upgrade-helper/commit/?h=boot_images_v2&id=8769030a02753f09ed6b5e7c98ed1c442a51a466

---
 README           |  3 ---
 upgradehelper.py | 20 --------------------
 2 files changed, 23 deletions(-)
README +0 -3
index 63c3b39..1b73034 100644
@@ -59,9 +59,6 @@ clean_sstate=yes
 # clean tmp directory before upgrading
 clean_tmp=yes
 
-# keep previous commits or not
-drop_previous_commits=yes
-
 # machines to test build with
 machines=qemux86 qemux86-64 qemuarm qemumips qemuppc
 
upgradehelper.py +0 -20
index 19327c8..d9fa48a 100755
@@ -276,26 +276,6 @@ class Updater(object):
 
     # this function will be called at the end of each recipe upgrade
     def pkg_upgrade_handler(self, pkg_ctx):
-        if self.opts['interactive'] and pkg_ctx['error'] and pkg_ctx['patch_file']:
-            answer = "N"
-            I(" %s: Do you want to keep the changes? (y/N)" % pkg_ctx['PN'])
-            answer = sys.stdin.readline().strip().upper()
-
-            if answer == '' or answer == 'N':
-                I(" %s: Dropping changes from git ..." % pkg_ctx['PN'])
-                self.git.reset_hard(1)
-                self.git.clean_untracked()
-                return
-
-        # drop last upgrade from git. It's safer this way if the upgrade has
-        # problems and other recipes depend on it. Give the other recipes a
-        # chance...
-        if (settings.get("drop_previous_commits", "no") == "yes" and
-                not pkg_ctx['error']) or (pkg_ctx['error'] and pkg_ctx['patch_file']):
-            I(" %s: Dropping changes from git ..." % pkg_ctx['PN'])
-            self.git.reset_hard(1)
-            self.git.clean_untracked()
-
         mail_header = \
             "Hello,\n\nYou are receiving this email because you are the maintainer\n" \
             "of *%s* recipe and this is to let you know that the automatic attempt\n" \
Signature · 1 line
2.1.4
#7 lore raw

[[AUH] 1/6] recipe/git.py: Add missing import of re module.

In reply to Aníbal Limón
---
 modules/recipe/git.py | 1 +
 1 file changed, 1 insertion(+)
modules/recipe/git.py +1 -0
index 7526a67..b1a6201 100644
@@ -23,6 +23,7 @@
 #
 
 from recipe.base import Recipe
+import re
 
 class GitRecipe(Recipe):
     def _extract_tag_from_ver(self, ver):
Signature · 1 line
2.1.4