Skip to contentSkip to conversation

AUH: buildhistory-diff support

PATCH In review Marius Avram 2 participants 9 messages Aug 13, 2014 – Aug 14, 2014

lore
Share & tools
lore
#0 lore raw

cover AUH: buildhistory-diff support

The maintainer will received the output given by buildhistory-diff. If no diff is found the attachement will not be created.

A branch with the changes included in this patcheset can also be found here:

http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=mavram/buildhistory-diff

Marius Avram (7):
Add buildhistory-diff as attachment to email Fixed bug of selected attachements for email Added error message when buildhistory not active
README: Added buildhistory configuration Info messages start with upper case
Fix adding new files to recipe bug
Add missing import

 README           |    8 ++++++++
 bitbake.py       |   36 ++++++++++++++++++++++++++++++++++++
 recipe.py        |   11 ++++++++---
 upgradehelper.py |   29 ++++++++++++++++++-----------
 4 files changed, 70 insertions(+), 14 deletions(-)
Signature · 1 line
1.7.9.5
#1 lore raw

PATCH 1/7 Add buildhistory-diff as attachment to email

buildhistory-diff is useful in the case new files have been added to the package.

---
 bitbake.py       |   28 ++++++++++++++++++++++++++++
 recipe.py        |    3 +--
 upgradehelper.py |   21 ++++++++++++++-------
 3 files changed, 43 insertions(+), 9 deletions(-)
bitbake.py +28 -0
index e56ee6d..2207634 100644
@@ -24,6 +24,7 @@
 
 import os
 import logging as log
+from logging import info as I
 from logging import debug as D
 from logging import error as E
 import sys
@@ -97,3 +98,30 @@ class Bitbake(object):
 
     def dependency_graph(self, package_list):
         return self._cmd(package_list, "-g")
+
+class BuildHistory(object):
+    def __init__(self, build_dir):
+        self.build_dir = build_dir
+        self.work_dir = None
+
+    def set_work_dir(self, work_dir):
+        self.work_dir = work_dir
+
+    # Return True if buildhistory-diff gives output
+    def diff(self, revision_steps):
+        os.chdir(self.build_dir)
+        cmd = "buildhistory-diff HEAD~" + str(revision_steps)
+
+        try:
+            stdout, stderr = bb.process.run(cmd)
+            # Write diff output to log file if there is any
+            if stdout and os.path.exists(self.work_dir):
+                with open(os.path.join(self.work_dir, "buildhistory.txt"), "w+") as log:
+                    log.write(stdout)
+                return True
+        except bb.process.ExecutionError as e:
+            D("%s returned:\n%s" % (cmd, e.__str__()))
+            raise Error("\'" + cmd + "\' failed", e.stdout, e.stderr)
+
+        return False
+
recipe.py +1 -2
index cbb8ded..a495b49 100644
@@ -408,7 +408,7 @@ class Recipe(object):
                 failed_tasks[task_log_match.group(2)] = (task_log_match.group(3), task_log_match.group(1))
             elif qa_issue_match:
                 # Improvise path to log file
-                failed_tasks[qa_issue_match.group(1)] = ("do_package", self.bb.get_stdout_log()) 
+                failed_tasks[qa_issue_match.group(1)] = ("do_package", self.bb.get_stdout_log())
             elif machine_match:
                 machine = machine_match.group(1)
 
@@ -647,4 +647,3 @@ class Recipe(object):
                     # throw a compilation exception for everything else. It
                     # doesn't really matter
                     raise CompilationError()
-
upgradehelper.py +14 -7
index 006dd0b..091881f 100755
@@ -123,6 +123,7 @@ class Updater(object):
             os.mkdir(self.uh_dir)
 
         self.bb = Bitbake(get_build_dir())
+        self.buildhistory = BuildHistory(get_build_dir())
         self.git = None
 
         self.author = None
@@ -253,6 +254,14 @@ class Updater(object):
             I(" %s: compiling for %s ..." % (self.pn, machine))
             self.recipe.compile(machine)
 
+    def _review(self):
+        # Check build_history
+        if not self.skip_compilation:
+            I(" %s: Checking buildhistory ..." % self.pn)
+            self.buildhistory.set_work_dir(self.workdir)
+            if self.buildhistory.diff(len(self.machines)):
+               I(" %s: Wrote buildhistory-diff output ..." % self.pn)
+
     def _check_upstream_versions(self, packages=[("universe", None, None)]):
         I(" Fetching upstream version(s) ...")
 
@@ -605,14 +614,12 @@ class UniverseUpdater(Updater, Email):
 
         msg_body += self.mail_footer
 
+        # Add possible attachments to list
         attachments = []
-        if self.patch_file is not None:
-            attachments.append(self.patch_file)
-        # License issue
-        if status == "LicenseError":
-            attachments.append(self.recipe.license_diff_file)
-        elif err is not None:
-            attachments.append(os.path.join(self.workdir, "bitbake_log.txt"))
+        for attachment in os.listdir(self.workdir):
+            attachment_fullpath = os.join(self.workdir, attachment)
+            if isfile(attachment_fullpath):
+                attachments.append(attachment_fullpath)
 
         self.send_email(to_addr, subject, msg_body, attachments)
 
Signature · 1 line
1.7.9.5
#2 lore raw

PATCH 2/7 Fixed bug of selected attachements for email

In reply to Marius Avram
---
 upgradehelper.py |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
upgradehelper.py +2 -2
index 091881f..ef59661 100755
@@ -617,8 +617,8 @@ class UniverseUpdater(Updater, Email):
         # Add possible attachments to list
         attachments = []
         for attachment in os.listdir(self.workdir):
-            attachment_fullpath = os.join(self.workdir, attachment)
-            if isfile(attachment_fullpath):
+            attachment_fullpath = os.path.join(self.workdir, attachment)
+            if os.path.isfile(attachment_fullpath):
                 attachments.append(attachment_fullpath)
 
         self.send_email(to_addr, subject, msg_body, attachments)
Signature · 1 line
1.7.9.5
#3 lore raw

PATCH 3/7 Added error message when buildhistory not active

In reply to Marius Avram

A graceful close of the script happens when it tries to do a buildhistory-diff but the bbclass is not inherited. Also a helping message is presented to the user to avoid the error at a next run.

---
 bitbake.py |   12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
bitbake.py +10 -2
index 2207634..3631896 100644
@@ -27,6 +27,7 @@ import logging as log
 from logging import info as I
 from logging import debug as D
 from logging import error as E
+from logging import critical as C
 import sys
 from errors import *
 
@@ -115,13 +116,20 @@ class BuildHistory(object):
         try:
             stdout, stderr = bb.process.run(cmd)
             # Write diff output to log file if there is any
+
             if stdout and os.path.exists(self.work_dir):
                 with open(os.path.join(self.work_dir, "buildhistory.txt"), "w+") as log:
                     log.write(stdout)
                 return True
         except bb.process.ExecutionError as e:
-            D("%s returned:\n%s" % (cmd, e.__str__()))
-            raise Error("\'" + cmd + "\' failed", e.stdout, e.stderr)
+            for line in e.stdout.split('\n'):
+                if line.find("Buildhistory directory \"buildhistory/\" does not exist") == 0:
+                    C(" \"buildhistory.bbclass\" not inherited. Consider adding "
+                      "the following to your local.conf:\n\n"
+                      "INHERIT =+ \"buildhistory\"\n"
+                      "BUILDHISTORY_COMMIT = \"1\"\n\n"
+                      "Do not remove any other inherited class in the process (e.g. distrodata)\n")
+                    exit(1)
 
         return False
 
Signature · 1 line
1.7.9.5
#4 lore raw

PATCH 4/7 README: Added buildhistory configuration

In reply to Marius Avram
---
 README |    8 ++++++++
 1 file changed, 8 insertions(+)
README +8 -0
index 5721d58..43e6800 100644
@@ -54,6 +54,14 @@ require conf/distro/include/maintainers.inc
 require conf/distro/include/upstream_tracking.inc
 --------------- snip ---------------
 
+4. Enable buildhistory by adding the following lines in the in your
+conf/local.conf file:
+
+INHERIT =+ "buildhistory"
+BUILDHISTORY_COMMIT" = "1"
+
+Do not remove any other inherited class in the process (e.g. distrodata).
+
 WARNING: if you are using the default maintainers.inc file supplied
          with Poky (in meta-yocto) and you don't set a
          maintainers_whitelist or maintainer_override in the
Signature · 1 line
1.7.9.5
#5 lore raw

PATCH 5/7 Info messages start with upper case

In reply to Marius Avram

For consistency sake I modified all the messages to start with upper case.

---
 recipe.py        |    2 +-
 upgradehelper.py |    6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
recipe.py +1 -1
index a495b49..e6430cd 100644
@@ -388,7 +388,7 @@ class Recipe(object):
                 W(" %s: license checksum failed for file %s."
                   " The recipe has been updated! Diff file located at %s" %
                   (self.env['PN'], license_file, self.license_diff_file))
-                I(" recompiling ...")
+                I(" Recompiling ...")
                 self.commit_msg += "License checksum changed for file " + license_file
                 return True
 
upgradehelper.py +3 -3
index ef59661..27de760 100755
@@ -367,20 +367,20 @@ class Updater(object):
             attempted_pkgs += 1
             I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs))
             try:
-                I(" %s: upgrading to %s" % (self.pn, self.new_ver))
+                I(" %s: Upgrading to %s" % (self.pn, self.new_ver))
                 for step, msg in self.upgrade_steps:
                     if msg is not None:
                         I(" %s: %s" % (self.pn, msg))
                     step()
 
-                I(" %s: upgrade SUCCESSFUL! Please test!" % self.pn)
+                I(" %s: Upgrade SUCCESSFUL! Please test!" % self.pn)
                 error = None
             except UpgradeNotNeededError as e:
                 I(" %s: %s" % (self.pn, e.message))
                 error = e
             except Error as e:
                 E(" %s: %s" % (self.pn, e.message))
-                E(" %s: upgrade FAILED! Logs and/or file diffs are available in %s" % (self.pn, self.workdir))
+                E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" % (self.pn, self.workdir))
                 error = e
 
             self._commit_changes()
Signature · 1 line
1.7.9.5
#6 lore raw

PATCH 6/7 Fix adding new files to recipe bug

In reply to Marius Avram

The upgrade failed when new files were added to the recipe and the compilation was made for multiple machines. Now this problem has been fixed.

---
 recipe.py |    6 ++++++
 1 file changed, 6 insertions(+)
recipe.py +6 -0
index e6430cd..8ecd2a2 100644
@@ -452,11 +452,15 @@ class Recipe(object):
           "/usr/lib/opie"   : "palmqtdir",
         }
 
+        I(" %s: Add new files in recipe ..." %  self.env['PN'])
         with open(package_log) as log:
             for line in log:
                 if re.match(".*Files/directories were installed but not shipped.*", line):
                     files_not_shipped = True
+                # Extract path
                 line = line.strip()
+                if line:
+                    line = line.split()[0]
                 if files_not_shipped and os.path.isabs(line):
                     # Count occurences for globbing
                     path_exists = False
@@ -587,6 +591,7 @@ class Recipe(object):
             self.git.delete_branch("remove_patches")
             self.git.reset_hard()
             self.git.reset_soft(1)
+            self.removed_patches = False
 
     def compile(self, machine):
         try:
@@ -597,6 +602,7 @@ class Recipe(object):
                 self.git.delete_branch("remove_patches")
                 self.git.reset_soft(1)
                 self.commit_msg += self.rm_patches_msg + "\n"
+                self.removed_patches = False
         except Error as e:
             if self._is_incompatible_host(e.stdout):
                 W(" %s: compilation failed: incompatible host" % self.env['PN'])
Signature · 1 line
1.7.9.5
#7 lore raw

PATCH 7/7 Add missing import

In reply to Marius Avram
---
 upgradehelper.py |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
upgradehelper.py +1 -1
index 27de760..cc90958 100755
@@ -42,7 +42,7 @@ from datetime import date
 import shutil
 from errors import *
 from git import Git
-from bitbake import Bitbake
+from bitbake import Bitbake, BuildHistory
 from emailhandler import Email
 from statistics import Statistics
 from recipe import Recipe
Signature · 1 line
1.7.9.5
#8 lore raw
In reply to Marius Avram
Quoted 10 lines · On Wednesday 13 August 2014 16:32:01 Marius Avram wrote:

The maintainer will received the output given by buildhistory-diff. If no diff is found the attachement will not be created.

A branch with the changes included in this patcheset can also be found here:

http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=mavram/buildhi
story-diff

Quoted 8 lines

Marius Avram (7):
Add buildhistory-diff as attachment to email Fixed bug of selected attachements for email Added error message when buildhistory not active
README: Added buildhistory configuration Info messages start with upper case
Fix adding new files to recipe bug
Add missing import

Thanks Marius - I've merged these, though I squashed the last commit into the first and tweaked the README addition slightly.

Cheers,
Paul

Signature · 2 lines
Paul Eggleton
Intel Open Source Technology Centre