buildhistory analysis

new file mode 100644
index 0000000..9f42fe3
@@ -0,0 +1,240 @@
+# Report significant differences in the buildhistory repository since a specific revision
+#
+# Copyright (C) 2012 Intel Corporation
+# Author: Paul Eggleton <[email protected]>
+#
+# Note: requires GitPython 0.3.1+
+#
+# You can use this from the command line by running scripts/buildhistory-diff
+#
+
+import sys
+import os.path
+import difflib
+import git
+
+
+# How to display fields
+pkg_list_fields = ['DEPENDS', 'RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILES', 'FILELIST']
+pkg_numeric_fields = ['PKGSIZE']
+# Fields to monitor
+pkg_monitor_fields = ['RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILELIST', 'PKGSIZE']
+# Percentage change to alert for numeric fields
+pkg_monitor_numeric_threshold = 20
+# Image files to monitor
+img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
+
+
+class ChangeRecord:
+    def __init__(self, path, fieldname, oldvalue, newvalue):
+        self.path = path
+        self.fieldname = fieldname
+        self.oldvalue = oldvalue
+        self.newvalue = newvalue
+        self.filechanges = None
+
+    def __str__(self):
+        if self.fieldname in pkg_list_fields:
+            aitems = self.oldvalue.split(' ')
+            bitems = self.newvalue.split(' ')
+            removed = list(set(aitems) - set(bitems))
+            added = list(set(bitems) - set(aitems))
+            return '%s: %s:%s%s' % (self.path, self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '')
+        elif self.fieldname in pkg_numeric_fields:
+            aval = int(self.oldvalue)
+            bval = int(self.newvalue)
+            percentchg = ((bval - aval) / float(aval)) * 100
+            return '%s: %s changed from %d to %d (%s%d%%)' % (self.path, self.fieldname, aval, bval, '+' if percentchg > 0 else '', percentchg)
+        elif self.fieldname in img_monitor_files:
+            out = 'Changes to %s (%s):\n  ' % (self.path, self.fieldname)
+            if self.filechanges:
+                out += '\n  '.join(['%s' % i for i in self.filechanges])
+            else:
+                alines = self.oldvalue.splitlines()
+                blines = self.newvalue.splitlines()
+                diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
+                out += '\n  '.join(list(diff))
+                out += '\n  --'
+            return out
+        else:
+            return '%s: %s changed from "%s" to "%s"' % (self.path, self.self.fieldname, self.oldvalue, self.newvalue)
+
+
+class FileChange:
+    changetype_add = 'A'
+    changetype_remove = 'R'
+    changetype_type = 'T'
+    changetype_perms = 'P'
+    changetype_ownergroup = 'O'
+    changetype_link = 'L'
+
+    def __init__(self, path, changetype, oldvalue = None, newvalue = None):
+        self.path = path
+        self.changetype = changetype
+        self.oldvalue = oldvalue
+        self.newvalue = newvalue
+
+    def _ftype_str(self, ftype):
+        if ftype == '-':
+            return 'file'
+        elif ftype == 'd':
+            return 'directory'
+        elif ftype == 'l':
+            return 'symlink'
+        elif ftype == 'c':
+            return 'char device'
+        elif ftype == 'b':
+            return 'block device'
+        elif ftype == 'p':
+            return 'fifo'
+        elif ftype == 's':
+            return 'socket'
+        else:
+            return 'unknown (%s)' % ftype
+
+    def __str__(self):
+        if self.changetype == self.changetype_add:
+            return '%s was added' % self.path
+        elif self.changetype == self.changetype_remove:
+            return '%s was removed' % self.path
+        elif self.changetype == self.changetype_type:
+            return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
+        elif self.changetype == self.changetype_perms:
+            return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
+        elif self.changetype == self.changetype_ownergroup:
+            return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
+        elif self.changetype == self.changetype_link:
+            return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
+        else:
+            return '%s changed (unknown)' % self.path
+
+
+def blob_to_dict(blob):
+    alines = blob.data_stream.read().splitlines()
+    adict = {}
+    for line in alines:
+        splitv = [i.strip() for i in line.split('=',1)]
+        if splitv.count > 1:
+            adict[splitv[0]] = splitv[1]
+    return adict
+
+
+def file_list_to_dict(lines):
+    adict = {}
+    for line in lines:
+        # Leave the last few fields intact so we handle file names containing spaces
+        splitv = line.split(None,4)
+        # Grab the path and remove the leading .
+        path = splitv[4][1:].strip()
+        # Handle symlinks
+        if(' -> ' in path):
+            target = path.split(' -> ')[1]
+            path = path.split(' -> ')[0]
+            adict[path] = splitv[0:3] + [target]
+        else:
+            adict[path] = splitv[0:3]
+    return adict
+
+
+def compare_file_lists(alines, blines):
+    adict = file_list_to_dict(alines)
+    bdict = file_list_to_dict(blines)
+    filechanges = []
+    for path, splitv in adict.iteritems():
+        newsplitv = bdict.pop(path, None)
+        if newsplitv:
+            # Check type
+            oldvalue = splitv[0][0]
+            newvalue = newsplitv[0][0]
+            if oldvalue != newvalue:
+                filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
+            # Check permissions
+            oldvalue = splitv[0][1:]
+            newvalue = newsplitv[0][1:]
+            if oldvalue != newvalue:
+                filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
+            # Check owner/group
+            oldvalue = '%s/%s' % (splitv[1], splitv[2])
+            newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
+            if oldvalue != newvalue:
+                filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
+            # Check symlink target
+            if newsplitv[0][0] == 'l':
+                if splitv.count > 3:
+                    oldvalue = splitv[3]
+                else:
+                    oldvalue = None
+                newvalue = newsplitv[3]
+                if oldvalue != newvalue:
+                    filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
+        else:
+            filechanges.append(FileChange(path, FileChange.changetype_remove))
+
+    # Whatever is left over has been added
+    for path in bdict:
+        filechanges.append(FileChange(path, FileChange.changetype_add))
+
+    return filechanges
+
+
+def compare_lists(alines, blines):
+    removed = list(set(alines) - set(blines))
+    added = list(set(blines) - set(alines))
+
+    filechanges = []
+    for pkg in removed:
+        filechanges.append(FileChange(pkg, FileChange.changetype_remove))
+    for pkg in added:
+        filechanges.append(FileChange(pkg, FileChange.changetype_add))
+
+    return filechanges
+
+
+def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False):
+    repo = git.Repo(repopath)
+    assert repo.bare == False
+    commit = repo.commit(revision1)
+    diff = commit.diff(revision2)
+
+    changes = []
+    for d in diff.iter_change_type('M'):
+        path = os.path.dirname(d.a_blob.path)
+        if path.startswith('packages/'):
+            adict = blob_to_dict(d.a_blob)
+            bdict = blob_to_dict(d.b_blob)
+
+            for key in adict:
+                if report_all or key in pkg_monitor_fields:
+                    if adict[key] != bdict[key]:
+                        if (not report_all) and key in pkg_numeric_fields:
+                            aval = int(adict[key])
+                            bval = int(bdict[key])
+                            percentchg = ((bval - aval) / float(aval)) * 100
+                            if percentchg < pkg_monitor_numeric_threshold:
+                                continue
+                        chg = ChangeRecord(path, key, adict[key], bdict[key])
+                        changes.append(chg)
+        elif path.startswith('images/'):
+            filename = os.path.basename(d.a_blob.path)
+            if filename in img_monitor_files:
+                if filename == 'files-in-image.txt':
+                    alines = d.a_blob.data_stream.read().splitlines()
+                    blines = d.b_blob.data_stream.read().splitlines()
+                    filechanges = compare_file_lists(alines,blines)
+                    if filechanges:
+                        chg = ChangeRecord(path, filename, None, None)
+                        chg.filechanges = filechanges
+                        changes.append(chg)
+                elif filename == 'installed-package-names.txt':
+                    alines = d.a_blob.data_stream.read().splitlines()
+                    blines = d.b_blob.data_stream.read().splitlines()
+                    filechanges = compare_lists(alines,blines)
+                    if filechanges:
+                        chg = ChangeRecord(path, filename, None, None)
+                        chg.filechanges = filechanges
+                        changes.append(chg)
+                else:
+                    chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read())
+                    changes.append(chg)
+
+    return changes

43 lines not on this page · raw message

index 06d3510..ba20914 100644
@@ -356,7 +356,8 @@ buildhistory_commit() {
 		repostatus=`git status --porcelain`
 		if [ "$repostatus" != "" ] ; then
 			git add ${BUILDHISTORY_DIR}/*
-			git commit ${BUILDHISTORY_DIR}/ -m "Build ${BUILDNAME} for machine ${MACHINE} configured for ${DISTRO} ${DISTRO_VERSION}" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
+			HOSTNAME=`cat /etc/hostname 2>/dev/null || echo unknown`
+			git commit ${BUILDHISTORY_DIR}/ -m "Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $HOSTNAME" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
 			if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
 				git push -q ${BUILDHISTORY_PUSH_REPO}
 			fi

index ba20914..f618903 100644
@@ -158,7 +158,7 @@ python buildhistory_emit_pkghistory() {
 			last_pr = lastversion.pr
 			r = bb.utils.vercmp((pe, pv, pr), (last_pe, last_pv, last_pr))
 			if r < 0:
-				bb.fatal("Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pe, last_pv, last_pr, pe, pv, pr))
+				bb.error("Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pe, last_pv, last_pr, pe, pv, pr))
 
 		pkginfo = PackageInfo(pkg)
 		pkginfo.pe = pe

index f618903..34cc297 100644
@@ -182,21 +182,10 @@ python buildhistory_emit_pkghistory() {
 
 		write_pkghistory(pkginfo, d)
 
-		if lastversion:
-			check_pkghistory(pkginfo, lastversion)
-
 		write_latestlink(pkg, pe, pv, pr, d)
 }
 
 
-def check_pkghistory(pkginfo, lastversion):
-
-	bb.debug(2, "Checking package history")
-	# RDEPENDS removed?
-	# PKG changed?
-	# Each file list of each package for file removals?
-
-
 def write_recipehistory(rcpinfo, d):
 	bb.debug(2, "Writing recipe history")