Add optional task signatures to buildhistory

index d8ca12d..e03ccc5 100755
@@ -46,6 +46,11 @@ def main():
         sys.exit(1)
 
     if not os.path.exists(options.buildhistory_dir):
+        if options.buildhistory_dir == 'buildhistory/':
+            cwd = os.getcwd()
+            if os.path.basename(cwd) == 'buildhistory':
+                options.buildhistory_dir = cwd
+    if not os.path.exists(options.buildhistory_dir):
         sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % options.buildhistory_dir)
         parser.print_help()
         sys.exit(1)

index 109b375..0cafad5 100644
@@ -596,6 +596,17 @@ POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_insta
 SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
 SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
 
+python buildhistory_write_sigs() {
+    if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
+        return
+
+    # Create sigs file
+    if hasattr(bb.parse.siggen, 'dump_siglist'):
+        taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task')
+        bb.utils.mkdirhier(taskoutdir)
+        bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'))
+}
+
 def buildhistory_get_build_id(d):
     if d.getVar('BB_WORKERCONTEXT') != '1':
         return ""

@@ -761,6 +772,7 @@ python buildhistory_eventhandler() {
                 shutil.rmtree(olddir)
             if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
                 bb.note("Writing buildhistory")
+                bb.build.exec_func("buildhistory_write_sigs", d)
                 localdata = bb.data.createCopy(e.data)
                 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
                 interrupted = getattr(e, '_interrupted', 0)

index 13fd3bd..56b33ba 100644
@@ -211,6 +211,16 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
                 f.write('    "\n')
             f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(l)))
 
+    def dump_siglist(self, sigfile):
+        with open(sigfile, "w") as f:
+            tasks = []
+            for taskitem in self.taskhash:
+                (fn, task) = taskitem.rsplit(".", 1)
+                pn = self.lockedpnmap[fn]
+                tasks.append((pn, task, self.taskhash[taskitem]))
+            for (pn, task, taskhash) in sorted(tasks):
+                f.write('%s.%s %s\n' % (pn, task, taskhash))
+
     def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
         warn_msgs = []
         error_msgs = []

index 19b3bc4..449446f 100644
@@ -1,6 +1,6 @@
 # Report significant differences in the buildhistory repository since a specific revision
 #
-# Copyright (C) 2012 Intel Corporation
+# Copyright (C) 2012-2013, 2016-2017 Intel Corporation
 # Author: Paul Eggleton <[email protected]>
 #
 # Note: requires GitPython 0.3.1+

@@ -410,13 +410,58 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
     return changes
 
 
-def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False):
+def compare_siglists(a_blob, b_blob):
+    # FIXME collapse down a recipe's tasks?
+    alines = a_blob.data_stream.read().decode('utf-8').splitlines()
+    blines = b_blob.data_stream.read().decode('utf-8').splitlines()
+    keys = []
+    pnmap = {}
+    def readsigs(lines):
+        sigs = {}
+        for line in lines:
+            linesplit = line.split()
+            if len(linesplit) > 2:
+                sigs[linesplit[0]] = linesplit[2]
+                if not linesplit[0] in keys:
+                    keys.append(linesplit[0])
+                pnmap[linesplit[1]] = linesplit[0].rsplit('.', 1)[0]
+        return sigs
+    adict = readsigs(alines)
+    bdict = readsigs(blines)
+    out = []
+    changecount = 0
+    addcount = 0
+    removecount = 0
+    for key in keys:
+        siga = adict.get(key, None)
+        sigb = bdict.get(key, None)
+        if siga is not None and sigb is not None and siga != sigb:
+            out.append('%s changed from %s to %s' % (key, siga, sigb))
+            changecount += 1
+        elif siga is None:
+            out.append('%s was added' % key)
+            addcount += 1
+        elif sigb is None:
+            removecount += 1
+            out.append('%s was removed' % key)
+    out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100)))
+    return '\n'.join(out)
+
+
+def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False):
     repo = git.Repo(repopath)
     assert repo.bare == False
     commit = repo.commit(revision1)
     diff = commit.diff(revision2)
 
     changes = []
+
+    if sigs:
+        for d in diff.iter_change_type('M'):
+            if d.a_blob.path == 'siglist.txt':
+                changes.append(compare_siglists(d.a_blob, d.b_blob))
+        return changes
+
     for d in diff.iter_change_type('M'):
         path = os.path.dirname(d.a_blob.path)
         if path.startswith('packages/'):

index 56b33ba..d5377db 100644
@@ -217,9 +217,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
             for taskitem in self.taskhash:
                 (fn, task) = taskitem.rsplit(".", 1)
                 pn = self.lockedpnmap[fn]
-                tasks.append((pn, task, self.taskhash[taskitem]))
-            for (pn, task, taskhash) in sorted(tasks):
-                f.write('%s.%s %s\n' % (pn, task, taskhash))
+                tasks.append((pn, task, fn, self.taskhash[taskitem]))
+            for (pn, task, fn, taskhash) in sorted(tasks):
+                f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash))
 
     def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
         warn_msgs = []

index e03ccc5..e8e3e11 100755
@@ -33,6 +33,9 @@ def main():
     parser.add_option("-a", "--report-all",
             help = "Report all changes, not just the default significant ones",
             action="store_true", dest="report_all", default=False)
+    parser.add_option("-s", "--signatures",
+            help = "Report on signature differences instead of output",
+            action="store_true", dest="sigs", default=False)
 
     options, args = parser.parse_args(sys.argv)
 

@@ -86,7 +89,7 @@ def main():
 
     import gitdb
     try:
-        changes = oe.buildhistory_analysis.process_changes(options.buildhistory_dir, fromrev, torev, options.report_all, options.report_ver)
+        changes = oe.buildhistory_analysis.process_changes(options.buildhistory_dir, fromrev, torev, options.report_all, options.report_ver, options.sigs)
     except gitdb.exc.BadObject as e:
         if len(args) == 1:
             sys.stderr.write("Unable to find previous build revision in buildhistory repository\n\n")

index 449446f..3a5b7b6 100644
@@ -13,7 +13,10 @@ import os.path
 import difflib
 import git
 import re
+import hashlib
+import collections
 import bb.utils
+import bb.tinfoil
 
 
 # How to display fields

@@ -410,7 +413,7 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
     return changes
 
 
-def compare_siglists(a_blob, b_blob):
+def compare_siglists(a_blob, b_blob, taskdiff=False):
     # FIXME collapse down a recipe's tasks?
     alines = a_blob.data_stream.read().decode('utf-8').splitlines()
     blines = b_blob.data_stream.read().decode('utf-8').splitlines()

@@ -429,26 +432,83 @@ def compare_siglists(a_blob, b_blob):
     adict = readsigs(alines)
     bdict = readsigs(blines)
     out = []
+
     changecount = 0
     addcount = 0
     removecount = 0
-    for key in keys:
-        siga = adict.get(key, None)
-        sigb = bdict.get(key, None)
-        if siga is not None and sigb is not None and siga != sigb:
-            out.append('%s changed from %s to %s' % (key, siga, sigb))
-            changecount += 1
-        elif siga is None:
-            out.append('%s was added' % key)
-            addcount += 1
-        elif sigb is None:
-            removecount += 1
-            out.append('%s was removed' % key)
+    if taskdiff:
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            tinfoil.prepare(config_only=True)
+
+            changes = collections.OrderedDict()
+
+            def compare_hashfiles(pn, taskname, hash1, hash2):
+                hashes = [hash1, hash2]
+                hashfiles = bb.siggen.find_siginfo(pn, taskname, hashes, tinfoil.config_data)
+
+                if not taskname:
+                    (pn, taskname) = pn.rsplit('.', 1)
+                    pn = pnmap.get(pn, pn)
+                desc = '%s.%s' % (pn, taskname)
+
+                if len(hashfiles) == 0:
+                    out.append("Unable to find matching sigdata for %s with hashes %s or %s" % (desc, hash1, hash2))
+                elif not hash1 in hashfiles:
+                    out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash1))
+                elif not hash2 in hashfiles:
+                    out.append("Unable to find matching sigdata for %s with hash %s" % (desc, hash2))
+                else:
+                    out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, collapsed=True)
+                    for line in out2:
+                        m = hashlib.sha256()
+                        m.update(line.encode('utf-8'))
+                        entry = changes.get(m.hexdigest(), (line, []))
+                        if desc not in entry[1]:
+                            changes[m.hexdigest()] = (line, entry[1] + [desc])
+
+            # Define recursion callback
+            def recursecb(key, hash1, hash2):
+                compare_hashfiles(key, None, hash1, hash2)
+                return []
+
+            for key in keys:
+                siga = adict.get(key, None)
+                sigb = bdict.get(key, None)
+                if siga is not None and sigb is not None and siga != sigb:
+                    changecount += 1
+                    (pn, taskname) = key.rsplit('.', 1)
+                    compare_hashfiles(pn, taskname, siga, sigb)
+                elif siga is None:
+                    addcount += 1
+                elif sigb is None:
+                    removecount += 1
+        for key, item in changes.items():
+            line, tasks = item
+            if len(tasks) == 1:
+                desc = tasks[0]
+            elif len(tasks) == 2:
+                desc = '%s and %s' % (tasks[0], tasks[1])
+            else:
+                desc = '%s and %d others' % (tasks[-1], len(tasks)-1)
+            out.append('%s: %s' % (desc, line))
+    else:
+        for key in keys:
+            siga = adict.get(key, None)
+            sigb = bdict.get(key, None)
+            if siga is not None and sigb is not None and siga != sigb:
+                out.append('%s changed from %s to %s' % (key, siga, sigb))
+                changecount += 1
+            elif siga is None:
+                out.append('%s was added' % key)
+                addcount += 1
+            elif sigb is None:
+                out.append('%s was removed' % key)
+                removecount += 1
     out.append('Summary: %d tasks added, %d tasks removed, %d tasks modified (%.1f%%)' % (addcount, removecount, changecount, (changecount / float(len(bdict)) * 100)))
     return '\n'.join(out)
 
 
-def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False):
+def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False):
     repo = git.Repo(repopath)
     assert repo.bare == False
     commit = repo.commit(revision1)

@@ -456,10 +516,10 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep
 
     changes = []
 
-    if sigs:
+    if sigs or sigsdiff:
         for d in diff.iter_change_type('M'):
             if d.a_blob.path == 'siglist.txt':
-                changes.append(compare_siglists(d.a_blob, d.b_blob))
+                changes.append(compare_siglists(d.a_blob, d.b_blob, taskdiff=sigsdiff))
         return changes
 
     for d in diff.iter_change_type('M'):

index e8e3e11..dd9745e 100755
@@ -34,8 +34,11 @@ def main():
             help = "Report all changes, not just the default significant ones",
             action="store_true", dest="report_all", default=False)
     parser.add_option("-s", "--signatures",
-            help = "Report on signature differences instead of output",
+            help = "Report list of signatures differing instead of output",
             action="store_true", dest="sigs", default=False)
+    parser.add_option("-S", "--signatures-with-diff",
+            help = "Report on actual signature differences instead of output (requires signature data to have been generated, either by running the actual tasks or using bitbake -S)",
+            action="store_true", dest="sigsdiff", default=False)
 
     options, args = parser.parse_args(sys.argv)
 

@@ -89,7 +92,7 @@ def main():
 
     import gitdb
     try:
-        changes = oe.buildhistory_analysis.process_changes(options.buildhistory_dir, fromrev, torev, options.report_all, options.report_ver, options.sigs)
+        changes = oe.buildhistory_analysis.process_changes(options.buildhistory_dir, fromrev, torev, options.report_all, options.report_ver, options.sigs, options.sigsdiff)
     except gitdb.exc.BadObject as e:
         if len(args) == 1:
             sys.stderr.write("Unable to find previous build revision in buildhistory repository\n\n")

index b578031..8f34483 100644
@@ -100,7 +100,7 @@ def enable_uninative(d):
         bb.debug(2, "Enabling uninative")
         d.setVar("NATIVELSBSTRING", "universal%s" % oe.utils.host_gcc_version(d))
         d.appendVar("SSTATEPOSTUNPACKFUNCS", " uninative_changeinterp")
-        d.setVarFlag("SSTATEPOSTUNPACKFUNCS", "vardepvalueexclude", " uninative_changeinterp")
+        d.appendVarFlag("SSTATEPOSTUNPACKFUNCS", "vardepvalueexclude", "| uninative_changeinterp")
         d.prependVar("PATH", "${STAGING_DIR}-uninative/${BUILD_ARCH}-linux${bindir_native}:")
 
 python uninative_changeinterp () {

index 0cafad5..748091d 100644
@@ -47,6 +47,11 @@ sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory"
 # then the value added to SSTATEPOSTINSTFUNCS:
 SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory"
 
+# Similarly for our function that gets the output signatures
+SSTATEPOSTUNPACKFUNCS_append = " buildhistory_emit_outputsigs"
+sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs"
+SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
+
 # All items excepts those listed here will be removed from a recipe's
 # build history directory by buildhistory_emit_pkghistory(). This is
 # necessary because some of these items (package directories, files that

@@ -292,6 +297,29 @@ python buildhistory_emit_pkghistory() {
     bb.build.exec_func("buildhistory_list_pkg_files", d)
 }
 
+python buildhistory_emit_outputsigs() {
+    if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
+        return
+
+    taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output')
+    bb.utils.mkdirhier(taskoutdir)
+    currenttask = d.getVar('BB_CURRENTTASK')
+    pn = d.getVar('PN')
+    taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask))
+
+    cwd = os.getcwd()
+    filesigs = {}
+    for root, _, files in os.walk(cwd):
+        for fname in files:
+            if fname == 'fixmepath':
+                continue
+            fullpath = os.path.join(root, fname)
+            filesigs[os.path.relpath(fullpath, cwd)] = bb.utils.sha256_file(fullpath)
+    with open(taskfile, 'w') as f:
+        for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]):
+            f.write('%s %s\n' % (fpath, fsig))
+}
+
 
 def write_recipehistory(rcpinfo, d):
     bb.debug(2, "Writing recipe history")