Refactoring + improve bitbake-diffsigs

index f473711..fa4e767 100755
@@ -6,10 +6,22 @@
 
 # Copyright (C) 2011 Mentor Graphics Corporation
 # Copyright (C) 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import cmd
 import logging
-import warnings
 import os
 import sys
 import fnmatch

@@ -23,26 +35,14 @@ import bb.cache
 import bb.cooker
 import bb.providers
 import bb.utils
-from bb.cooker import state
-import bb.fetch2
+import bb.tinfoil
 
 
 logger = logging.getLogger('BitBake')
 
-warnings.filterwarnings("ignore", category=DeprecationWarning)
 
 def main(args):
-    # Set up logging
-    console = logging.StreamHandler(sys.stdout)
-    format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-    bb.msg.addDefaultlogFilter(console)
-    console.setFormatter(format)
-    logger.addHandler(console)
-
-    initialenv = os.environ.copy()
-    bb.utils.clean_environment()
-
-    cmds = Commands(initialenv)
+    cmds = Commands()
     if args:
         # Allow user to specify e.g. show-layers instead of show_layers
         args = [args[0].replace('-', '_')] + args[1:]

@@ -53,46 +53,11 @@ def main(args):
 
 
 class Commands(cmd.Cmd):
-    def __init__(self, initialenv):
+    def __init__(self):
         cmd.Cmd.__init__(self)
+        self.bbhandler = bb.tinfoil.Tinfoil()
         self.returncode = 0
-        self.config = Config(parse_only=True)
-        self.cooker = bb.cooker.BBCooker(self.config,
-                                         self.register_idle_function,
-                                         initialenv)
-        self.config_data = self.cooker.configuration.data
-        bb.providers.logger.setLevel(logging.ERROR)
-        self.cooker_data = None
-        self.bblayers = (self.config_data.getVar('BBLAYERS', True) or "").split()
-
-    def register_idle_function(self, function, data):
-        pass
-
-    def prepare_cooker(self):
-        sys.stderr.write("Parsing recipes..")
-        logger.setLevel(logging.WARNING)
-
-        try:
-            while self.cooker.state in (state.initial, state.parsing):
-                self.cooker.updateCache()
-        except KeyboardInterrupt:
-            self.cooker.shutdown()
-            self.cooker.updateCache()
-            sys.exit(2)
-
-        logger.setLevel(logging.INFO)
-        sys.stderr.write("done.\n")
-
-        self.cooker_data = self.cooker.status
-        self.cooker_data.appends = self.cooker.appendlist
-
-    def check_prepare_cooker(self, config_only = False):
-        if not self.cooker_data:
-            if config_only:
-                self.cooker.parseConfiguration()
-                self.cooker_data = self.cooker.status
-            else:
-                self.prepare_cooker()
+        self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split()
 
     def default(self, line):
         """Handle unrecognised commands"""

@@ -117,13 +82,13 @@ class Commands(cmd.Cmd):
 
     def do_show_layers(self, args):
         """show current configured layers"""
-        self.check_prepare_cooker(config_only = True)
+        self.bbhandler.prepare(config_only = True)
         logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
         logger.plain('=' * 74)
         for layerdir in self.bblayers:
             layername = self.get_layer_name(layerdir)
             layerpri = 0
-            for layer, _, regex, pri in self.cooker.status.bbfile_config_priorities:
+            for layer, _, regex, pri in self.bbhandler.cooker.status.bbfile_config_priorities:
                 if regex.match(os.path.join(layerdir, 'test')):
                     layerpri = pri
                     break

@@ -154,7 +119,7 @@ Options:
        recipes with the ones they overlay indented underneath
   -s   only list overlayed recipes where the version is the same
 """
-        self.check_prepare_cooker()
+        self.bbhandler.prepare()
 
         show_filenames = False
         show_same_ver_only = False

@@ -186,7 +151,7 @@ Options:
         # factor - however, each layer.conf is free to either prepend or append to
         # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
         # not be exactly the order present in bblayers.conf either.
-        bbpath = str(self.config_data.getVar('BBPATH', True))
+        bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
         overlayed_class_found = False
         for (classfile, classdirs) in classes.items():
             if len(classdirs) > 1:

@@ -237,7 +202,7 @@ Options:
   -m   only list where multiple recipes (in the same layer or different
        layers) exist for the same recipe name
 """
-        self.check_prepare_cooker()
+        self.bbhandler.prepare()
 
         show_filenames = False
         show_multi_provider_only = False

@@ -259,15 +224,15 @@ Options:
 
 
     def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only):
-        pkg_pn = self.cooker.status.pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.cooker.configuration.data, self.cooker.status, pkg_pn)
-        allproviders = bb.providers.allProviders(self.cooker.status)
+        pkg_pn = self.bbhandler.cooker.status.pkg_pn
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.bbhandler.cooker.configuration.data, self.bbhandler.cooker.status, pkg_pn)
+        allproviders = bb.providers.allProviders(self.bbhandler.cooker.status)
 
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
         # but we have no choice since skipped recipes are not fully parsed
-        skiplist = self.cooker.skiplist.keys()
-        skiplist.sort( key=lambda fileitem: self.cooker.calc_bbfile_priority(fileitem) )
+        skiplist = self.bbhandler.cooker.skiplist.keys()
+        skiplist.sort( key=lambda fileitem: self.bbhandler.cooker.calc_bbfile_priority(fileitem) )
         skiplist.reverse()
         for fn in skiplist:
             recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')

@@ -375,7 +340,7 @@ build results (as the layer priority order has effectively changed).
             logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
             return
 
-        self.check_prepare_cooker()
+        self.bbhandler.prepare()
         layers = self.bblayers
         if len(arglist) > 2:
             layernames = arglist[:-1]

@@ -405,8 +370,8 @@ build results (as the layer priority order has effectively changed).
         appended_recipes = []
         for layer in layers:
             overlayed = []
-            for f in self.cooker.overlayed.iterkeys():
-                for of in self.cooker.overlayed[f]:
+            for f in self.bbhandler.cooker.overlayed.iterkeys():
+                for of in self.bbhandler.cooker.overlayed[f]:
                     if of.startswith(layer):
                         overlayed.append(of)
 

@@ -430,8 +395,8 @@ build results (as the layer priority order has effectively changed).
                                     logger.warn('Overwriting file %s', fdest)
                             bb.utils.copyfile(f1full, fdest)
                             if ext == '.bb':
-                                if f1 in self.cooker_data.appends:
-                                    appends = self.cooker_data.appends[f1]
+                                if f1 in self.bbhandler.cooker.appendlist:
+                                    appends = self.bbhandler.cooker.appendlist[f1]
                                     if appends:
                                         logger.plain('  Applying appends to %s' % fdest )
                                         for appendname in appends:

@@ -440,9 +405,9 @@ build results (as the layer priority order has effectively changed).
                                     appended_recipes.append(f1)
 
         # Take care of when some layers are excluded and yet we have included bbappends for those recipes
-        for recipename in self.cooker_data.appends.iterkeys():
+        for recipename in self.bbhandler.cooker.appendlist.iterkeys():
             if recipename not in appended_recipes:
-                appends = self.cooker_data.appends[recipename]
+                appends = self.bbhandler.cooker.appendlist[recipename]
                 first_append = None
                 for appendname in appends:
                     layer = layer_path_match(appendname)

@@ -460,14 +425,14 @@ build results (as the layer priority order has effectively changed).
         # have come from)
         first_regex = None
         layerdir = layers[0]
-        for layername, pattern, regex, _ in self.cooker.status.bbfile_config_priorities:
+        for layername, pattern, regex, _ in self.bbhandler.cooker.status.bbfile_config_priorities:
             if regex.match(os.path.join(layerdir, 'test')):
                 first_regex = regex
                 break
 
         if first_regex:
             # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
-            bbfiles = str(self.config_data.getVar('BBFILES', True)).split()
+            bbfiles = str(self.bbhandler.config_data.getVar('BBFILES', True)).split()
             bbfiles_layer = []
             for item in bbfiles:
                 if first_regex.match(item):

@@ -490,7 +455,7 @@ build results (as the layer priority order has effectively changed).
                                 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
 
     def get_file_layer(self, filename):
-        for layer, _, regex, _ in self.cooker.status.bbfile_config_priorities:
+        for layer, _, regex, _ in self.bbhandler.cooker.status.bbfile_config_priorities:
             if regex.match(filename):
                 for layerdir in self.bblayers:
                     if regex.match(os.path.join(layerdir, 'test')):

@@ -516,14 +481,14 @@ usage: show-appends
 
 Recipes are listed with the bbappends that apply to them as subitems.
 """
-        self.check_prepare_cooker()
-        if not self.cooker_data.appends:
+        self.bbhandler.prepare()
+        if not self.bbhandler.cooker.appendlist:
             logger.plain('No append files found')
             return
 
         logger.plain('=== Appended recipes ===')
 
-        pnlist = list(self.cooker_data.pkg_pn.keys())
+        pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys())
         pnlist.sort()
         for pn in pnlist:
             self.show_appends_for_pn(pn)

@@ -531,19 +496,19 @@ Recipes are listed with the bbappends that apply to them as subitems.
         self.show_appends_for_skipped()
 
     def show_appends_for_pn(self, pn):
-        filenames = self.cooker_data.pkg_pn[pn]
+        filenames = self.bbhandler.cooker_data.pkg_pn[pn]
 
         best = bb.providers.findBestProvider(pn,
-                                             self.cooker.configuration.data,
-                                             self.cooker_data,
-                                             self.cooker_data.pkg_pn)
+                                             self.bbhandler.cooker.configuration.data,
+                                             self.bbhandler.cooker_data,
+                                             self.bbhandler.cooker_data.pkg_pn)
         best_filename = os.path.basename(best[3])
 
         self.show_appends_output(filenames, best_filename)
 
     def show_appends_for_skipped(self):
         filenames = [os.path.basename(f)
-                    for f in self.cooker.skiplist.iterkeys()]
+                    for f in self.bbhandler.cooker.skiplist.iterkeys()]
         self.show_appends_output(filenames, None, " (skipped)")
 
     def show_appends_output(self, filenames, best_filename, name_suffix = ''):

@@ -569,7 +534,7 @@ Recipes are listed with the bbappends that apply to them as subitems.
                 continue
 
             basename = os.path.basename(filename)
-            appends = self.cooker_data.appends.get(basename)
+            appends = self.bbhandler.cooker.appendlist.get(basename)
             if appends:
                 appended.append((basename, list(appends)))
             else:

@@ -577,22 +542,5 @@ Recipes are listed with the bbappends that apply to them as subitems.
         return appended, notappended
 
 
-class Config(object):
-    def __init__(self, **options):
-        self.pkgs_to_build = []
-        self.debug_domains = []
-        self.extra_assume_provided = []
-        self.prefile = []
-        self.postfile = []
-        self.debug = 0
-        self.__dict__.update(options)
-
-    def __getattr__(self, attribute):
-        try:
-            return super(Config, self).__getattribute__(attribute)
-        except AttributeError:
-            return None
-
-
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]) or 0)

98 lines not on this page · raw message

index 02a4268..b04a8bc 100644
@@ -185,8 +185,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
         if task in dataCache.file_checksums[fn]:
             checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
             for (f,cs) in checksums:
-               self.file_checksum_values[k][f] = cs
-               data = data + cs
+                self.file_checksum_values[k][f] = cs
+                data = data + cs
 
         taint = self.read_taint(fn, task, dataCache.stamp[fn])
         if taint:

@@ -372,24 +372,24 @@ def compare_sigfiles(a, b):
         changed, added, removed = dict_diff(a, b)
         if added:
             for dep in added:
-	        bdep_found = False
-	        if removed:
-		    for bdep in removed:
-		    	if a[dep] == b[bdep]:
-			   #print "Dependency on task %s was replaced by %s with same hash" % (dep, bdep)
-			   bdep_found = True
-		if not bdep_found:
+                bdep_found = False
+                if removed:
+                    for bdep in removed:
+                        if a[dep] == b[bdep]:
+                            #print "Dependency on task %s was replaced by %s with same hash" % (dep, bdep)
+                            bdep_found = True
+                if not bdep_found:
                     print "Dependency on task %s was added with hash %s" % (dep, a[dep])
         if removed:
             for dep in removed:
-	        adep_found = False
-	        if added:
-		    for adep in added:
-		    	if a[adep] == b[dep]:
-			   #print "Dependency on task %s was replaced by %s with same hash" % (adep, dep)
-			   adep_found = True
-		if not adep_found:
-                   print "Dependency on task %s was removed with hash %s" % (dep, b[dep])
+                adep_found = False
+                if added:
+                    for adep in added:
+                        if a[adep] == b[dep]:
+                            #print "Dependency on task %s was replaced by %s with same hash" % (adep, dep)
+                            adep_found = True
+                if not adep_found:
+                    print "Dependency on task %s was removed with hash %s" % (dep, b[dep])
         if changed:
             for dep in changed:
                 print "Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep])

index b04a8bc..d0a4d18 100644
@@ -290,7 +290,7 @@ def dump_this_task(outfile, d):
 
 def clean_basepath(a):
     if a.startswith("virtual:"):
-        b = a.rsplit(":", 1)[0] + a.rsplit("/", 1)[1]
+        b = a.rsplit(":", 1)[0] + ":" + a.rsplit("/", 1)[1]
     else:
         b = a.rsplit("/", 1)[1]
     return b

index 5eb77ce..146cab8 100755
@@ -7,6 +7,9 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), '
 import bb.siggen
 
 if len(sys.argv) > 2:
-    bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
+    output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
 else:
-    bb.siggen.dump_sigfile(sys.argv[1])
+    output = bb.siggen.dump_sigfile(sys.argv[1])
+
+if output:
+    print '\n'.join(output)

index 59740c8..ccbc412 100755
@@ -6,4 +6,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), '
 
 import bb.siggen
 
-bb.siggen.dump_sigfile(sys.argv[1])
+output = bb.siggen.dump_sigfile(sys.argv[1])
+if output:
+    print '\n'.join(output)

152 lines not on this page · raw message

index 146cab8..62f6827 100755
@@ -1,15 +1,102 @@
 #!/usr/bin/env python
+
+# bitbake-diffsigs
+# BitBake task signature data comparison utility
+#
+# Copyright (C) 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
 import os
 import sys
 import warnings
+import fnmatch
+import optparse
+import logging
+
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
+import bb.tinfoil
 import bb.siggen
 
-if len(sys.argv) > 2:
-    output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
+logger = logging.getLogger('BitBake')
+
+def find_compare_task(bbhandler, pn, taskname):
+    """ Find the most recent signature files for the specified PN/task and compare them """
+
+    if not hasattr(bb.siggen, 'find_siginfo'):
+        logger.error('Metadata does not support finding signature data files')
+        sys.exit(1)
+
+    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
+    if not latestfiles:
+        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
+        sys.exit(1)
+    elif len(latestfiles) < 2:
+        logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
+        sys.exit(1)
+    else:
+        # Define recursion callback
+        def recursecb(key, hash1, hash2):
+            hashes = [hash1, hash2]
+            hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data)
+
+            recout = []
+            if len(hashfiles) == 2:
+                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
+                recout.extend(list('  ' + l for l in out2))
+            else:
+                recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+
+            return recout
+
+        # Recurse into signature comparison
+        output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb)
+        if output:
+            print '\n'.join(output)
+    sys.exit(0)
+
+
+
+parser = optparse.OptionParser(
+    usage = """
+  %prog -t recipename taskname
+  %prog sigdatafile1 sigdatafile2
+  %prog sigdatafile1""")
+
+parser.add_option("-t", "--task",
+        help = "find the signature data files for last two runs of the specified task and compare them",
+        action="store_true", dest="taskmode")
+
+options, args = parser.parse_args(sys.argv)
+
+if len(args) == 1:
+    parser.print_help()
 else:
-    output = bb.siggen.dump_sigfile(sys.argv[1])
+    tinfoil = bb.tinfoil.Tinfoil()
+    if options.taskmode:
+        if len(args) < 3:
+            logger.error("Please specify a recipe and task name")
+            sys.exit(1)
+        tinfoil.prepare(config_only = True)
+        find_compare_task(tinfoil, args[1], args[2])
+    else:
+        if len(args) == 2:
+            output = bb.siggen.dump_sigfile(sys.argv[1])
+        else:
+            output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2])
 
-if output:
-    print '\n'.join(output)
+        if output:
+            print '\n'.join(output)

index 8d1501b..8fe59b9 100644
@@ -301,7 +301,7 @@ def clean_basepaths(a):
         b[clean_basepath(x)] = a[x]
     return b
 
-def compare_sigfiles(a, b):
+def compare_sigfiles(a, b, recursecb = None):
     output = []
 
     p1 = pickle.Unpickler(open(a, "rb"))

@@ -369,8 +369,8 @@ def compare_sigfiles(a, b):
 
 
     if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
-        a = clean_basepaths(a_data['runtaskhashes'])
-        b = clean_basepaths(b_data['runtaskhashes'])
+        a = a_data['runtaskhashes']
+        b = b_data['runtaskhashes']
         changed, added, removed = dict_diff(a, b)
         if added:
             for dep in added:

@@ -381,7 +381,7 @@ def compare_sigfiles(a, b):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
                             bdep_found = True
                 if not bdep_found:
-                    output.append("Dependency on task %s was added with hash %s" % (dep, a[dep]))
+                    output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), a[dep]))
         if removed:
             for dep in removed:
                 adep_found = False

@@ -391,11 +391,14 @@ def compare_sigfiles(a, b):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
                             adep_found = True
                 if not adep_found:
-                    output.append("Dependency on task %s was removed with hash %s" % (dep, b[dep]))
+                    output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), b[dep]))
         if changed:
             for dep in changed:
-                output.append("Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep]))
-
+                output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep]))
+                if callable(recursecb):
+                    recout = recursecb(dep, a[dep], b[dep])
+                    if recout:
+                        output.extend(recout)
 
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)