bitbake-layers improvements

index b4cfd55..3007d51 100755
@@ -4,6 +4,9 @@
 # displaying useful information, or acting against them.
 # See the help output for details on available commands.
 
+# Copyright (C) 2011 Mentor Graphics Corporation
+# Copyright (C) 2012 Intel Corporation
+
 import cmd
 import logging
 import os

index 3007d51..041aa19 100755
@@ -41,6 +41,8 @@ def main(args):
 
     cmds = Commands(initialenv)
     if args:
+        # Allow user to specify e.g. show-layers instead of show_layers
+        args = [args[0].replace('-', '_')] + args[1:]
         cmds.onecmd(' '.join(args))
     else:
         cmds.do_help('')

@@ -93,14 +95,14 @@ class Commands(cmd.Cmd):
         """display general help or help on a specified command"""
         if topic:
             sys.stdout.write('%s: ' % topic)
-            cmd.Cmd.do_help(self,topic)
+            cmd.Cmd.do_help(self, topic.replace('-', '_'))
         else:
             sys.stdout.write("usage: bitbake-layers <command> [arguments]\n\n")
             sys.stdout.write("Available commands:\n")
             procnames = self.get_names()
             for procname in procnames:
                 if procname[:3] == 'do_':
-                    sys.stdout.write("  %s\n" % procname[3:])
+                    sys.stdout.write("  %s\n" % procname[3:].replace('_', '-'))
                     doc = getattr(self, procname).__doc__
                     if doc:
                         sys.stdout.write("    %s\n" % doc.splitlines()[0])

@@ -126,7 +128,7 @@ class Commands(cmd.Cmd):
     def do_show_overlayed(self, args):
         """list overlayed recipes (where there is a recipe in another layer that has a higher layer priority)
 
-usage: show_overlayed
+usage: show-overlayed
 
 Highest priority recipes are listed with the recipes they overlay as subitems.
 """

@@ -197,7 +199,7 @@ build results (as the layer priority order has effectively changed).
 
             for layername in layernames:
                 if not layername in found_layernames:
-                    logger.error('Unable to find layer %s in current configuration, please run "%s show_layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
+                    logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
                     return
             layers = found_layerdirs
         else:

@@ -316,7 +318,7 @@ build results (as the layer priority order has effectively changed).
     def do_show_appends(self, args):
         """list bbappend files and recipe files they apply to
 
-usage: show_appends
+usage: show-appends
 
 Recipes are listed with the bbappends that apply to them as subitems.
 """

index 041aa19..9d453ca 100755
@@ -60,6 +60,7 @@ class Commands(cmd.Cmd):
         self.config_data = self.cooker.configuration.data
         bb.providers.logger.setLevel(logging.ERROR)
         self.cooker_data = None
+        self.bblayers = (self.config_data.getVar('BBLAYERS', True) or "").split()
 
     def register_idle_function(self, function, data):
         pass

@@ -113,13 +114,11 @@ class Commands(cmd.Cmd):
         logger.plain('')
         logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
         logger.plain('=' * 74)
-        layerdirs = str(self.config_data.getVar('BBLAYERS', True)).split()
-        for layerdir in layerdirs:
-            layername = '?'
+        for layerdir in self.bblayers:
+            layername = self.get_layer_name(layerdir)
             layerpri = 0
             for layer, _, regex, pri in self.cooker.status.bbfile_config_priorities:
                 if regex.match(os.path.join(layerdir, 'test')):
-                    layername = layer
                     layerpri = pri
                     break
 

@@ -184,18 +183,16 @@ build results (as the layer priority order has effectively changed).
             return
 
         self.check_prepare_cooker()
-        layers = (self.config_data.getVar('BBLAYERS', True) or "").split()
+        layers = self.bblayers
         if len(arglist) > 2:
             layernames = arglist[:-1]
             found_layernames = []
             found_layerdirs = []
             for layerdir in layers:
-                for layername, _, regex, _ in self.cooker.status.bbfile_config_priorities:
-                    if layername in layernames:
-                        if regex.match(os.path.join(layerdir, 'test')):
-                            found_layerdirs.append(layerdir)
-                            found_layernames.append(layername)
-                            break
+                layername = self.get_layer_name(layerdir)
+                if layername in layernames:
+                    found_layerdirs.append(layerdir)
+                    found_layernames.append(layername)
 
             for layername in layernames:
                 if not layername in found_layernames:

@@ -271,10 +268,9 @@ build results (as the layer priority order has effectively changed).
         first_regex = None
         layerdir = layers[0]
         for layername, pattern, regex, _ in self.cooker.status.bbfile_config_priorities:
-            if (not layernames) or layername in layernames:
-                if regex.match(os.path.join(layerdir, 'test')):
-                    first_regex = regex
-                    break
+            if regex.match(os.path.join(layerdir, 'test')):
+                first_regex = regex
+                break
 
         if first_regex:
             # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)

@@ -300,17 +296,22 @@ build results (as the layer priority order has effectively changed).
                             if not entry_found:
                                 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
 
-    def get_append_layer(self, appendname):
+    def get_file_layer(self, filename):
         for layer, _, regex, _ in self.cooker.status.bbfile_config_priorities:
-            if regex.match(appendname):
-                return layer
+            if regex.match(filename):
+                for layerdir in self.bblayers:
+                    if regex.match(os.path.join(layerdir, 'test')):
+                        return self.get_layer_name(layerdir)
         return "?"
 
+    def get_layer_name(self, layerdir):
+        return os.path.basename(layerdir.rstrip(os.sep))
+
     def apply_append(self, appendname, recipename):
         appendfile = open(appendname, 'r')
         recipefile = open(recipename, 'a')
         recipefile.write('\n')
-        recipefile.write('##### bbappended from %s #####\n' % self.get_append_layer(appendname))
+        recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
         recipefile.writelines(appendfile.readlines())
         recipefile.close()
         appendfile.close()

index 9d453ca..abcfb99 100755
@@ -12,6 +12,7 @@ import logging
 import os
 import sys
 import fnmatch
+from collections import defaultdict
 
 bindir = os.path.dirname(__file__)
 topdir = os.path.dirname(bindir)

@@ -124,22 +125,115 @@ class Commands(cmd.Cmd):
 
             logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), layerpri))
 
+
+    def version_str(self, pe, pv, pr = None):
+        verstr = "%s" % pv
+        if pr:
+            verstr = "%s-%s" % (verstr, pr)
+        if pe:
+            verstr = "%s:%s" % (pe, verstr)
+        return verstr
+
+
     def do_show_overlayed(self, args):
-        """list overlayed recipes (where there is a recipe in another layer that has a higher layer priority)
+        """list overlayed recipes (where the same recipe exists in another layer that has a higher layer priority)
+
+usage: show-overlayed [-f] [-s]
 
-usage: show-overlayed
+Lists the names of overlayed recipes and the available versions in each
+layer, with the preferred version first. Note that skipped recipes that
+are overlayed will also be listed, with a " (skipped)" suffix.
 
-Highest priority recipes are listed with the recipes they overlay as subitems.
+Options:
+  -f   instead of the default formatting, list filenames of higher priority
+       recipes with the ones they overlay indented underneath
+  -s   only list overlayed recipes where the version is the same
 """
         self.check_prepare_cooker()
-        if self.cooker.overlayed:
-            logger.plain('Overlayed recipes:')
-            for f in self.cooker.overlayed.iterkeys():
-                logger.plain('%s' % f)
-                for of in self.cooker.overlayed[f]:
-                    logger.plain('  %s' % of)
-        else:
-            logger.plain('No overlayed recipes found')
+
+        show_filenames = False
+        show_same_ver_only = False
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            elif arg == '-s':
+                show_same_ver_only = True
+            else:
+                sys.stderr.write("show-overlayed: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+
+        pkg_pn = self.cooker.status.pkg_pn
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.cooker.configuration.data, self.cooker.status, pkg_pn)
+        allproviders = bb.providers.allProviders(self.cooker.status)
+
+        # Ensure we list skipped recipes
+        # We are largely guessing about PN, PV and the preferred version here,
+        # but we have no choice since skipped recipes are not fully parsed
+        skiplist = self.cooker.skiplist.keys()
+        skiplist.sort( key=lambda fileitem: self.cooker.calc_bbfile_priority(fileitem) )
+        skiplist.reverse()
+        for fn in skiplist:
+            recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
+            p = recipe_parts[0]
+            if len(recipe_parts) > 1:
+                ver = (None, recipe_parts[1], None)
+            else:
+                ver = (None, 'unknown', None)
+            allproviders[p].append((ver, fn))
+            if not p in pkg_pn:
+                pkg_pn[p] = 'dummy'
+                preferred_versions[p] = (ver, fn)
+
+        def print_item(f, pn, ver, layer, ispref):
+            if f in skiplist:
+                skipped = ' (skipped)'
+            else:
+                skipped = ''
+            if show_filenames:
+                if ispref:
+                    logger.plain("%s%s", f, skipped)
+                else:
+                    logger.plain("  %s%s", f, skipped)
+            else:
+                if ispref:
+                    logger.plain("%s:", pn)
+                logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
+
+        preffiles = []
+        items_listed = False
+        for p in sorted(pkg_pn):
+            if len(allproviders[p]) > 1:
+                pref = preferred_versions[p]
+                preffile = bb.cache.Cache.virtualfn2realfn(pref[1])[0]
+                if preffile not in preffiles:
+                    preflayer = self.get_file_layer(preffile)
+                    multilayer = False
+                    same_ver = True
+                    provs = []
+                    for prov in allproviders[p]:
+                        provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
+                        provlayer = self.get_file_layer(provfile)
+                        provs.append((provfile, provlayer, prov[0]))
+                        if provlayer != preflayer:
+                            multilayer = True
+                        if prov[0] != pref[0]:
+                            same_ver = False
+
+                    if multilayer and (same_ver or not show_same_ver_only):
+                        if not items_listed:
+                            logger.plain('=== Overlayed recipes ===')
+                            items_listed = True
+                        print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
+                        for (provfile, provlayer, provver) in provs:
+                            if provfile != preffile:
+                                print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
+                        # Ensure we don't show two entries for BBCLASSEXTENDed recipes
+                        preffiles.append(preffile)
+
+        if not items_listed:
+            logger.note('No overlayed files found')
+
 
     def do_flatten(self, args):
         """flattens layer configuration into a separate output directory.

index b6bd740..652cd5d 100644
@@ -259,20 +259,8 @@ class BBCooker:
         # Need files parsed
         self.updateCache()
 
-        # Need to ensure data store is expanded
-        localdata = data.createCopy(self.configuration.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
-
         pkg_pn = self.status.pkg_pn
-        preferred_versions = {}
-        latest_versions = {}
-
-        # Sort by priority
-        for pn in pkg_pn:
-            (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, localdata, self.status)
-            preferred_versions[pn] = (pref_ver, pref_file)
-            latest_versions[pn] = (last_ver, last_file)
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.configuration.data, self.status, pkg_pn)
 
         logger.plain("%-35s %25s %25s", "Package Name", "Latest Version", "Preferred Version")
         logger.plain("%-35s %25s %25s\n", "============", "==============", "=================")

index 398c8ea..1dc6a8e 100644
@@ -24,6 +24,7 @@
 import re
 import logging
 from bb import data, utils
+from collections import defaultdict
 import bb
 
 logger = logging.getLogger("BitBake.Provider")

@@ -35,6 +36,41 @@ class NoRProvider(bb.BBHandledException):
     """Exception raised when no provider of a runtime dependency can be found"""
 
 
+def findProviders(cfgData, dataCache, pkg_pn = None):
+    """
+    Convenience function to get latest and preferred providers in pkg_pn
+    """
+
+    if not pkg_pn:
+        pkg_pn = dataCache.pkg_pn
+
+    # Need to ensure data store is expanded
+    localdata = data.createCopy(cfgData)
+    bb.data.update_data(localdata)
+    bb.data.expandKeys(localdata)
+
+    preferred_versions = {}
+    latest_versions = {}
+
+    for pn in pkg_pn:
+        (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+        preferred_versions[pn] = (pref_ver, pref_file)
+        latest_versions[pn] = (last_ver, last_file)
+
+    return (latest_versions, preferred_versions)
+
+
+def allProviders(dataCache):
+    """
+    Find all providers for each pn
+    """
+    all_providers = defaultdict(list)
+    for (fn, pn) in dataCache.pkg_fn.items():
+        ver = dataCache.pkg_pepvpr[fn]
+        all_providers[pn].append((ver, fn))
+    return all_providers
+
+
 def sortPriorities(pn, dataCache, pkg_pn = None):
     """
     Reorder pkg_pn by file priority and default preference

index abcfb99..95150de 100755
@@ -163,6 +163,50 @@ Options:
                 self.do_help('')
                 return
 
+        items_listed = self.list_recipes('Overlayed recipes', None, True, show_same_ver_only, show_filenames, True)
+
+        if not items_listed:
+            logger.note('No overlayed files found')
+
+
+    def do_show_recipes(self, args):
+        """list available recipes, showing the layer they are provided by
+
+usage: show-recipes [-f] [-m] [pnspec]
+
+Lists the names of overlayed recipes and the available versions in each
+layer, with the preferred version first. Optionally you may specify
+pnspec to match a specified recipe name (supports wildcards). Note that
+skipped recipes will also be listed, with a " (skipped)" suffix.
+
+Options:
+  -f   instead of the default formatting, list filenames of higher priority
+       recipes with other available recipes indented underneath
+  -m   only list where multiple recipes (in the same layer or different
+       layers) exist for the same recipe name
+"""
+        self.check_prepare_cooker()
+
+        show_filenames = False
+        show_multi_provider_only = False
+        pnspec = None
+        title = 'Available recipes:'
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            elif arg == '-m':
+                show_multi_provider_only = True
+            elif not arg.startswith('-'):
+                pnspec = arg
+                title = 'Available recipes matching %s:' % pnspec
+            else:
+                sys.stderr.write("show-recipes: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+        self.list_recipes(title, pnspec, False, False, show_filenames, show_multi_provider_only)
+
+
+    def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only):
         pkg_pn = self.cooker.status.pkg_pn
         (latest_versions, preferred_versions) = bb.providers.findProviders(self.cooker.configuration.data, self.cooker.status, pkg_pn)
         allproviders = bb.providers.allProviders(self.cooker.status)

@@ -203,7 +247,11 @@ Options:
         preffiles = []
         items_listed = False
         for p in sorted(pkg_pn):
-            if len(allproviders[p]) > 1:
+            if pnspec:
+                if not fnmatch.fnmatch(p, pnspec):
+                    continue
+
+            if len(allproviders[p]) > 1 or not show_multi_provider_only:
                 pref = preferred_versions[p]
                 preffile = bb.cache.Cache.virtualfn2realfn(pref[1])[0]
                 if preffile not in preffiles:

@@ -220,9 +268,9 @@ Options:
                         if prov[0] != pref[0]:
                             same_ver = False
 
-                    if multilayer and (same_ver or not show_same_ver_only):
+                    if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
                         if not items_listed:
-                            logger.plain('=== Overlayed recipes ===')
+                            logger.plain('=== %s ===' % title)
                             items_listed = True
                         print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
                         for (provfile, provlayer, provver) in provs:

@@ -231,8 +279,7 @@ Options:
                         # Ensure we don't show two entries for BBCLASSEXTENDed recipes
                         preffiles.append(preffile)
 
-        if not items_listed:
-            logger.note('No overlayed files found')
+        return items_listed
 
 
     def do_flatten(self, args):

index 95150de..c85e26e 100755
@@ -165,6 +165,53 @@ Options:
 
         items_listed = self.list_recipes('Overlayed recipes', None, True, show_same_ver_only, show_filenames, True)
 
+        # Check for overlayed .bbclass files
+        classes = defaultdict(list)
+        for layerdir in self.bblayers:
+            classdir = os.path.join(layerdir, 'classes')
+            if os.path.exists(classdir):
+                for classfile in os.listdir(classdir):
+                    if os.path.splitext(classfile)[1] == '.bbclass':
+                        classes[classfile].append(classdir)
+
+        # Locating classes and other files is a bit more complicated than recipes -
+        # layer priority is not a factor; instead BitBake uses the first matching
+        # file in BBPATH, which is manipulated directly by each layer's
+        # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
+        # factor - however, each layer.conf is free to either prepend or append to
+        # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
+        # not be exactly the order present in bblayers.conf either.
+        bbpath = str(self.config_data.getVar('BBPATH', True))
+        overlayed_class_found = False
+        for (classfile, classdirs) in classes.items():
+            if len(classdirs) > 1:
+                if not overlayed_class_found:
+                    logger.plain('=== Overlayed classes ===')
+                    overlayed_class_found = True
+
+                mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
+                if show_filenames:
+                    logger.plain('%s' % mainfile)
+                else:
+                    # We effectively have to guess the layer here
+                    logger.plain('%s:' % classfile)
+                    mainlayername = '?'
+                    for layerdir in self.bblayers:
+                        classdir = os.path.join(layerdir, 'classes')
+                        if mainfile.startswith(classdir):
+                            mainlayername = self.get_layer_name(layerdir)
+                    logger.plain('  %s' % mainlayername)
+                for classdir in classdirs:
+                    fullpath = os.path.join(classdir, classfile)
+                    if fullpath != mainfile:
+                        if show_filenames:
+                            print('  %s' % fullpath)
+                        else:
+                            print('  %s' % self.get_layer_name(os.path.dirname(classdir)))
+
+        if overlayed_class_found:
+            items_listed = True;
+
         if not items_listed:
             logger.note('No overlayed files found')