Layer tooling improvements v2

index 6b5ad5a..fced88e 100755
@@ -73,6 +73,16 @@ class Commands(cmd.Cmd):
     def do_show_layers(self, args):
         logger.info(str(self.config_data.getVar('BBLAYERS', True)))
 
+    def do_show_overlayed(self, args):
+        if self.cooker.overlayed:
+            logger.info('Overlayed recipes:')
+            for f in self.cooker.overlayed.iterkeys():
+                logger.info('%s' % f)
+                for of in self.cooker.overlayed[f]:
+                    logger.info('  %s' % of)
+        else:
+            logger.info('No overlayed recipes found')
+
     def do_show_appends(self, args):
         if not self.cooker_data.appends:
             logger.info('No append files found')

index ebf963d..5fc2baa 100644
@@ -33,6 +33,7 @@ import threading
 from cStringIO import StringIO
 from contextlib import closing
 from functools import wraps
+from collections import defaultdict
 import bb, bb.exceptions
 from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue
 

@@ -1053,6 +1054,18 @@ class BBCooker:
                self.appendlist[base] = []
             self.appendlist[base].append(f)
 
+        # Find overlayed recipes
+        # bbfiles will be in priority order which makes this easy
+        bbfile_seen = dict()
+        self.overlayed = defaultdict(list)
+        for f in reversed(bbfiles):
+            base = os.path.basename(f)
+            if base not in bbfile_seen:
+                bbfile_seen[base] = f
+            else:
+                topfile = bbfile_seen[base]
+                self.overlayed[topfile].append(f)
+
         return (bbfiles, masked)
 
     def get_file_appends(self, fn):

index 5fc2baa..59519f8 100644
@@ -702,26 +702,87 @@ class BBCooker:
         """Handle collections"""
         self.status.bbfile_config_priorities = []
         if collections:
+            collection_priorities = {}
+            collection_depends = {}
             collection_list = collections.split()
+            min_prio = 0
             for c in collection_list:
+                # Get collection priority if defined explicitly
+                priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
+                if priority:
+                    try:
+                        prio = int(priority)
+                    except ValueError:
+                        parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
+                    if min_prio == 0 or prio < min_prio:
+                        min_prio = prio
+                    collection_priorities[c] = prio
+                else:
+                    collection_priorities[c] = None
+
+                # Check dependencies and store information for priority calculation
+                deps = bb.data.getVar("LAYERDEPENDS_%s" % c, self.configuration.data, 1)
+                if deps:
+                    depnamelist = []
+                    deplist = deps.split()
+                    for dep in deplist:
+                        depsplit = dep.split(':')
+                        if len(depsplit) > 1:
+                            try:
+                                depver = int(depsplit[1])
+                            except ValueError:
+                                parselog.error("invalid version value in LAYERDEPENDS_%s: \"%s\"", c, dep)
+                                continue
+                        else:
+                            depver = None
+                        dep = depsplit[0]
+                        depnamelist.append(dep)
+
+                        if dep in collection_list:
+                            if depver:
+                                layerver = bb.data.getVar("LAYERVERSION_%s" % dep, self.configuration.data, 1)
+                                if layerver:
+                                    try:
+                                        lver = int(layerver)
+                                    except ValueError:
+                                        parselog.error("invalid value for LAYERVERSION_%s: \"%s\"", c, layerver)
+                                        continue
+                                    if lver <> depver:
+                                        parselog.error("Layer dependency %s of layer %s is at version %d, expected %d", dep, c, lver, depver)
+                                else:
+                                    parselog.error("Layer dependency %s of layer %s has no version, expected %d", dep, c, depver)
+                        else:
+                            parselog.error("Layer dependency %s of layer %s not found", dep, c)
+                    collection_depends[c] = depnamelist
+                else:
+                    collection_depends[c] = []
+
+            # Recursively work out collection priorities based on dependencies
+            def calc_layer_priority(collection):
+                if not collection_priorities[collection]:
+                    max_depprio = min_prio
+                    for dep in collection_depends[collection]:
+                        calc_layer_priority(dep)
+                        depprio = collection_priorities[dep]
+                        if depprio > max_depprio:
+                            max_depprio = depprio
+                    max_depprio += 1
+                    parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio)
+                    collection_priorities[collection] = max_depprio
+
+            # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
+            for c in collection_list:
+                calc_layer_priority(c)
                 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
                 if regex == None:
                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
                     continue
-                priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
-                if priority == None:
-                    parselog.error("BBFILE_PRIORITY_%s not defined" % c)
-                    continue
                 try:
                     cre = re.compile(regex)
                 except re.error:
                     parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
                     continue
-                try:
-                    pri = int(priority)
-                    self.status.bbfile_config_priorities.append((c, regex, cre, pri))
-                except ValueError:
-                    parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
+                self.status.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
 
     def buildSetVars(self):
         """

index fced88e..110e3c8 100755
@@ -18,6 +18,7 @@ sys.path[0:0] = [os.path.join(topdir, 'lib')]
 import bb.cache
 import bb.cooker
 import bb.providers
+import bb.utils
 from bb.cooker import state
 
 

@@ -83,6 +84,64 @@ class Commands(cmd.Cmd):
         else:
             logger.info('No overlayed recipes found')
 
+    def do_flatten(self, args):
+        arglist = args.split()
+        if len(arglist) != 1:
+            logger.error('syntax: flatten <outputdir>')
+            return
+
+        if os.path.exists(arglist[0]) and os.listdir(arglist[0]):
+            logger.error('Directory %s exists and is non-empty, please clear it out first' % arglist[0])
+            return
+
+        layers = (self.config_data.getVar('BBLAYERS', True) or "").split()
+        for layer in layers:
+            overlayed = []
+            for f in self.cooker.overlayed.iterkeys():
+                for of in self.cooker.overlayed[f]:
+                    if of.startswith(layer):
+                        overlayed.append(of)
+
+            logger.info('Copying files from %s...' % layer )
+            for root, dirs, files in os.walk(layer):
+                for f1 in files:
+                    f1full = os.sep.join([root, f1])
+                    if f1full in overlayed:
+                        logger.info('  Skipping overlayed file %s' % f1full )
+                    else:
+                        ext = os.path.splitext(f1)[1]
+                        if ext != '.bbappend':
+                            fdest = f1full[len(layer):]
+                            fdest = os.path.normpath(os.sep.join([arglist[0],fdest]))
+                            bb.utils.mkdirhier(os.path.dirname(fdest))
+                            if os.path.exists(fdest):
+                                if f1 == 'layer.conf' and root.endswith('/conf'):
+                                    logger.info('  Skipping layer config file %s' % f1full )
+                                    continue
+                                else:
+                                    logger.warn('Overwriting file %s', fdest)
+                            bb.utils.copyfile(f1full, fdest)
+                            if ext == '.bb':
+                                if f1 in self.cooker_data.appends:
+                                    appends = self.cooker_data.appends[f1]
+                                    if appends:
+                                        logger.info('  Applying appends to %s' % fdest )
+                                        for appendname in appends:
+                                            self.apply_append(appendname, fdest)
+
+    def get_append_layer(self, appendname):
+        for layer, _, regex, _ in self.cooker.status.bbfile_config_priorities:
+            if regex.match(appendname):
+                return layer
+        return "?"
+
+    def apply_append(self, appendname, recipename):
+        appendfile = open(appendname, 'r')
+        recipefile = open(recipename, 'a')
+        recipefile.write('\n')
+        recipefile.write('##### bbappended from %s #####\n' % self.get_append_layer(appendname))
+        recipefile.writelines(appendfile.readlines())
+
     def do_show_appends(self, args):
         if not self.cooker_data.appends:
             logger.info('No append files found')

index 110e3c8..dfbdb70 100755
@@ -149,7 +149,9 @@ class Commands(cmd.Cmd):
 
         logger.info('State of append files:')
 
-        for pn in self.cooker_data.pkg_pn:
+        pnlist = list(self.cooker_data.pkg_pn.keys())
+        pnlist.sort( key=lambda item: item.pn )
+        for pn in pnlist:
             self.show_appends_for_pn(pn)
 
         self.show_appends_for_skipped()

@@ -188,7 +190,7 @@ class Commands(cmd.Cmd):
 
 
     def get_appends_for_files(self, filenames):
-        appended, notappended = set(), set()
+        appended, notappended = [], []
         for filename in filenames:
             _, cls = bb.cache.Cache.virtualfn2realfn(filename)
             if cls:

@@ -197,9 +199,9 @@ class Commands(cmd.Cmd):
             basename = os.path.basename(filename)
             appends = self.cooker_data.appends.get(basename)
             if appends:
-                appended.add((basename, frozenset(appends)))
+                appended.append((basename, list(appends)))
             else:
-                notappended.add(basename)
+                notappended.append(basename)
         return appended, notappended
 
     def show_appends_with_no_recipes(self):