Several fixes for layerindex

index d599ed5..526d2c2 100644
@@ -32,7 +32,7 @@ class LayerConfParse:
 
         self.config_data_copy = bb.data.createCopy(self.tinfoil.config_data)
 
-    def parse_layer(self, layerbranch, layerdir):
+    def parse_layer(self, layerdir):
 
         # This is not a valid layer, parsing will cause exception.
         if not utils.is_layer_valid(layerdir):

index ad16e40..a806dd1 100755
@@ -377,7 +377,7 @@ def main():
                         layerdep.save()
                     layerconfparser = LayerConfParse(logger=logger)
                     try:
-                        config_data = layerconfparser.parse_layer(layerbranch, layerdir)
+                        config_data = layerconfparser.parse_layer(layerdir)
                         if config_data:
                             utils.add_dependencies(layerbranch, config_data, logger=logger)
                             utils.add_recommends(layerbranch, config_data, logger=logger)

index 9fe96fb..05ab3e7 100644
@@ -267,7 +267,7 @@ def main():
                     sys.exit(0)
 
                 layerconfparser = layerconfparse.LayerConfParse(logger=logger, bitbakepath=bitbakepath, tinfoil=tinfoil)
-                config_data = layerconfparser.parse_layer(layerbranch, layerdir)
+                config_data = layerconfparser.parse_layer(layerdir)
                 if not config_data:
                     logger.debug("Layer %s does not appear to be valid for branch %s" % (layer.name, branchdesc))
                     sys.exit(0)

@@ -355,7 +355,7 @@ def main():
                 logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
 
                 layerconfparser = layerconfparse.LayerConfParse(logger=logger, tinfoil=tinfoil)
-                layer_config_data = layerconfparser.parse_layer(layerbranch, layerdir)
+                layer_config_data = layerconfparser.parse_layer(layerdir)
                 if not layer_config_data:
                     logger.info("Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc))
                     layerconfparser.shutdown()

index 3b45b07..1a57c07 100644
@@ -88,6 +88,7 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T
         logger.debug('Error parsing %s_%s for %s\n%s' % (var, var_name, layer_name, str(vse)))
         return
 
+    need_delete = None
     for dep, ver_list in list(dep_dict.items()):
         ver_str = None
         if ver_list:

@@ -106,8 +107,14 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T
                 logger.error('Cannot resolve %s %s (version %s) for %s' % (name, dep, ver_str, layer_name))
                 continue
 
+        # Preparing to remove obsolete ones
+        if not need_delete:
+            need_delete = LayerDependency.objects.filter(layerbranch=layerbranch).filter(required=required).exclude(dependency=dep_layer)
+        else:
+            need_delete = need_delete.exclude(dependency=dep_layer)
+
         # Skip existing entries.
-        existing = list(LayerDependency.objects.filter(layerbranch=layerbranch).filter(dependency=dep_layer))
+        existing = list(LayerDependency.objects.filter(layerbranch=layerbranch).filter(required=required).filter(dependency=dep_layer))
         if existing:
             logger.debug('Skipping %s - already a dependency for %s' % (dep, layer_name))
             continue

@@ -121,6 +128,10 @@ def _add_dependency(var, name, layerbranch, config_data, logger=None, required=T
         layerdep.required = required
         layerdep.save()
 
+    if need_delete:
+        logger.debug("Removing obsolete dependencies: %s" % need_delete)
+        need_delete.delete()
+
 def set_layerbranch_collection_version(layerbranch, config_data, logger=None):
     layerbranch.collection = config_data.getVar('BBFILE_COLLECTIONS', True)
     ver_str = "LAYERVERSION_"

index 05ab3e7..bcf7056 100644
@@ -360,9 +360,9 @@ def main():
                     logger.info("Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc))
                     layerconfparser.shutdown()
                     sys.exit(1)
+                utils.set_layerbranch_collection_version(layerbranch, layer_config_data, logger=logger)
                 utils.add_dependencies(layerbranch, layer_config_data, logger=logger)
                 utils.add_recommends(layerbranch, layer_config_data, logger=logger)
-                utils.set_layerbranch_collection_version(layerbranch, layer_config_data, logger=logger)
                 layerbranch.save()
 
                 try:

index d5c56cd..9c131cb 100755
@@ -202,7 +202,7 @@ def main():
                             if not os.path.exists(repodir):
                                 out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger, printerr=False)
                             else:
-                                out = utils.runcmd("git fetch", repodir, logger=logger, printerr=False)
+                                out = utils.runcmd("git fetch -p", repodir, logger=logger, printerr=False)
                         except subprocess.CalledProcessError as e:
                             logger.error("Fetch of layer %s failed: %s" % (layer.name, e.output))
                             failedrepos[layer.vcs_url] = e.output

@@ -217,7 +217,7 @@ def main():
                 if not os.path.exists(bitbakepath):
                     out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger)
                 else:
-                    out = utils.runcmd("git fetch", bitbakepath, logger=logger)
+                    out = utils.runcmd("git fetch -p", bitbakepath, logger=logger)
 
             # Process and extract data from each layer
             # We now do this by calling out to a separate script; doing otherwise turned out to be

index 9c131cb..541155f 100755
@@ -152,10 +152,13 @@ def main():
         sys.exit(1)
 
     if options.layers:
-        layerquery = LayerItem.objects.filter(classic=False).filter(name__in=options.layers.split(','))
-        if layerquery.count() == 0:
-            logger.error('No layers matching specified query "%s"' % options.layers)
-            sys.exit(1)
+        layers = options.layers.split(',')
+        for layer in layers:
+            layerquery = LayerItem.objects.filter(classic=False).filter(name=layer)
+            if layerquery.count() == 0:
+                logger.error('No layers matching specified query "%s"' % layer)
+                sys.exit(1)
+        layerquery = LayerItem.objects.filter(classic=False).filter(name__in=layers)
     else:
         # We deliberately exclude status == 'X' ("no update") here
         layerquery = LayerItem.objects.filter(classic=False).filter(status='P')

index dd85bc3..f2a5235 100644
@@ -70,10 +70,13 @@ def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout
     if not os.path.exists(settings.TEMP_BASE_DIR):
         os.makedirs(settings.TEMP_BASE_DIR)
     tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
+    saved_cwd = os.getcwd()
     os.chdir(tempdir)
 
     tinfoil = utils.setup_tinfoil(bitbakepath, enable_tracking)
 
+    os.chdir(saved_cwd)
+
     # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log)
     oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True)
     if not os.path.exists(oe_tmpdir):

index 541155f..f847b85 100755
@@ -18,6 +18,8 @@ import signal
 from datetime import datetime, timedelta
 from distutils.version import LooseVersion
 import utils
+import operator
+import recipeparse
 
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)

@@ -151,6 +153,11 @@ def main():
         logger.error("Please set LAYER_FETCH_DIR in settings.py")
         sys.exit(1)
 
+    layerquery_all = LayerItem.objects.filter(classic=False).filter(status='P')
+    if layerquery_all.count() == 0:
+        logger.info("No published layers to update")
+        sys.exit(1)
+
     if options.layers:
         layers = options.layers.split(',')
         for layer in layers:

@@ -161,10 +168,7 @@ def main():
         layerquery = LayerItem.objects.filter(classic=False).filter(name__in=layers)
     else:
         # We deliberately exclude status == 'X' ("no update") here
-        layerquery = LayerItem.objects.filter(classic=False).filter(status='P')
-        if layerquery.count() == 0:
-            logger.info("No published layers to update")
-            sys.exit(1)
+        layerquery = layerquery_all
 
     if not os.path.exists(fetchdir):
         os.makedirs(fetchdir)

@@ -186,6 +190,7 @@ def main():
     try:
         lockfn = os.path.join(fetchdir, "layerindex.lock")
         lockfile = utils.lock_file(lockfn)
+        tinfoil = None
         if not lockfile:
             logger.error("Layer index lock timeout expired")
             sys.exit(1)

@@ -228,8 +233,82 @@ def main():
             # they never get used during normal operation).
             last_rev = {}
             for branch in branches:
+                # If layer_A depends(or recommends) on layer_B, add layer_B before layer_A
+                deps_dict_all = {}
+                layerquery_sorted = []
+                collections_done = set()
                 branchobj = utils.get_branch(branch)
+                try:
+                    (tinfoil, tempdir) = recipeparse.init_parser(settings, branchobj, bitbakepath, nocheckout=options.nocheckout, logger=logger)
+                except recipeparse.RecipeParseError as e:
+                    logger.error(str(e))
+                    sys.exit(1)
+                for layer in layerquery_all:
+                    # Get all collections from database, but we can't trust the
+                    # one which will be updated since its collections maybe
+                    # changed (different from database).
+                    if layer in layerquery:
+                        continue
+                    layerbranch = layer.get_layerbranch(branch)
+                    if layerbranch:
+                        collections_done.add((layerbranch.collection, layerbranch.version))
+
                 for layer in layerquery:
+                    errmsg = failedrepos.get(layer.vcs_url, '')
+                    if errmsg:
+                        continue
+                    config_data = utils.copy_tinfoil_data(tinfoil.config_data)
+                    layerbranch_source = layer.get_layerbranch(None)
+                    if not layerbranch_source:
+                        logger.error('Failed to get layerbranch_source for %s' % layer.name)
+                        sys.exit(1)
+                    urldir = layer.get_fetch_dir()
+                    repodir = os.path.join(fetchdir, urldir)
+                    layerdir = os.path.join(repodir, layerbranch_source.vcs_subdir)
+                    utils.parse_layer_conf(layerdir, config_data, logger=logger)
+
+                    deps = utils.get_layer_var(config_data, 'LAYERDEPENDS') or ''
+                    recs = utils.get_layer_var(config_data, 'LAYERRECOMMENDS') or ''
+                    col = (utils.get_layer_var(config_data, 'BBFILE_COLLECTIONS') or '').strip()
+                    ver = utils.get_layer_var(config_data, 'LAYERVERSION') or ''
+
+                    deps_dict = utils.explode_dep_versions2(deps + ' ' + recs)
+                    if len(deps_dict) == 0:
+                        # No depends, add it firstly
+                        layerquery_sorted.append(layer)
+                        collections_done.add((col, ver))
+                        continue
+                    deps_dict_all[layer] = {'requires': deps_dict, 'collection': col, 'version': ver}
+
+                # Move deps_dict_all to layerquery_sorted orderly
+                logger.info("Sorting layers for branch %s" % branch)
+                while True:
+                    deps_dict_all_copy = deps_dict_all.copy()
+                    for layer, value in deps_dict_all_copy.items():
+                        for req_col, req_ver_list in value['requires'].copy().items():
+                            matched = False
+                            if req_ver_list:
+                                req_ver = req_ver_list[0]
+                            else:
+                                req_ver = None
+                            if utils.is_deps_satisfied(req_col, req_ver, collections_done):
+                                del(value['requires'][req_col])
+                        if not value['requires']:
+                            # All the depends are in collections_done:
+                            del(deps_dict_all[layer])
+                            layerquery_sorted.append(layer)
+                            collections_done.add((value['collection'], value['version']))
+
+                    if not len(deps_dict_all):
+                        break
+
+                    # Something is wrong if nothing changed after a run
+                    if operator.eq(deps_dict_all_copy, deps_dict_all):
+                        logger.error("The dependencies of %s can't be satisfied on branch %s: %s" % (branch, value['collection'], deps_dict_all))
+                        logger.error("Known collections: %s" % collections_done)
+                        sys.exit(1)
+
+                for layer in layerquery_sorted:
                     layerupdate = LayerUpdate()
                     layerupdate.update = update
 

@@ -269,33 +348,10 @@ def main():
                     if ret == 254:
                         # Interrupted by user, break out of loop
                         break
-
-            # Since update_layer may not be called in the correct order to have the
-            # dependencies created before trying to link them, we now have to loop
-            # back through all the branches and layers and try to link in the
-            # dependencies that may have been missed.  Note that creating the
-            # dependencies is a best-effort and continues if they are not found.
-            for branch in branches:
-                branchobj = utils.get_branch(branch)
-                for layer in layerquery:
-                    layerbranch = layer.get_layerbranch(branch)
-                    if layerbranch:
-                        if not (options.reload or options.fullreload):
-                            # Skip layers that did not change.
-                            layer_last_rev = last_rev.get(layerbranch, None)
-                            if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev:
-                                continue
-
-                        logger.info('Updating layer dependencies for %s on branch %s' % (layer.name, branch))
-                        cmd = prepare_update_layer_command(options, branchobj, layer, updatedeps=True)
-                        logger.debug('Running update dependencies command: %s' % cmd)
-                        ret, output = run_command_interruptible(cmd)
-                        if ret == 254:
-                            # Interrupted by user, break out of loop
-                            break
-
         finally:
             utils.unlock_file(lockfile)
+            if tinfoil:
+                tinfoil.shutdown()
 
     finally:
         update.log = ''.join(listhandler.read())

index 1a57c07..c9861b9 100644
@@ -27,6 +27,33 @@ def get_layer(layername):
         return res[0]
     return None
 
+def get_layer_var(config_data, var):
+    collection = config_data.getVar('BBFILE_COLLECTIONS', True)
+    if collection:
+        collection = collection.strip()
+    value = config_data.getVar('%s_%s' % (var, collection), True)
+    if not value:
+        value = config_data.getVar(var, True)
+    return value
+
+def is_deps_satisfied(req_col, req_ver, collections):
+    """ Check whether required collection and version are in collections"""
+    for existed_col, existed_ver in collections:
+        if req_col == existed_col:
+            # If there is no version constraint, return True when collection matches
+            if not req_ver:
+                return True
+            else:
+                # If there is no version in the found layer, then don't use this layer.
+                if not existed_ver:
+                    continue
+                (op, dep_version) = req_ver.split()
+                success = bb.utils.vercmp_string_op(existed_ver, dep_version, op)
+                if success:
+                    return True
+    # Return False when not found
+    return False
+
 def get_dependency_layer(depname, version_str=None, logger=None):
     from layerindex.models import LayerItem, LayerBranch
 

@@ -156,6 +183,14 @@ def setup_tinfoil(bitbakepath, enable_tracking):
 
     return tinfoil
 
+def copy_tinfoil_data(config_data):
+    import bb.data
+    return bb.data.createCopy(config_data)
+
+def explode_dep_versions2(deps):
+    import bb.utils
+    return bb.utils.explode_dep_versions2(deps)
+
 def checkout_layer_branch(layerbranch, repodir, logger=None):
 
     branchname = layerbranch.branch.name

index f847b85..1347e70 100755
@@ -87,6 +87,41 @@ def prepare_update_layer_command(options, branch, layer, updatedeps=False):
         cmd += ' -q'
     return cmd
 
+def update_actual_branch(layerquery, fetchdir, branch, actual_branch, update_bitbake, bitbakepath):
+    """Update actual branch for layers and bitbake in database"""
+    to_save = set()
+    if update_bitbake:
+        branchobj = utils.get_branch(branch)
+        if actual_branch != branchobj.bitbake_branch:
+            if utils.is_branch_valid(bitbakepath, actual_branch):
+                logger.info("bitbake: %s.bitbake_branch: %s -> %s" % (branch, branchobj.bitbake_branch, actual_branch))
+                branchobj.bitbake_branch = actual_branch
+                to_save.add(branchobj)
+            else:
+                logger.info("Skipping update bitbake_branch for bitbake - branch %s doens't exist" % actual_branch)
+        else:
+            logger.info("bitbake: %s.bitbake_branch is already %s, so no change" % (branch, actual_branch))
+
+    for layer in layerquery:
+        urldir = layer.get_fetch_dir()
+        repodir = os.path.join(fetchdir, urldir)
+        if not utils.is_branch_valid(repodir, actual_branch):
+            logger.info("Skipping update actual_branch for %s - branch %s doens't exist" % (layer.name, actual_branch))
+            continue
+        layerbranch = layer.get_layerbranch(branch)
+        if not layerbranch:
+            logger.info("Skipping update actual_branch for %s - layerbranch %s doens't exist" % (layer.name, branch))
+            continue
+        if actual_branch != layerbranch.actual_branch:
+            logger.info("%s: %s.actual_branch: %s -> %s" % (layer.name, branch, layerbranch.actual_branch, actual_branch))
+            layerbranch.actual_branch = actual_branch
+            to_save.add(layerbranch)
+        else:
+            logger.info("%s: %s.actual_branch is already %s, so no change" % (layer.name, branch, actual_branch))
+
+    # At last, do the save
+    for s in to_save:
+        s.save()
 
 def main():
     if LooseVersion(git.__version__) < '0.3.1':

@@ -104,6 +139,9 @@ def main():
     parser.add_option("-l", "--layer",
             help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
             action="store", dest="layers")
+    parser.add_option("-a", "--actual-branch",
+            help = "Update actual branch for layer and bitbake",
+            action="store", dest="actual_branch", default='')
     parser.add_option("-r", "--reload",
             help = "Reload recipe data instead of updating since last update",
             action="store_true", dest="reload")

@@ -158,8 +196,13 @@ def main():
         logger.info("No published layers to update")
         sys.exit(1)
 
+    # For -a option to update bitbake branch
+    update_bitbake = False
     if options.layers:
         layers = options.layers.split(',')
+        if 'bitbake' in layers:
+            update_bitbake = True
+            layers.remove('bitbake')
         for layer in layers:
             layerquery = LayerItem.objects.filter(classic=False).filter(name=layer)
             if layerquery.count() == 0:

@@ -168,6 +211,7 @@ def main():
         layerquery = LayerItem.objects.filter(classic=False).filter(name__in=layers)
     else:
         # We deliberately exclude status == 'X' ("no update") here
+        update_bitbake = True
         layerquery = layerquery_all
 
     if not os.path.exists(fetchdir):

@@ -217,7 +261,7 @@ def main():
                             continue
                         fetchedrepos.append(layer.vcs_url)
 
-                if not fetchedrepos:
+                if not (fetchedrepos or update_bitbake):
                     logger.error("No repositories could be fetched, exiting")
                     sys.exit(1)
 

@@ -227,6 +271,16 @@ def main():
                 else:
                     out = utils.runcmd("git fetch -p", bitbakepath, logger=logger)
 
+            if options.actual_branch:
+                if not options.branch:
+                    logger.error("-a option requires -b")
+                    sys.exit(1)
+                elif len(branches) != 1:
+                    logger.error("Only one branch should be used with -a")
+                    sys.exit(1)
+                update_actual_branch(layerquery, fetchdir, branches[0], options.actual_branch, update_bitbake, bitbakepath)
+                return
+
             # Process and extract data from each layer
             # We now do this by calling out to a separate script; doing otherwise turned out to be
             # unreliable due to leaking memory (we're using bitbake internals in a manner in which

index c9861b9..ab0dea7 100644
@@ -206,6 +206,17 @@ def is_layer_valid(layerdir):
         return False
     return True
 
+def is_branch_valid(layerdir, branch):
+    import git
+
+    g = git.cmd.Git(layerdir)
+    assert g.rev_parse('--is-bare-repository') == 'false'
+    try:
+        g.rev_parse('--verify', 'origin/%s' % branch)
+    except Exception:
+        return False
+    return True
+
 def parse_conf(conf_file, d):
     if hasattr(bb.parse, "handle"):
         # Newer BitBake