toaster: Import external layers outside poky from toasterconf.json

index a8d8398..2b3c452 100644
@@ -192,6 +192,21 @@ class LocalhostBEController(BuildEnvironmentController):
         #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
         return local_checkout_path
 
+    def getHeadLayers(self, gitrepos, layers):
+        """ HEAD layers are special cases. They don't need to be cloned. """
+
+        layerlist = []
+        headlayers = [commit for giturl, commit in gitrepos.keys() if commit == "HEAD"]
+        if set(headlayers) == set(["HEAD"]):
+           # Populate layerlist if all of them are HEAD
+            for layer_version in Layer_Version.objects.all().filter(local_path__startswith = "/"):
+                if layer_version.commit == "HEAD":
+                    if layer_version.local_path not in layerlist:
+                        layerlist.append(layer_version.local_path)
+                        if os.path.exists(os.path.join(os.path.dirname(layer_version.local_path), "oe-init-build-env")):
+                            self.pokydirname = os.path.dirname(layer_version.local_path)
+        return layerlist
+
 
     def setLayers(self, bitbakes, layers, targets):
         """ a word of attention: by convention, the first layer for any build will be poky! """

@@ -239,51 +254,58 @@ class LocalhostBEController(BuildEnvironmentController):
         layerlist = []
 
 
-        # 3. checkout the repositories
-        for giturl, commit in gitrepos.keys():
-            localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
-            logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
+        # Verify HEAD layers
 
-            # make sure our directory is a git repository
-            if os.path.exists(localdirname):
-                localremotes = self._shellcmd("git remote -v", localdirname)
-                if not giturl in localremotes:
-                    raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
-            else:
-                if giturl in cached_layers:
-                    logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
-                    self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
-                    self._shellcmd("git remote remove origin", localdirname)
-                    self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+        layerlist = self.getHeadLayers(gitrepos,layers)
+
+        # If the layerlist doesn't have already cloned layers then checkout repositories
+        if len(layerlist) == 0:
+
+            # 3. checkout the repositories
+            for giturl, commit in gitrepos.keys():
+                localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
+                logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
+
+                # make sure our directory is a git repository
+                if os.path.exists(localdirname):
+                    localremotes = self._shellcmd("git remote -v", localdirname)
+                    if not giturl in localremotes:
+                        raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
                 else:
-                    logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
-                    self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
-
-            # branch magic name "HEAD" will inhibit checkout
-            if commit != "HEAD":
-                logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
-                ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
-                self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
-
-            # take the localdirname as poky dir if we can find the oe-init-build-env
-            if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
-                logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
-                self.pokydirname = localdirname
-
-                # make sure we have a working bitbake
-                if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
-                    logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
-                    self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
-
-            # verify our repositories
-            for name, dirpath in gitrepos[(giturl, commit)]:
-                localdirpath = os.path.join(localdirname, dirpath)
-                logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
-                if not os.path.exists(localdirpath):
-                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
-
-                if name != "bitbake":
-                    layerlist.append(localdirpath.rstrip("/"))
+                    if giturl in cached_layers:
+                        logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
+                        self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
+                        self._shellcmd("git remote remove origin", localdirname)
+                        self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+                    else:
+                        logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
+                        self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
+
+                # branch magic name "HEAD" will inhibit checkout
+                if commit != "HEAD":
+                    logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
+                    ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
+                    self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
+
+                # take the localdirname as poky dir if we can find the oe-init-build-env
+                if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
+                    logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
+                    self.pokydirname = localdirname
+
+                    # make sure we have a working bitbake
+                    if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
+                        logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
+                        self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
+
+                # verify our repositories
+                for name, dirpath in gitrepos[(giturl, commit)]:
+                    localdirpath = os.path.join(localdirname, dirpath)
+                    logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+                    if not os.path.exists(localdirpath):
+                        raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+
+                    if name != "bitbake":
+                        layerlist.append(localdirpath.rstrip("/"))
 
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 

index 5022b59..21bc380 100644
@@ -45,12 +45,14 @@ class Command(BaseCommand):
         for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
             assert i in data
 
-        def _read_git_url_from_local_repository(address):
+        def _read_git_url_from_local_repository(address, path = None):
             url = None
+            if not path:
+                path = os.path.dirname(filepath)
             # we detect the remote name at runtime
             import subprocess
             (remote, remote_name) = address.split(":", 1)
-            cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
+            cmd = subprocess.Popen("git remote -v", shell=True, cwd = path, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
             (out,err) = cmd.communicate()
             if cmd.returncode != 0:
                 logging.warning("Error while importing layer vcs_url: git error: %s" % err)

@@ -121,7 +123,11 @@ class Command(BaseCommand):
                         logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
 
                     if layerinfo['vcs_url'].startswith("remote:"):
-                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
+                        if not layerinfo['local_path'].startswith("/"):
+                           path = None
+                        else:
+                           path = layerinfo['local_path']
+                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'], path)
                         if lo.vcs_url is None:
                             logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
 

@@ -138,6 +144,8 @@ class Command(BaseCommand):
                                 commit = branch.name,
                                 layer = lo)
                         lvo.dirpath = layerinfo['dirpath']
+                        if len(layerinfo['local_path']) > 1 and layerinfo['local_path'].startswith("/") and branch.name == "HEAD":
+                            lvo.local_path = layerinfo['local_path']
                         lvo.save()
         # set releases
         for ri in data['releases']:

index 9fdde29..5e924ca 100644
@@ -373,7 +373,7 @@ class ORMWrapper(object):
                 if not localdirname.startswith("/"):
                     localdirname = os.path.join(bc.be.sourcedir, localdirname)
                 #logger.debug(1, "Localdirname %s lcal_path %s" % (localdirname, layer_information['local_path']))
-                if localdirname.startswith(layer_information['local_path']):
+                if localdirname.startswith(layer_information['local_path']) or os.path.exists(layer_information['local_path']):
                   # If the build request came from toaster this field
                   # should contain the information from the layer_version
                   # That created this build request.

@@ -383,13 +383,14 @@ class ORMWrapper(object):
                     # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build()
                     #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname))
 
-                    for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name):
+                    for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = layer_information['name']):
                         if pl.layercommit.layer.vcs_url == brl.giturl :
                             layer = pl.layercommit.layer
                             layer.save()
                             return layer
 
-            raise NotExisting("Unidentified layer %s" % pformat(layer_information))
+            if os.path.exists(localdirname) == False and os.path.exists(layer_information['local_path']) == False:
+               raise NotExisting("Unidentified layer %s" % pformat(layer_information))
 
 
     def save_target_file_information(self, build_obj, target_obj, filedata):

@@ -813,7 +814,7 @@ class BuildInfoHelper(object):
                 # we get a relative path, unless running in HEAD mode where the path is absolute
                 if not localdirname.startswith("/"):
                     localdirname = os.path.join(bc.be.sourcedir, localdirname)
-                if path.startswith(localdirname):
+                if path.startswith(localdirname) or (brl.dirpath in path):
                     # If the build request came from toaster this field
                     # should contain the information from the layer_version
                     # That created this build request.

index 5e924ca..7018e4e 100644
@@ -377,7 +377,7 @@ class ORMWrapper(object):
                   # If the build request came from toaster this field
                   # should contain the information from the layer_version
                   # That created this build request.
-                    if brl.layer_version:
+                    if (layer_information['name'] == brl.name) and (brl.layer_version):
                         return brl.layer_version
 
                     # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build()

@@ -511,8 +511,9 @@ class ORMWrapper(object):
         errormsg = ""
         for p in packagedict:
             searchname = p
-            if 'OPKGN' in pkgpnmap[p].keys():
-                searchname = pkgpnmap[p]['OPKGN']
+            if p in pkgpnmap:
+                if 'OPKGN' in pkgpnmap[p].keys():
+                    searchname = pkgpnmap[p]['OPKGN']
 
             packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = searchname )
             if created or packagedict[p]['object'].size == -1:    # save the data anyway we can, not just if it was not created here; bug [YOCTO #6887]