From: Sujith H <sujith.h@gmail.com>
To: toaster@yoctoproject.org
Date: Mon,  5 Oct 2015 20:23:50 +0530
Message-Id: <1444056832-25706-1-git-send-email-sujith.h@gmail.com>
Subject: [review-request][PATCH 1/3] toaster: Import external layers outside poky from toasterconf.json
Precedence: list

This patch helps to import external layers from toasterconf.json
which are already checked out. The branch of the checkedout layers
is HEAD. Hence no checkout happens when toaster is using the configuration.
This essentially speeds up toaster, while using locally checked out layers.

Signed-off-by: Sujith H <sujith.h@gmail.com>
Signed-off-by: Sujith Haridasan <Sujith_Haridasan@mentor.com>
---
 .../toaster/bldcontrol/localhostbecontroller.py    | 108 +++++++++++++--------
 .../bldcontrol/management/commands/loadconf.py     |  14 ++-
 2 files changed, 76 insertions(+), 46 deletions(-)

diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index a8d8398..2b3c452 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -192,6 +192,21 @@ class LocalhostBEController(BuildEnvironmentController):
         #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
         return local_checkout_path
 
+    def getHeadLayers(self, gitrepos, layers):
+        """ HEAD layers are special cases. They don't need to be cloned. """
+
+        layerlist = []
+        headlayers = [commit for giturl, commit in gitrepos.keys() if commit == "HEAD"]
+        if set(headlayers) == set(["HEAD"]):
+           # Populate layerlist if all of them are HEAD
+            for layer_version in Layer_Version.objects.all().filter(local_path__startswith = "/"):
+                if layer_version.commit == "HEAD":
+                    if layer_version.local_path not in layerlist:
+                        layerlist.append(layer_version.local_path)
+                        if os.path.exists(os.path.join(os.path.dirname(layer_version.local_path), "oe-init-build-env")):
+                            self.pokydirname = os.path.dirname(layer_version.local_path)
+        return layerlist
+
 
     def setLayers(self, bitbakes, layers, targets):
         """ a word of attention: by convention, the first layer for any build will be poky! """
@@ -239,51 +254,58 @@ class LocalhostBEController(BuildEnvironmentController):
         layerlist = []
 
 
-        # 3. checkout the repositories
-        for giturl, commit in gitrepos.keys():
-            localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
-            logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
+        # Verify HEAD layers
 
-            # make sure our directory is a git repository
-            if os.path.exists(localdirname):
-                localremotes = self._shellcmd("git remote -v", localdirname)
-                if not giturl in localremotes:
-                    raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
-            else:
-                if giturl in cached_layers:
-                    logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
-                    self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
-                    self._shellcmd("git remote remove origin", localdirname)
-                    self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+        layerlist = self.getHeadLayers(gitrepos,layers)
+
+        # If the layerlist doesn't have already cloned layers then checkout repositories
+        if len(layerlist) == 0:
+
+            # 3. checkout the repositories
+            for giturl, commit in gitrepos.keys():
+                localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
+                logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
+
+                # make sure our directory is a git repository
+                if os.path.exists(localdirname):
+                    localremotes = self._shellcmd("git remote -v", localdirname)
+                    if not giturl in localremotes:
+                        raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
                 else:
-                    logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
-                    self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
-
-            # branch magic name "HEAD" will inhibit checkout
-            if commit != "HEAD":
-                logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
-                ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
-                self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
-
-            # take the localdirname as poky dir if we can find the oe-init-build-env
-            if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
-                logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
-                self.pokydirname = localdirname
-
-                # make sure we have a working bitbake
-                if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
-                    logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
-                    self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
-
-            # verify our repositories
-            for name, dirpath in gitrepos[(giturl, commit)]:
-                localdirpath = os.path.join(localdirname, dirpath)
-                logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
-                if not os.path.exists(localdirpath):
-                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
-
-                if name != "bitbake":
-                    layerlist.append(localdirpath.rstrip("/"))
+                    if giturl in cached_layers:
+                        logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
+                        self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
+                        self._shellcmd("git remote remove origin", localdirname)
+                        self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+                    else:
+                        logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
+                        self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
+
+                # branch magic name "HEAD" will inhibit checkout
+                if commit != "HEAD":
+                    logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
+                    ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
+                    self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
+
+                # take the localdirname as poky dir if we can find the oe-init-build-env
+                if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
+                    logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
+                    self.pokydirname = localdirname
+
+                    # make sure we have a working bitbake
+                    if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
+                        logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
+                        self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
+
+                # verify our repositories
+                for name, dirpath in gitrepos[(giturl, commit)]:
+                    localdirpath = os.path.join(localdirname, dirpath)
+                    logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+                    if not os.path.exists(localdirpath):
+                        raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+
+                    if name != "bitbake":
+                        layerlist.append(localdirpath.rstrip("/"))
 
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
index 5022b59..21bc380 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -45,12 +45,14 @@ class Command(BaseCommand):
         for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
             assert i in data
 
-        def _read_git_url_from_local_repository(address):
+        def _read_git_url_from_local_repository(address, path = None):
             url = None
+            if not path:
+                path = os.path.dirname(filepath)
             # we detect the remote name at runtime
             import subprocess
             (remote, remote_name) = address.split(":", 1)
-            cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
+            cmd = subprocess.Popen("git remote -v", shell=True, cwd = path, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
             (out,err) = cmd.communicate()
             if cmd.returncode != 0:
                 logging.warning("Error while importing layer vcs_url: git error: %s" % err)
@@ -121,7 +123,11 @@ class Command(BaseCommand):
                         logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
 
                     if layerinfo['vcs_url'].startswith("remote:"):
-                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
+                        if not layerinfo['local_path'].startswith("/"):
+                           path = None
+                        else:
+                           path = layerinfo['local_path']
+                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'], path)
                         if lo.vcs_url is None:
                             logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
 
@@ -138,6 +144,8 @@ class Command(BaseCommand):
                                 commit = branch.name,
                                 layer = lo)
                         lvo.dirpath = layerinfo['dirpath']
+                        if len(layerinfo['local_path']) > 1 and layerinfo['local_path'].startswith("/") and branch.name == "HEAD":
+                            lvo.local_path = layerinfo['local_path']
                         lvo.save()
         # set releases
         for ri in data['releases']:
-- 
1.9.1