Return-Path: Received: by yocto-www.yoctoproject.org (Postfix, from userid 118) id B018DE00BF3; Tue, 15 Sep 2015 07:26:47 -0700 (PDT) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on yocto-www.yoctoproject.org X-Spam-Level: X-Spam-Status: No, score=-2.7 required=5.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, FREEMAIL_FROM, RCVD_IN_DNSWL_LOW autolearn=ham version=3.3.1 X-Spam-HAM-Report: * 0.0 FREEMAIL_FROM Sender email is commonly abused enduser mail provider * (sujith.h[at]gmail.com) * -0.7 RCVD_IN_DNSWL_LOW RBL: Sender listed at http://www.dnswl.org/, low * trust * [209.85.212.179 listed in list.dnswl.org] * -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% * [score: 0.0000] * -0.1 DKIM_VALID_AU Message has a valid DKIM or DK signature from author's * domain * 0.1 DKIM_SIGNED Message has a DKIM or DK signature, not necessarily * valid * -0.1 DKIM_VALID Message has at least one valid DKIM or DK signature Received: from mail-wi0-f179.google.com (mail-wi0-f179.google.com [209.85.212.179]) by yocto-www.yoctoproject.org (Postfix) with ESMTP id 577EEE00B51 for ; Tue, 15 Sep 2015 07:26:45 -0700 (PDT) Received: by wiclk2 with SMTP id lk2so30063372wic.1 for ; Tue, 15 Sep 2015 07:26:44 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=from:to:cc:subject:date:message-id; bh=IjQ4w9BalQ/GuF6/WvetaY7r3wvB77KrZrpGbCXToUs=; b=i/4ux6LAHGJz2Dr1Ks00oduAzJAyaYt4mmIDK5P1ZGqV3I2RN5DpTgQZ8c3PYytxTv KYxnBDkG532/c+KtNsx0B5x5zK0x9SMofHJhsVZtA2KcusPFKTHlou5FNYqPMV0rozyi idW7tjbwB39aLUHZmYRHlJnUTaMdi5EPdD64rXAXoe9x44UrGaTVfaak7976Mgo9kBXW A/PjrByUAtjIP7AgwlPy0tWDd4Hh4htSXX5XMW+HKZ2AM9gaE4SE3CbiLxsrcHIwcTSE 29udFCnmVMTrIlfNNdA1zoK5n6l25YvOFzkI8kH40YCBpM5pmM5MDX+BCCXOIBjJC8Ku 8I+g== X-Received: by 10.194.248.234 with SMTP id yp10mr10989242wjc.24.1442327204703; Tue, 15 Sep 2015 07:26:44 -0700 (PDT) Received: from kdekidd0.inp-wifi.mentorg.com ([139.181.35.34]) by smtp.gmail.com with ESMTPSA id gt4sm20204842wib.21.2015.09.15.07.26.41 (version=TLSv1.2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); Tue, 15 Sep 2015 07:26:44 -0700 (PDT) From: Sujith H To: toaster@yoctoproject.org Date: Tue, 15 Sep 2015 19:55:53 +0530 Message-Id: <1442327153-21743-1-git-send-email-sujith.h@gmail.com> X-Mailer: git-send-email 1.9.1 Subject: [review-request][PATCH] toaster: Import external layers outside poky from toasterconf.json X-BeenThere: toaster@yoctoproject.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: Web based interface for BitBake List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 15 Sep 2015 14:26:47 -0000 This patch helps to import external layers from toasterconf.json which are already checked out. The branch of the checkedout layers is HEAD. Hence no checkout happens when toaster is using the configuration. This essentially speeds up toaster, while using locally checked out layers. Signed-off-by: Sujith H Signed-off-by: Sujith Haridasan --- .../toaster/bldcontrol/localhostbecontroller.py | 112 +++++++++++++-------- .../bldcontrol/management/commands/loadconf.py | 14 ++- 2 files changed, 79 insertions(+), 47 deletions(-) diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 231a7d3..efa8b11 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -32,6 +32,8 @@ from toastermain import settings from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException +from orm.models import Layer_Version + import logging logger = logging.getLogger("toaster") @@ -196,6 +198,21 @@ class LocalhostBEController(BuildEnvironmentController): #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) return local_checkout_path + def getHeadLayers(self, gitrepos, layers): + """ HEAD layers are special cases. They don't need to be cloned. """ + + layerlist = [] + headlayers = [commit for giturl, commit in gitrepos.keys() if commit == "HEAD"] + if set(headlayers) == set(["HEAD"]): + # Populate layerlist if all of them are HEAD + for layer_version in Layer_Version.objects.all().filter(local_path__startswith = "/"): + if layer_version.commit == "HEAD": + if layer_version.local_path not in layerlist: + layerlist.append(layer_version.local_path) + if os.path.exists(os.path.join(os.path.dirname(layer_version.local_path), "oe-init-build-env")): + self.pokydirname = os.path.dirname(layer_version.local_path) + return layerlist + def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ @@ -242,51 +259,58 @@ class LocalhostBEController(BuildEnvironmentController): layerlist = [] + # Verify HEAD layers + + layerlist = self.getHeadLayers(gitrepos,layers) + + # If the layerlist doesn't have already cloned layers then checkout repositories + if len(layerlist) == 0: + + # 3. checkout the repositories + for giturl, commit in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) + logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) + + # make sure our directory is a git repository + if os.path.exists(localdirname): + localremotes = self._shellcmd("git remote -v", localdirname) + if not giturl in localremotes: + raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) + else: + if giturl in cached_layers: + logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) + self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) + self._shellcmd("git remote remove origin", localdirname) + self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) + else: + logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) + self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) + + # branch magic name "HEAD" will inhibit checkout + if commit != "HEAD": + logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) + self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname) + + # take the localdirname as poky dir if we can find the oe-init-build-env + if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): + logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) + self.pokydirname = localdirname + + # make sure we have a working bitbake + if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): + logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) + + # verify our repositories + for name, dirpath in gitrepos[(giturl, commit)]: + localdirpath = os.path.join(localdirname, dirpath) + logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) + if not os.path.exists(localdirpath): + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + + if name != "bitbake": + layerlist.append(localdirpath.rstrip("/")) - # 3. checkout the repositories - for giturl, commit in gitrepos.keys(): - localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) - logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) - - # make sure our directory is a git repository - if os.path.exists(localdirname): - localremotes = self._shellcmd("git remote -v", localdirname) - if not giturl in localremotes: - raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) - else: - if giturl in cached_layers: - logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) - self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) - self._shellcmd("git remote remove origin", localdirname) - self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) - else: - logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) - self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) - - # branch magic name "HEAD" will inhibit checkout - if commit != "HEAD": - logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) - self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname) - - # take the localdirname as poky dir if we can find the oe-init-build-env - if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): - logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) - self.pokydirname = localdirname - - # make sure we have a working bitbake - if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): - logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) - self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) - - # verify our repositories - for name, dirpath in gitrepos[(giturl, commit)]: - localdirpath = os.path.join(localdirname, dirpath) - logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) - if not os.path.exists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) - - if name != "bitbake": - layerlist.append(localdirpath.rstrip("/")) logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py index 5022b59..21bc380 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py @@ -45,12 +45,14 @@ class Command(BaseCommand): for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: assert i in data - def _read_git_url_from_local_repository(address): + def _read_git_url_from_local_repository(address, path = None): url = None + if not path: + path = os.path.dirname(filepath) # we detect the remote name at runtime import subprocess (remote, remote_name) = address.split(":", 1) - cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) + cmd = subprocess.Popen("git remote -v", shell=True, cwd = path, stdout=subprocess.PIPE, stderr = subprocess.PIPE) (out,err) = cmd.communicate() if cmd.returncode != 0: logging.warning("Error while importing layer vcs_url: git error: %s" % err) @@ -121,7 +123,11 @@ class Command(BaseCommand): logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path) if layerinfo['vcs_url'].startswith("remote:"): - lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) + if not layerinfo['local_path'].startswith("/"): + path = None + else: + path = layerinfo['local_path'] + lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'], path) if lo.vcs_url is None: logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name']) @@ -138,6 +144,8 @@ class Command(BaseCommand): commit = branch.name, layer = lo) lvo.dirpath = layerinfo['dirpath'] + if len(layerinfo['local_path']) > 1 and layerinfo['local_path'].startswith("/") and branch.name == "HEAD": + lvo.local_path = layerinfo['local_path'] lvo.save() # set releases for ri in data['releases']: -- 1.9.1