fetch2/repo: Implement AUTOREV for repo fetcher
PATCH v2 1/1 fetch2/repo: Implement AUTOREV for repo fetcher
From: Martin Koppehel <[email protected]>
- Implement AUTOINC and submodule support for REPO provider
- Implement full srcrev support
- Add comments and fixup empty DL_DIR initialization
- Distinguish between artificial and plain rev
- Comments/documentation
The previous implementation of the repo fetcher could not handle updates to the repo manifest file, nor deal with dynamic refspecs within this manifest.
This patch fixes these shortcomings as follows:
During the recipe parsing phase, the repository containing the repo manifest is cloned. This is done, as we need to parse the XML file contained within, in order to discover all involved git repositories. A combined hash is then calculated from the manifest repo, as well as any git repo specified in the manifest. This hash is used for determining the necessity of an update.
Additionally, the recipe will throw an error if the repo source is set to a fixed revision but one or more repositories within the manifest reference a dynamic refspec. This is done to ensure the reproducibility of a version-pinned recipe.
--- lib/bb/fetch2/repo.py | 259 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 230 insertions(+), 29 deletions(-)
lib/bb/fetch2/repo.py +230 -29
@@ -3,6 +3,7 @@ BitBake "Fetch" repo (git) implementation """ +# Copyright (C) 2021 Martin Koppehel <[email protected]>, iris-GmbH infrared & intelligent sensors # Copyright (C) 2009 Tom Rini <[email protected]> # # Based on git.py which is:
@@ -13,9 +14,11 @@ BitBake "Fetch" repo (git) implementation import os import bb +import hashlib +import xml.etree.ElementTree as ET from bb.fetch2 import FetchMethod from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger + class Repo(FetchMethod): """Class to fetch a module or modules from repo (git) repositories"""
@@ -27,46 +30,84 @@ class Repo(FetchMethod): def urldata_init(self, ud, d): """ - We don"t care about the git rev of the manifests repository, but - we do care about the manifest to use. The default is "default". - We also care about the branch or tag to be used. The default is - "master". + We do care about the rev of the manifests repository, as well as the + manifest file. However, when SRCREV=AUTOINC, then we use the specified + branch in SRC_URI, with a fallback to master. + use sm=fetch to fetch possibly referenced submodules in repositories. """ ud.basecmd = d.getVar("FETCHCMD_repo") or "/usr/bin/env repo" + ud.gitcmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" ud.proto = ud.parm.get('protocol', 'git') ud.branch = ud.parm.get('branch', 'master') + + ud.submodules = ud.parm.get('sm', 'fetch') ud.manifest = ud.parm.get('manifest', 'default.xml') if not ud.manifest.endswith('.xml'): ud.manifest += '.xml' - ud.localfile = d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch)) + repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo") + gitsrcname = "%s%s.%s" % (ud.host, ud.path.replace("/", "."), ud.manifest) + ud.codir = os.path.join(repodir, d.getVar("BPN"), gitsrcname) + + if ud.user: + ud.username = ud.user + "@" + else: + ud.username = "" + ud.remoteRepo = "%s://%s%s%s" % (ud.proto, ud.username, ud.host, ud.path) + + ud.repodir = os.path.join(ud.codir, "repo") + # a temporary directory to compute _latest_revision + ud.tempdir = os.path.join(ud.codir, "temp") + ud.stampfile = os.path.join(ud.codir, "__hash.txt") + ud.setup_revisions(d) + + # ud.localfile is used to fill localpath, where the downloaded tarball is stored. + # in our case, we want something like repo_$GIT_URL_$MANIFEST_$SRCREV + # todo: do we want the packagename? + ud.localfile = "repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, d.getVar("SRCREV")) + + def need_update(self, ud, d): + if d.getVar("SRCREV") == "AUTOINC": + return True + return os.path.exists(ud.localfile) def download(self, ud, d): """Fetch url""" - if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK): - logger.debug("%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) - return + bb.utils.mkdirhier(ud.repodir) - repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo") - gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) - codir = os.path.join(repodir, gitsrcname, ud.manifest) - - if ud.user: - username = ud.user + "@" + # we want to run a repo init *always* in case the branch or manifest name changes. + # if not os.path.exists(os.path.join(repodir, ".repo")): + if ud.submodules == "fetch": + submodules = "--fetch-submodules" else: - username = "" + submodules = "" - repodir = os.path.join(codir, "repo") - bb.utils.mkdirhier(repodir) - if not os.path.exists(os.path.join(repodir, ".repo")): - bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url) - runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir) + # fixup the revision -> when it starts with underscore, it's an artificial one + # therefore we then use the specified remote branch used to generate + # the artificial revision in _latest_revision + realRevision = ud.revision + if ud.revision.startswith("_"): + realRevision = ud.branch - bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url) - runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir) + # always run repo init, because we might want to switch branch or manifests. + bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s" % (ud.basecmd, ud.manifest, realRevision, ud.remoteRepo), ud.url) + runfetchcmd("%s init -m %s -b %s -u %s" % (ud.basecmd, ud.manifest, realRevision, ud.remoteRepo), d, workdir=ud.repodir) + + bb.fetch2.check_network_access(d, "%s sync %s %s" % (ud.basecmd, submodules, ud.url), ud.url) + runfetchcmd("%s sync %s" % (ud.basecmd, submodules), d, workdir=ud.repodir) + + # at this point, we have a checked out version, so make sure that revisions are pinned if the user requests that + if self._checkIsFixedRevision(ud, d, realRevision, ud.repodir): + # our manifest repo has a tag or a fixed sha revision => ensure all linked repos have the same. + def projectIterator(project, remote, revision): + if not self._checkIsFixedRevision(ud, d, revision, os.path.join(ud.repodir, project.get("path"))): + raise bb.fetch2.FetchError("repo recipe is set to a fixed revision but project %s has variable revision %s.\nTo fix this, specify tags or SHA-hashes in your repo manifest file.\nThis is important to guarantee reproducibility." % (project.get('name'), revision), ud.url) + + output = runfetchcmd("%s manifest" % ud.basecmd, d, workdir=ud.repodir) + self._scanRepoManifest(ud, d, output, projectIterator, content=True) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep":
@@ -75,13 +116,173 @@ class Repo(FetchMethod): tar_flags = "--exclude='.repo' --exclude='.git'" # Create a cache - runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=codir) + runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*")), d, workdir=ud.codir) def supports_srcrev(self): - return False + return True + + def clean(self, ud, d): + """ clean the repo directory """ + + to_remove = [ud.localpath, ud.repodir, ud.tempdir, ud.stampfile] + # The localpath is a symlink to clonedir when it is cloned from a + # mirror, so remove both of them. + if os.path.islink(ud.localpath): + clonedir = os.path.realpath(ud.localpath) + to_remove.append(clonedir) + + for r in to_remove: + if os.path.exists(r): + bb.utils.remove(r, True) + + # this is taken from the git fetcher + def _lsremote(self, ud, d, search, repo): + """ + Run git ls-remote with the specified search string + """ + # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR, + # and WORKDIR is in PATH (as a result of RSS), our call to + # runfetchcmd() exports PATH so this function will get called again (!) + # In this scenario the return call of the function isn't actually + # important - WORKDIR isn't needed in PATH to call git ls-remote + # anyway. + if d.getVar('_BB_REPO_IN_LSREMOTE', False): + return '' + d.setVar('_BB_REPO_IN_LSREMOTE', '1') + try: + cmd = "%s ls-remote \"%s\" %s" % \ + (ud.gitcmd, repo, search) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, cmd, ud.remoteRepo) + output = runfetchcmd(cmd, d, True) + if not output: + raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) + finally: + d.delVar('_BB_REPO_IN_LSREMOTE') + return output + + # simple helper function to check if a specific object exists. + # used to check that things reference commits + def _revParse(self, ud, d, obj, repoDir, expect): + try: + cmd = "%s rev-parse -q %s" % (ud.gitcmd, obj) + runfetchcmd(cmd, d, workdir = repoDir) + except: + return not expect + return expect + + # checks that the given revision is *not* a branch. + def _checkIsFixedRevision(self, ud, d, commithash, repodir): + if not self._revParse(ud, d, "refs/heads/%s" % commithash, repodir, False): + return False + return self._revParse(ud, d, "refs/tags/%s" % commithash, repodir, True) or self._revParse(ud, d, commithash, repodir, True) + + def _resolveBranchToSHA1(self, ud, d, name, repo): + output = self._lsremote(ud, d, name, repo) + searchstring = "refs/heads/%s" % name + found = False + for line in output.strip().split('\n'): + sha1, ref = line.split() + if searchstring == ref: + return sha1 + break + + if not found: + raise bb.fetch2.FetchError("Could not determine remote ref!") + + def _scanRepoManifest(self, ud, d, path, callback, content=False): + # parse the specified XML manifest + if content: + xml = ET.fromstring(path) + else: + xml = ET.parse(path) + + # repo manifest *may* specify a <default> element, specifying fallback remotes and revisions + defaultObject = xml.find('default') + + # parse all remotes and their corresponding default revisions + remotes = {} + remoteRevisions = {} + for remote in xml.findall('remote'): + remotes[remote.get('name')] = remote.get('fetch') + remoteRevisions[remote.get('name')] = remote.get('revision') + + + # iterate through the <project> elements, resolving the correct remote + # and revision + for project in xml.findall('project'): + # resolve the remote of the project + # when no remote is specified in the project take the one from <default> + # when both aren't specified, throw + remoteName = project.get('remote') + if remoteName is None and defaultObject is not None: + remoteName = defaultObject.get('remote') + if remoteName is None: + raise bb.fetch2.FetchError("repo manifest specifies no remote for %s" % project.get('name')) + + # resolve the remoteName to a git remote URL and optionally + # the revision if it was specified in <remote> + if remotes[remoteName] is not None: + remoteRev = remoteRevisions[remoteName] + remote = remotes[remoteName] + + # use revision in the project, when not specified use the one from <remote> + # when that is not specified use <default> and when we not have anything specified + # throw an exception + revision = project.get('revision') or remoteRev + if revision is None and defaultObject is not None: + revision = defaultObject.get('revision') + if revision is None: + raise bb.fetch2.FetchError("repo manifest specifies no revision for %s" % project.get('name')) + callback(project, remote, revision) + + def _build_revision(self, ud, d, name): + return ud.revisions[name] + + def _revision_key(self, ud, d, name): + return "%s-%s" % (d.getVar("BPN"), name) + + def _latest_revision(self, ud, d, name): + """ + Computes an artificial revision from the manifest repository and all + referenced repositories and their remote revisions. + name is ignored because we can only have a single branch/name + """ + if d.getVar('_BB_REPO_IN_LATEST_REV', False): + return '' + d.setVar('_BB_REPO_IN_LATEST_REV', '1') + + # we use a sha256 to mixup all the hashes we have + hashCalc = hashlib.sha256() + + # first, add the hash of the repo itself + sha1 = self._resolveBranchToSHA1(ud, d, ud.branch, ud.remoteRepo) + hashUpdate = bytes(sha1, 'utf-8') + hashCalc.update(hashUpdate) + + # Parse the repo XML files, remove things + try: + # create/cleanup temporary dir where to clone the repo-manifest URL + if os.path.isdir(ud.tempdir): + bb.utils.prunedir(ud.tempdir) + bb.utils.mkdirhier(ud.tempdir) + + # clone the manifest repo to the temporary dir we just set up + bb.fetch2.check_network_access(d, "%s clone -b %s --depth 1 --single-branch %s ." % (ud.gitcmd, ud.branch, ud.remoteRepo), ud.url) + runfetchcmd("%s clone -b %s --depth 1 --single-branch %s %s" % (ud.gitcmd, ud.branch, ud.remoteRepo, ud.tempdir), d, workdir=ud.tempdir) + + def projectIterator(project, remote, revision): + # perform an ls-remote on the branch, update the checksum with the commit hash + gitRemotePath = "%s/%s" % (remote, project.get('name')) + + sha1 = self._resolveBranchToSHA1(ud, d, revision, gitRemotePath) + hashUpdate = bytes(sha1, 'utf-8') + hashCalc.update(hashUpdate) + + self._scanRepoManifest(ud, d, "%s/%s" % (ud.tempdir, ud.manifest), projectIterator) + finally: + d.delVar('_BB_REPO_IN_LATEST_REV') + digest = "_" + hashCalc.hexdigest() + return digest - def _build_revision(self, ud, d): - return ud.manifest - def _want_sortable_revision(self, ud, d): - return False
Signature · 1 line
2.33.1
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256
v2 fixes a bug we discovered in version pinning the refspecs in the repo manifest. Also, we removed the "racecondition bugfix" from this patch series (for now), as we were so far unable to reproduce it again.
- --
With best regards
Jasper Orschulko
DevOps Engineer
Tel. +49 30 58 58 14 265
Fax +49 30 58 58 14 999
[email protected]
• • • • • • • • • • • • • • • • • • • • • • • • • •
iris-GmbH
infrared & intelligent sensors
Schnellerstraße 1-5 | 12439 Berlin
Quoted 422 lines · On Thu, 2021-11-11 at 13:53 +0100, [email protected] wrote:
From: Martin Koppehel <[email protected]>
- Implement AUTOINC and submodule support for REPO provider
- Implement full srcrev support
- Add comments and fixup emptyDL_DIRinitialization
- Distinguish between artificial and plain rev
- Comments/documentationThe previous implementation of the repo fetcher could not handle updates
to the repo manifest file, nor deal with dynamic refspecs within this manifest.This patch fixes these shortcomings as follows:
During the recipe parsing phase, the repository containing the repo manifest is cloned. This is done, as we need to parse the XML file contained within, in order to discover all involved git repositories. A
combined hash is then calculated from the manifest repo, as well as any
git repo specified in the manifest. This hash is used for determining the
necessity of an update.Additionally, the recipe will throw an error if the repo source is set to
a fixed revision but one or more repositories within the manifest reference a dynamic refspec. This is done to ensure the reproducibility
of a version-pinned recipe.Signed-off-by Jasper Orschulko <[email protected]>---
lib/bb/fetch2/repo.py | 259 +++++++++++++++++++++++++++++++++++++--- --1 file changed, 230 insertions(+), 29 deletions(-)lib/bb/fetch2/repo.py +0
-0@@ -3,6 +3,7 @@ BitBake "Fetch" repo (git) implementation
"""
+# Copyright (C) 2021 Martin Koppehel <[email protected]>, iris-GmbH infrared & intelligent sensors
# Copyright (C) 2009 Tom Rini <[email protected]> #
# Based ongit.pywhich is:lib/bb/fetch2/repo.py +0
-0@@ -13,9 +14,11 @@ BitBake "Fetch" repo (git) implementation
import os
import bblib/bb/fetch2/repo.py +2
-0+import hashlib +import xml.etree.ElementTree as ETfrom bb.fetch2 import FetchMethod
from bb.fetch2 import runfetchcmdlib/bb/fetch2/repo.py +1
-1-from bb.fetch2 import logger +
class Repo(FetchMethod):
"""Class to fetch a module or modules from repo (git) repositories"""lib/bb/fetch2/repo.py +0
-0@@ -27,46 +30,84 @@ class Repo(FetchMethod):
defurldata_init(self, ud, d):
"""
- We don"t care about the git rev of the manifests repository, but
- we do care about the manifest to use. The default is "default".
- We also care about the branch or tag to be used. The default is
- "master".
+ We do care about the rev of the manifests repository, as well as the
+ manifest file. However, when SRCREV=AUTOINC, then we use the specified
+ branch inSRC_URI, with a fallback to master.
+ use sm=fetch to fetch possibly referenced submodules in repositories.
"""
ud.basecmd =d.getVar("FETCHCMD_repo")or "/usr/bin/envrepo"
+ ud.gitcmd =d.getVar("FETCHCMD_git")or "git -c core.fsyncobjectfiles=0"
ud.proto =ud.parm.get('protocol', 'git')ud.branch =ud.parm.get('branch', 'master')
+
+ ud.submodules =ud.parm.get('sm', 'fetch')ud.manifest =ud.parm.get('manifest', 'default.xml')if notud.manifest.endswith('.xml'):
ud.manifest += '.xml'
- ud.localfile =d.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch))
+ repodir =d.getVar("REPODIR")or (d.getVar("DL_DIR")+ "/repo")
+ gitsrcname = "%s%s.%s" % (ud.host,ud.path.replace("/", "."), ud.manifest)
+ ud.codir =os.path.join(repodir, d.getVar("BPN"), gitsrcname)
+
+ if ud.user:
+ ud.username = ud.user + "@"
+ else:
+ ud.username = ""
+ ud.remoteRepo = "%s://%s%s%s" % (ud.proto, ud.username, ud.host, ud.path)lib/bb/fetch2/repo.py +8
-0+ + ud.repodir = os.path.join(ud.codir, "repo") + # a temporary directory to compute _latest_revision + ud.tempdir = os.path.join(ud.codir, "temp") + ud.stampfile = os.path.join(ud.codir, "__hash.txt") + ud.setup_revisions(d) + + # ud.localfile is used to fill localpath, where thedownloaded tarball is stored.
+ # in our case, we want something like repo_$GIT_URL_$MANIFEST_$SRCREV
+ # todo: do we want the packagename?
+ ud.localfile = "repo_%s%s_%s_%s.tar.gz" % (ud.host,ud.path.replace("/", "."), ud.manifest,d.getVar("SRCREV")) +
+ defneed_update(self, ud, d):
+ ifd.getVar("SRCREV")== "AUTOINC":
+ return True
+ returnos.path.exists(ud.localfile)
defdownload(self, ud, d):
"""Fetch url"""
- ifos.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
- logger.debug("%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
- return
+bb.utils.mkdirhier(ud.repodir)
- repodir =d.getVar("REPODIR")or (d.getVar("DL_DIR")+ "/repo")
- gitsrcname = "%s%s" % (ud.host,ud.path.replace("/", ".")) - codir =os.path.join(repodir, gitsrcname, ud.manifest)-
- if ud.user:
- username = ud.user + "@"
+ # we want to run a repo init *always* in case the branch or manifest name changes.
+ # if notos.path.exists(os.path.join(repodir, ".repo")):
+ if ud.submodules == "fetch":
+ submodules = "--fetch-submodules"
else:
- username = ""
+ submodules = ""
- repodir =os.path.join(codir, "repo")
-bb.utils.mkdirhier(repodir)
- if notos.path.exists(os.path.join(repodir, ".repo")):
- bb.fetch2.check_network_access(d, "%s init -m %s -b %s - u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
- runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+ # fixup the revision -> when it starts with underscore, it's an artificial one
+ # therefore we then use the specified remote branch used to generate
+ # the artificial revision in_latest_revision
+ realRevision = ud.revision
+ ifud.revision.startswith("_"):
+ realRevision = ud.branch
-bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url)
-runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir)+ # always run repo init, because we might want to switch branch or manifests.
+bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s" % (ud.basecmd, ud.manifest, realRevision, ud.remoteRepo), ud.url)+ runfetchcmd("%s init -m %s -b %s -u %s" % (ud.basecmd, ud.manifest, realRevision, ud.remoteRepo), d, workdir=ud.repodir) +
+bb.fetch2.check_network_access(d, "%s sync %s %s" % (ud.basecmd, submodules, ud.url), ud.url)
+runfetchcmd("%s sync %s" % (ud.basecmd, submodules), d, workdir=ud.repodir)
+
+ # at this point, we have a checked out version, so make sure that revisions are pinned if the user requests that
+ ifself._checkIsFixedRevision(ud, d, realRevision, ud.repodir):
+ # our manifest repo has a tag or a fixed sha revision => ensure all linked repos have the same.
+ defprojectIterator(project, remote, revision):
+ if notself._checkIsFixedRevision(ud, d, revision, os.path.join(ud.repodir, project.get("path"))):
+ raise bb.fetch2.FetchError("repo recipe is set to a fixed revision but project %s has variable revision %s.\nTo fix this, specify tags or SHA-hashes in your repo manifest file.\nThis is important to guarantee reproducibility." % (project.get('name'), revision), ud.url)
+
+ output =runfetchcmd("%s manifest" % ud.basecmd, d, workdir=ud.repodir)
+self._scanRepoManifest(ud, d, output, projectIterator, content=True)
scmdata =ud.parm.get("scmdata", "")
if scmdata == "keep":lib/bb/fetch2/repo.py +0
-0@@ -75,13 +116,173 @@ class Repo(FetchMethod):
tar_flags= "--exclude='.repo' --exclude='.git'"
# Create a cache
-runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=codir)
+runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*")), d, workdir=ud.codir)
defsupports_srcrev(self):
- return False
+ return Truelib/bb/fetch2/repo.py +5
-0+ + def clean(self, ud, d): + """ clean the repo directory """ + + to_remove = [ud.localpath, ud.repodir, ud.tempdir,ud.stampfile]
+ # The localpath is a symlink to clonedir when it is cloned from a
+ # mirror, so remove both of them.
+ ifos.path.islink(ud.localpath):
+ clonedir =os.path.realpath(ud.localpath)
+to_remove.append(clonedir)lib/bb/fetch2/repo.py +11
-0+ + for r in to_remove: + if os.path.exists(r): + bb.utils.remove(r, True) + + # this is taken from the git fetcher + def _lsremote(self, ud, d, search, repo): + """ + Run git ls-remote with the specified search string + """ + # Prevent recursion e.g. in OE if SRCPV is in PV, PV is inWORKDIR,
+ # and WORKDIR is in PATH (as a result of RSS), our call to + #runfetchcmd()exports PATH so this function will get called again (!)
+ # In this scenario the return call of the function isn't actually
+ # important - WORKDIR isn't needed in PATH to call git ls- remote
+ # anyway.
+ ifd.getVar('_BB_REPO_IN_LSREMOTE', False):
+ return ''
+d.setVar('_BB_REPO_IN_LSREMOTE', '1')
+ try:
+ cmd = "%s ls-remote \"%s\" %s" % \
+ (ud.gitcmd, repo, search)
+ ifud.proto.lower()!= 'file':
+bb.fetch2.check_network_access(d, cmd, ud.remoteRepo)
+ output =runfetchcmd(cmd, d, True)
+ if not output:
+ raisebb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
+ finally:
+d.delVar('_BB_REPO_IN_LSREMOTE')
+ return outputlib/bb/fetch2/repo.py +14
-0+ + # simple helper function to check if a specific object exists. + # used to check that things reference commits + def _revParse(self, ud, d, obj, repoDir, expect): + try: + cmd = "%s rev-parse -q %s" % (ud.gitcmd, obj) + runfetchcmd(cmd, d, workdir = repoDir) + except: + return not expect + return expect + + # checks that the given revision is *not* a branch. + def _checkIsFixedRevision(self, ud, d, commithash, repodir): + if not self._revParse(ud, d, "refs/heads/%s" % commithash,repodir, False):
+ return False
+ returnself._revParse(ud, d, "refs/tags/%s" % commithash, repodir, True)orself._revParse(ud, d, commithash, repodir, True)lib/bb/fetch2/repo.py +13
-0+ + def _resolveBranchToSHA1(self, ud, d, name, repo): + output = self._lsremote(ud, d, name, repo) + searchstring = "refs/heads/%s" % name + found = False + for line in output.strip().split('\n'): + sha1, ref = line.split() + if searchstring == ref: + return sha1 + break + + if not found: + raise bb.fetch2.FetchError("Could not determine remoteref!")
+
+ def_scanRepoManifest(self, ud, d, path, callback, content=False):
+ # parse the specified XML manifest
+ if content:
+ xml =ET.fromstring(path)
+ else:
+ xml =ET.parse(path)
+
+ # repo manifest *may* specify a <default> element, specifying fallback remotes and revisions
+ defaultObject =xml.find('default')
+
+ # parse all remotes and their corresponding default revisions
+ remotes = {}
+ remoteRevisions = {}
+ for remote inxml.findall('remote'):
+ remotes[remote.get('name')] =remote.get('fetch')+ remoteRevisions[remote.get('name')] =remote.get('revision')lib/bb/fetch2/repo.py +3
-0+ + + # iterate through the <project> elements, resolving thecorrect remote
+ # and revision
+ for project inxml.findall('project'):
+ # resolve the remote of the project
+ # when no remote is specified in the project take the one from <default>
+ # when both aren't specified, throw
+ remoteName =project.get('remote')
+ if remoteName is None and defaultObject is not None:
+ remoteName =defaultObject.get('remote')+ if remoteName is None:
+ raisebb.fetch2.FetchError("repo manifest specifies no remote for %s" % project.get('name'))
+
+ # resolve the remoteName to a git remote URL and optionally
+ # the revision if it was specified in <remote> + if remotes[remoteName] is not None:
+ remoteRev = remoteRevisions[remoteName] + remote = remotes[remoteName]
+
+ # use revision in the project, when not specified use the one from <remote>
+ # when that is not specified use <default> and when we not have anything specified
+ # throw an exception
+ revision =project.get('revision')or remoteRev + if revision is None and defaultObject is not None:
+ revision =defaultObject.get('revision')+ if revision is None:
+ raisebb.fetch2.FetchError("repo manifest specifies no revision for %s" % project.get('name'))
+callback(project, remote, revision)lib/bb/fetch2/repo.py +10
-0+ + def _build_revision(self, ud, d, name): + return ud.revisions[name] + + def _revision_key(self, ud, d, name): + return "%s-%s" % (d.getVar("BPN"), name) + + def _latest_revision(self, ud, d, name): + """ + Computes an artificial revision from the manifest repositoryand all
+ referenced repositories and their remote revisions. + name is ignored because we can only have a single branch/name
+ """
+ ifd.getVar('_BB_REPO_IN_LATEST_REV', False):
+ return ''
+d.setVar('_BB_REPO_IN_LATEST_REV', '1')lib/bb/fetch2/repo.py +6
-0+ + # we use a sha256 to mixup all the hashes we have + hashCalc = hashlib.sha256() + + # first, add the hash of the repo itself + sha1 = self._resolveBranchToSHA1(ud, d, ud.branch,ud.remoteRepo)
+ hashUpdate =bytes(sha1, 'utf-8')
+hashCalc.update(hashUpdate)
+
+ # Parse the repo XML files, remove things
+ try:
+ # create/cleanup temporary dir where to clone the repo- manifest URL
+ ifos.path.isdir(ud.tempdir):
+bb.utils.prunedir(ud.tempdir)
+bb.utils.mkdirhier(ud.tempdir)
+
+ # clone the manifest repo to the temporary dir we just set up
+ bb.fetch2.check_network_access(d, "%s clone -b %s -- depth 1 --single-branch %s ." % (ud.gitcmd, ud.branch, ud.remoteRepo), ud.url)
+ runfetchcmd("%s clone -b %s --depth 1 --single-branch %s %s" % (ud.gitcmd, ud.branch, ud.remoteRepo, ud.tempdir), d, workdir=ud.tempdir)
+
+ defprojectIterator(project, remote, revision):
+ # perform an ls-remote on the branch, update the checksum with the commit hash
+ gitRemotePath = "%s/%s" % (remote,project.get('name'))
+
+ sha1 =self._resolveBranchToSHA1(ud, d, revision, gitRemotePath)
+ hashUpdate =bytes(sha1, 'utf-8')
+hashCalc.update(hashUpdate)
+
+self._scanRepoManifest(ud, d, "%s/%s" % (ud.tempdir, ud.manifest), projectIterator)
+ finally:
+d.delVar('_BB_REPO_IN_LATEST_REV')
Quoted 8 lines
+ digest = "_" +
hashCalc.hexdigest()
+ return digest
- def_build_revision(self, ud, d):
- return ud.manifest
- def_want_sortable_revision(self, ud, d):
- return False
-----BEGIN PGP SIGNATURE-----
iQEzBAEBCAAdFiEE4WyPMIC5Ap4+Ooo1Ygqew07VMNUFAmGNExEACgkQYgqew07V MNWx4AgAgxY8ZYKUoUVYr/235V2kswHcrMDBoFF9DpUJbRRCYhC8ZmRdhcd3a80X ukR439hqvde/QKAY4yXvUrMTfKDrwoVH15G8Cz64OXQJszHcBHKAkHyUUlVyAdQ+ /eOUuNEAaxiPqUSWH22LynSeUyInQvIHXbtSAvauZq+R2FsTBtGt3OsDWFreOMc4 PD9gOjfmHaJHuATCr+8tRWPzZuAx6IN/d9C3LFc1+GcXPcefkzSp5VqHob06LI1b 7YBVsKuxT4WxdKGnPHQIYzs6+hjjRQZDv416l/Z8WXLUzk8wM4cf0H/YcLDyoZKm UP2o6mrd4LDRmKo/fwP4IwXAQt1ATg==
=Sudh
-----END PGP SIGNATURE-----