git fetcher unpack optimization V2
cover git fetcher unpack optimization V2
From: Yu Ke <[email protected]>
This is the remaining patches of last fetcher patch pull, including
- optimize git unpack method to use git local clone
- not to create the source tree mirror tar ball
With Richar and Bruce's comment, this patch is revised accordingly, the major chagnes compared with V1 is:
- use git local clone instead of git repo copy to save disk space
it has been tested with "bitbake poky-image-sato" with empty DL_DIR
Pull URL: git://git.pokylinux.org/poky-contrib.git
Branch: kyu3/git-unpack-2
Browse: http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=kyu3/git-unpack-2
Thanks,
Yu Ke <[email protected]>
---
Yu Ke (3):
bb.fetch2: add git unpack
kernel-yocto.bbclass: adjust the git dir to fit with git unpack git.py: remove the source tree tar ball
bitbake/lib/bb/fetch2/git.py | 66 ++++++++++-------------------------- meta/classes/kernel-yocto.bbclass | 4 +- 2 files changed, 21 insertions(+), 49 deletions(-)
PATCH 1/3 bb.fetch2: add git unpack
From: Yu Ke <[email protected]>
git download will clone git repo to local, so git unpack can be simplified to only checkouting the code to the work dir. For fullclone case, we also need to manually copy all the ref info, which is needed by the later do_kernel_checkout()
--- bitbake/lib/bb/fetch2/git.py | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-)
bitbake/lib/bb/fetch2/git.py +25 -0
@@ -194,6 +194,31 @@ class Git(Fetch): os.chdir(ud.clonedir) bb.utils.prunedir(codir) + def unpack(self, ud, destdir, d): + """ unpack the downloaded src to destdir""" + + subdir = ud.parm.get("subpath", "") + if subdir != "": + readpathspec = ":%s" % (subdir) + else: + readpathspec = "" + + destdir = os.path.join(destdir, "git/") + if os.path.exists(destdir): + bb.utils.prunedir(destdir) + + if 'fullclone' in ud.parm: + runfetchcmd("git clone -l -n %s %s" % (ud.clonedir, destdir), d) + if os.path.exists("%s/.git/refs/remotes/origin" % ud.clonedir): + runfetchcmd("cp -af %s/.git/refs/remotes/origin/* %s/.git/refs/remotes/origin/" %(ud.clonedir, destdir), d) + if os.path.exists("%s/.git/packed-refs" % ud.clonedir): + runfetchcmd("cp -af %s/.git/packed-refs %s/.git/" %(ud.clonedir, destdir), d) + else: + os.chdir(ud.clonedir) + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) + runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d) + return True + def supports_srcrev(self): return True
Signature · 1 line
1.7.0.4
PATCH 2/3 kernel-yocto.bbclass: adjust the git dir to fit with git unpack
In reply to Yu KeFrom: Yu Ke <[email protected]>
--- meta/classes/kernel-yocto.bbclass | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
meta/classes/kernel-yocto.bbclass +2 -2
@@ -89,11 +89,11 @@ do_patch() { } do_kernel_checkout() { - if [ -d ${WORKDIR}/.git/refs/remotes/origin ]; then + if [ -d ${WORKDIR}/git/.git/refs/remotes/origin ]; then echo "Fixing up git directory for ${LINUX_KERNEL_TYPE}/${KMACHINE}" rm -rf ${S} mkdir ${S} - mv ${WORKDIR}/.git ${S} + mv ${WORKDIR}/git/.git ${S} if [ -e ${S}/.git/packed-refs ]; then cd ${S}
Signature · 1 line
1.7.0.4
PATCH 3/3 git.py: remove the source tree tar ball
In reply to Yu KeFrom: Yu Ke <[email protected]>
we already create repo tar ball, so no need to create the source tree tar ball
--- bitbake/lib/bb/fetch2/git.py | 63 +++-------------------------------------- 1 files changed, 5 insertions(+), 58 deletions(-)
bitbake/lib/bb/fetch2/git.py +5 -58
@@ -66,18 +66,7 @@ class Git(Fetch): if not ud.tag or ud.tag == "master": ud.tag = self.latest_revision(url, ud, d) - subdir = ud.parm.get("subpath", "") - if subdir != "": - if subdir.endswith("/"): - subdir = subdir[:-1] - subdirpath = os.path.join(ud.path, subdir); - else: - subdirpath = ud.path; - - if 'fullclone' in ud.parm: - ud.localfile = ud.mirrortarball - else: - ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d) + ud.localfile = ud.mirrortarball if 'noclone' in ud.parm: ud.localfile = None
@@ -90,8 +79,6 @@ class Git(Fetch): return True if 'noclone' in ud.parm: return False - if os.path.exists(ud.localpath): - return False if not self._contains_ref(ud.tag, d): return True return False
@@ -116,6 +103,8 @@ class Git(Fetch): repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) + ud.repochanged = not os.path.exists(repofile) + # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.clonedir) and os.path.exists(repofile): bb.mkdirhier(ud.clonedir)
@@ -140,60 +129,18 @@ class Git(Fetch): runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) + ud.repochanged = True def build_mirror_data(self, url, ud, d): # Generate a mirror tarball if needed - coname = '%s' % (ud.tag) - codir = os.path.join(ud.clonedir, coname) repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) os.chdir(ud.clonedir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) - if mirror_tarballs != "0" or 'fullclone' in ud.parm: + if (mirror_tarballs != "0" or 'fullclone' in ud.parm) and ud.repochanged: logger.info("Creating tarball of git repository") runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) - if 'fullclone' in ud.parm: - return - - if os.path.exists(codir): - bb.utils.prunedir(codir) - - subdir = ud.parm.get("subpath", "") - if subdir != "": - if subdir.endswith("/"): - subdirbase = os.path.basename(subdir[:-1]) - else: - subdirbase = os.path.basename(subdir) - else: - subdirbase = "" - - if subdir != "": - readpathspec = ":%s" % (subdir) - codir = os.path.join(codir, "git") - coprefix = os.path.join(codir, subdirbase, "") - else: - readpathspec = "" - coprefix = os.path.join(codir, "git", "") - - scmdata = ud.parm.get("scmdata", "") - if scmdata == "keep": - runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d) - os.chdir(coprefix) - runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d) - else: - bb.mkdirhier(codir) - os.chdir(ud.clonedir) - runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) - runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d) - - os.chdir(codir) - logger.info("Creating tarball of git checkout") - runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) - - os.chdir(ud.clonedir) - bb.utils.prunedir(codir) - def unpack(self, ud, destdir, d): """ unpack the downloaded src to destdir"""
Signature · 1 line
1.7.0.4