Skip to contentSkip to conversation

fetch2/git: Use git shallow fetch to implement clone_shallow_local()

PATCH New [email protected] 1 participant 3 messages Jul 7, 2024

lore
Share & tools
lore
#0 lore raw

v2 cover fetch2/git: Use git shallow fetch to implement clone_shallow_local()

From: Robert Yang <[email protected]>

* V2:
- Fix bitbake-selftest on ubuntu 20.04 (git version: 2.25.1)

# Advertise the revision for lower version git such as 2.25.1:
# error: Server does not allow request for unadvertised object. # The ud.clonedir is a local temporary dir, will be removed when # fetch is done, so we can do anything on it.
adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision) runfetchcmd(adv_cmd, d, workdir=ud.clonedir)

* V1:
bitbake-selftest works well
bitbake world works well with the shallow tarballs

// Robert

The following changes since commit 0979646a1bd224e3690380ab9fb3e0079bcafcf4:

abi_version/package: Bump hashequiv version and package class version (2024-07-05 10:23:05 +0100)

are available in the Git repository at:

https://github.com/robertlinux/yocto rbt/shallow
https://github.com/robertlinux/yocto/tree/rbt/shallow

Robert Yang (2):
fetch2/git: Use git shallow fetch to implement clone_shallow_local()
bitbake: tests/fetch: Update GitShallowTest for clone_shallow_local()

 bitbake/lib/bb/fetch2/git.py  | 78 +++++++++++++++++++++++------------
 bitbake/lib/bb/tests/fetch.py | 13 +++---
 2 files changed, 60 insertions(+), 31 deletions(-)
Signature · 1 line
2.25.1
#1 lore raw

PATCH v2 1/2 fetch2/git: Use git shallow fetch to implement clone_shallow_local()

From: Robert Yang <[email protected]>

This patch can make the following settings much more faster:
BB_GIT_SHALLOW = "1"
BB_GENERATE_MIRROR_TARBALLS = "1"

* The previous implementation was:
- Make a full clone for the repo from local ud.clonedir
- Use git-make-shallow to remove unneeded revs

It was very slow for recipes which have a lot of SRC_URIs, for example vulkan-samples and docker-compose, the docker-compose can't be done after 5 hours.

  $ bitbake vulkan-samples -cfetch
  Before: 12 minutes
  Now: 2 minutes
  $ bitbake docker-compose -cfetch
  Before: More than 300 minutes
  Now: 15 minutes

* The patch uses git shallow fetch to fetch the repo from local ud.clonedir:
- For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
- For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev

Then the git repo will be shallow, and git-make-shallow is not needed any more.

And git shallow fetch will download less commits than before since it doesn't need "rev^" to parse the dependencies, the previous code always need 'rev^'.

---
 bitbake/lib/bb/fetch2/git.py | 78 ++++++++++++++++++++++++------------
 1 file changed, 52 insertions(+), 26 deletions(-)
bitbake/lib/bb/fetch2/git.py +52 -26
index c7ff769fdf..16bc1f1cf8 100644
@@ -551,18 +551,31 @@ class Git(FetchMethod):
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
-        """Clone the repo and make it shallow.
+        """
+        Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
+        - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
+        - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
+        """
+
+        bb.utils.mkdirhier(dest)
+        init_cmd = "%s init -q" % ud.basecmd
+        if ud.bareclone:
+            init_cmd += " --bare"
+        runfetchcmd(init_cmd, d, workdir=dest)
+        runfetchcmd("%s remote add origin %s" % (ud.basecmd, ud.clonedir), d, workdir=dest)
 
-        The upstream url of the new clone isn't set at this time, as it'll be
-        set correctly when unpacked."""
-        runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
+        # Check the histories which should be excluded
+        shallow_exclude = ''
+        for revision in ud.shallow_revs:
+            shallow_exclude += " --shallow-exclude=%s" % revision
 
-        to_parse, shallow_branches = [], []
         for name in ud.names:
             revision = ud.revisions[name]
             depth = ud.shallow_depths[name]
-            if depth:
-                to_parse.append('%s~%d^{}' % (revision, depth - 1))
+
+            # The --depth and --shallow-exclude can't be used together
+            if depth and shallow_exclude:
+                raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
 
             # For nobranch, we need a ref, otherwise the commits will be
             # removed, and for non-nobranch, we truncate the branch to our
@@ -575,36 +588,49 @@ class Git(FetchMethod):
             else:
                 ref = "refs/remotes/origin/%s" % branch
 
-            shallow_branches.append(ref)
-            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+            fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
+            if depth:
+                fetch_cmd += " --depth %s" % depth
+
+            if shallow_exclude:
+                fetch_cmd += shallow_exclude
 
-        # Map srcrev+depths to revisions
-        parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
+            # Advertise the revision for lower version git such as 2.25.1:
+            # error: Server does not allow request for unadvertised object.
+            # The ud.clonedir is a local temporary dir, will be removed when
+            # fetch is done, so we can do anything on it.
+            adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
+            runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
 
-        # Resolve specified revisions
-        parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
-        shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
+            runfetchcmd(fetch_cmd, d, workdir=dest)
+            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
 
         # Apply extra ref wildcards
-        all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
-                               d, workdir=dest).splitlines()
+        all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
+                                        d, workdir=dest).splitlines()
+        all_refs = []
+        for line in all_refs_remote:
+            all_refs.append(line.split()[-1])
+        extra_refs = []
         for r in ud.shallow_extra_refs:
             if not ud.bareclone:
                 r = r.replace('refs/heads/', 'refs/remotes/origin/')
 
             if '*' in r:
                 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
-                shallow_branches.extend(matches)
+                extra_refs.extend(matches)
             else:
-                shallow_branches.append(r)
-
-        # Make the repository shallow
-        shallow_cmd = [self.make_shallow_path, '-s']
-        for b in shallow_branches:
-            shallow_cmd.append('-r')
-            shallow_cmd.append(b)
-        shallow_cmd.extend(shallow_revisions)
-        runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
+                extra_refs.append(r)
+
+        for ref in extra_refs:
+            ref_fetch = os.path.basename(ref)
+            runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest)
+            revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest)
+            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+
+        # The url is local ud.clonedir, set it to upstream one
+        repourl = self._get_repo_url(ud)
+        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
Signature · 1 line
2.25.1
#2 lore raw

PATCH v2 2/2 bitbake: tests/fetch: Update GitShallowTest for clone_shallow_local()

In reply to [email protected]

From: Robert Yang <[email protected]>

Update the test cases since the implementation is changed:

* test_shallow_multi_one_uri()
The a_branch and v0.0 had the same revision, and it required fetch a_branch and remove histories of v0.0 which were conflicted, and bitbake reported:
fatal: no commits selected for shallow requests

Make a_branch and v0.0 have different revs to fix the problem.

And now the 'rev^' is not needed, so update self.assertRevCount() as well.

* test_shallow_multi_one_uri_depths()
Update self.assertRevCount(), now git only fetches the required revs.

* test_shallow_fetch_missing_revs()
The command is:

  $ git fetch --shallow-exclude=v0.0 master

But master and v0.0 uses the same revision, so there is no commit to fetch.

* test_shallow_fetch_missing_revs_fails()
Two unneeded committs are not fetched now:
- rev^
- One not specified or required tag.

So update self.assertRevCount()

---
 bitbake/lib/bb/tests/fetch.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)
bitbake/lib/bb/tests/fetch.py +8 -5
index 701129d138..2ef2063436 100644
@@ -2034,9 +2034,9 @@ class GitShallowTest(FetcherTest):
         self.add_empty_file('b')
         self.git('checkout -b a_branch', cwd=self.srcdir)
         self.add_empty_file('c')
+        self.git('tag v0.0 HEAD', cwd=self.srcdir)
         self.add_empty_file('d')
         self.git('checkout master', cwd=self.srcdir)
-        self.git('tag v0.0 a_branch', cwd=self.srcdir)
         self.add_empty_file('e')
         self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
         self.add_empty_file('f')
@@ -2052,7 +2052,7 @@ class GitShallowTest(FetcherTest):
 
         self.fetch_shallow(uri)
 
-        self.assertRevCount(5)
+        self.assertRevCount(4)
         self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
 
     def test_shallow_multi_one_uri_depths(self):
@@ -2199,7 +2199,7 @@ class GitShallowTest(FetcherTest):
 
         self.fetch_shallow()
 
-        self.assertRevCount(5)
+        self.assertRevCount(2)
 
     def test_shallow_invalid_revs(self):
         self.add_empty_file('a')
@@ -2218,7 +2218,10 @@ class GitShallowTest(FetcherTest):
         self.git('tag v0.0 master', cwd=self.srcdir)
         self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
         self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
-        self.fetch_shallow()
+
+        with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
+            self.fetch_shallow()
+        self.assertIn("fatal: no commits selected for shallow requests", cm.output[0])
 
     def test_shallow_fetch_missing_revs_fails(self):
         self.add_empty_file('a')
@@ -2249,7 +2252,7 @@ class GitShallowTest(FetcherTest):
         revs = len(self.git('rev-list master').splitlines())
         self.assertNotEqual(orig_revs, revs)
         self.assertRefs(['master', 'origin/master'])
-        self.assertRevCount(orig_revs - 1758)
+        self.assertRevCount(orig_revs - 1760)
 
     def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
         self.add_empty_file('a')
Signature · 1 line
2.25.1