Skip to contentSkip to conversation

fetch2/gitsm: Unpack shallow mirror tarballs

PATCH In review Paul Barker 2 participants 3 messages Mar 9, 2020 – Jul 21, 2020

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 fetch2/gitsm: Unpack shallow mirror tarballs

When a shallow mirror tarball is used to satisfy a gitsm URI it needs to be unpacked temporarily so that the .gitmodules file can be examined.

---
 lib/bb/fetch2/gitsm.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
lib/bb/fetch2/gitsm.py +16 -5
index aa121cbe..e7083001 100644
@@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 import os
 import bb
 import copy
+import shutil
+import tempfile
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -130,7 +132,7 @@ class GitSM(Git):
             ld.setVar('SRCPV', d.getVar('SRCPV'))
             ld.setVar('SRCREV_FORMAT', module)
 
-            function(ud, url, module, paths[module], ld)
+            function(ud, url, module, paths[module], workdir, ld)
 
         return submodules != []
 
@@ -152,7 +154,7 @@ class GitSM(Git):
         return False
 
     def download(self, ud, d):
-        def download_submodule(ud, url, module, modpath, d):
+        def download_submodule(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Is the following still needed?
@@ -163,16 +165,25 @@ class GitSM(Git):
                 newfetch.download()
                 # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
                 runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
+                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir)
             except Exception as e:
                 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
                 raise
 
         Git.download(self, ud, d)
-        self.process_submodules(ud, ud.clonedir, download_submodule, d)
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+            self.process_submodules(ud, tmpdir, download_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, download_submodule, d)
 
     def unpack(self, ud, destdir, d):
-        def unpack_submodules(ud, url, module, modpath, d):
+        def unpack_submodules(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Figure out where we clone over the bare submodules...
Signature · 1 line
2.20.1
#1 lore raw

Sorry, I missed the new test when copying this from a poky checkout to a bitbake checkout to generate the patch. I'll send the test as a follow-up.

Quoted 79 lines · Paul Barker

When a shallow mirror tarball is used to satisfy a gitsm URI it needs to be unpacked temporarily so that the .gitmodules file can be examined.

---
 lib/bb/fetch2/gitsm.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
lib/bb/fetch2/gitsm.py +0 -0
index aa121cbe..e7083001 100644
@@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://"

requires a clean of your r
import os
import bb
import copy

lib/bb/fetch2/gitsm.py +9 -4
+import shutil
+import tempfile
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -130,7 +132,7 @@ class GitSM(Git):
             ld.setVar('SRCPV', d.getVar('SRCPV'))
             ld.setVar('SRCREV_FORMAT', module)
 
-            function(ud, url, module, paths[module], ld)
+            function(ud, url, module, paths[module], workdir, ld)
 
         return submodules != []
 
@@ -152,7 +154,7 @@ class GitSM(Git):
         return False
 
     def download(self, ud, d):
-        def download_submodule(ud, url, module, modpath, d):
+        def download_submodule(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Is the following still needed?
@@ -163,16 +165,25 @@ class GitSM(Git):
                 newfetch.download()
                 # Drop a nugget to add each of the srcrevs we've fetched
 (used by need_update)
                 runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]), d,
 workdir=ud.clonedir)
+                            (ud.basecmd, ud.revisions[ud.names[0]]), d,
 workdir=workdir)
             except Exception as e:
                 logger.error('gitsm: submodule download failed: %s %s' %
 (type(e).__name__, str(e)))
                 raise
 
         Git.download(self, ud, d)
-        self.process_submodules(ud, ud.clonedir, download_submodule, d)
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and

self.need_update(ud, d):

lib/bb/fetch2/gitsm.py +6 -0
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+            self.process_submodules(ud, tmpdir, download_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, download_submodule,

d)

def unpack(self, ud, destdir, d):

lib/bb/fetch2/gitsm.py +1 -2
-        def unpack_submodules(ud, url, module, modpath, d):
+        def unpack_submodules(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Figure out where we clone over the bare submodules...
--

2.20.1

#2 lore raw

Re: (EXT) [bitbake-devel] [PATCH v2] fetch2/gitsm: Unpack shallow mirror tarballs

In reply to Paul Barker
Quoted 5 lines · Paul Barker

When a shallow mirror tarball is used to satisfy a gitsm URI it needs to
be unpacked temporarily so that the .gitmodules file can be examined.

Hi,

can we get this backported to the 1.44 branch?

Kind regards,
Matthias

Quoted 79 lines
---
 lib/bb/fetch2/gitsm.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
lib/bb/fetch2/gitsm.py +0 -0
index aa121cbe..e7083001 100644
@@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to

"gitsm://" requires a clean of your r
import os
import bb
import copy

lib/bb/fetch2/gitsm.py +4 -2
+import shutil
+import tempfile
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -130,7 +132,7 @@ class GitSM(Git):
             ld.setVar('SRCPV', d.getVar('SRCPV'))
             ld.setVar('SRCREV_FORMAT', module)
 
-            function(ud, url, module, paths[module], ld)
+            function(ud, url, module, paths[module], workdir, ld)
 
         return submodules != []
 
@@ -152,7 +154,7 @@ class GitSM(Git):
         return False
 
     def download(self, ud, d):
-        def download_submodule(ud, url, module, modpath, d):
+        def download_submodule(ud, url, module, modpath, workdir,

d):
url += ";bareclone=1;nobranch=1"

# Is the following still needed?

lib/bb/fetch2/gitsm.py +3 -2
@@ -163,16 +165,25 @@ class GitSM(Git):
                 newfetch.download()
                 # Drop a nugget to add each of the srcrevs we've
 fetched (used by need_update)
                 runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]),
 d, workdir=ud.clonedir)
+                            (ud.basecmd, ud.revisions[ud.names[0]]),
 d, workdir=workdir)
             except Exception as e:
                 logger.error('gitsm: submodule download failed: %s
 %s' % (type(e).__name__, str(e)))
                 raise
 
         Git.download(self, ud, d)
-        self.process_submodules(ud, ud.clonedir, download_submodule,
 d)
+
+        # If we're using a shallow mirror tarball it needs to be

unpacked

lib/bb/fetch2/gitsm.py +2 -0
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and

self.need_update(ud, d):

lib/bb/fetch2/gitsm.py +2 -0
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d,

workdir=tmpdir)
+ self.process_submodules(ud, tmpdir, download_submodule,
d)

lib/bb/fetch2/gitsm.py +3 -0
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir,

download_submodule, d)

def unpack(self, ud, destdir, d):

lib/bb/fetch2/gitsm.py +1 -1
-        def unpack_submodules(ud, url, module, modpath, d):
+        def unpack_submodules(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Figure out where we clone over the bare submodules...
Signature · 1 line
2.20.1