Skip to contentSkip to conversation

gitsm fetcher / test improvements

PATCH New Paul Eggleton 1 participant 5 messages Feb 24, 2014

lore
Share & tools
lore
#0 lore raw

cover gitsm fetcher / test improvements

The following changes since commit 348971d410bfd5d8b1757468d73e1d24ae78a594:

fetch2: Fix mirror repo tarball creation (2014-02-24 13:00:29 +0000)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib paule/gitsm
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/gitsm

Paul Eggleton (4):
fetch2: fix fetching git submodules with git 1.7.9.x or older
bitbake-selftest: enable specifying tests to run on command line
tests: add missing import
tests: add test for gitsm fetcher

 bin/bitbake-selftest   | 21 ++++++++++++++++-----
 lib/bb/fetch2/gitsm.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 lib/bb/tests/data.py   |  1 +
 lib/bb/tests/fetch.py  |  7 +++++++
 4 files changed, 72 insertions(+), 5 deletions(-)
Signature · 1 line
1.8.5.3
#1 lore raw

PATCH 1/4 fetch2: fix fetching git submodules with git 1.7.9.x or older

Git versions older than 1.7.10 put absolute paths in configuration files for the submodule repositories, leading to errors when the repository checkout is moved. We move the repository as a matter of course in the gitsm fetcher; the failure occurs in do_unpack). Change the absolute paths to be relative during processing to fix this.

(At the time of writing, Ubuntu 12.04.4 LTS ships Git version 1.7.9.5, hence the desire to fix this rather than just mandating a newer Git version.)

Fixes [YOCTO #5525].

---
 lib/bb/fetch2/gitsm.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
lib/bb/fetch2/gitsm.py +48 -0
index 9fdde46..1a76215 100644
@@ -42,6 +42,53 @@ class GitSM(Git):
                 pass
         return False
 
+    def _set_relative_paths(self, repopath):
+        """
+        Fix submodule paths to be relative instead of absolute,
+        so that when we move the repo it doesn't break
+        (In Git 1.7.10+ this is done automatically)
+        """
+        submodules = []
+        with open(os.path.join(repopath, '.gitmodules'), 'r') as f:
+            for line in f.readlines():
+                if line.startswith('[submodule'):
+                    submodules.append(line.split('"')[1])
+
+        for module in submodules:
+            repo_conf = os.path.join(repopath, module, '.git')
+            if os.path.exists(repo_conf):
+                with open(repo_conf, 'r') as f:
+                    lines = f.readlines()
+                newpath = ''
+                for i, line in enumerate(lines):
+                    if line.startswith('gitdir:'):
+                        oldpath = line.split(': ')[-1].rstrip()
+                        if oldpath.startswith('/'):
+                            newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module
+                            lines[i] = 'gitdir: %s\n' % newpath
+                            break
+                if newpath:
+                    with open(repo_conf, 'w') as f:
+                        for line in lines:
+                            f.write(line)
+
+            repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config')
+            if os.path.exists(repo_conf2):
+                with open(repo_conf2, 'r') as f:
+                    lines = f.readlines()
+                newpath = ''
+                for i, line in enumerate(lines):
+                    if line.lstrip().startswith('worktree = '):
+                        oldpath = line.split(' = ')[-1].rstrip()
+                        if oldpath.startswith('/'):
+                            newpath = '../' * (module.count('/') + 3) + module
+                            lines[i] = '\tworktree = %s\n' % newpath
+                            break
+                if newpath:
+                    with open(repo_conf2, 'w') as f:
+                        for line in lines:
+                            f.write(line)
+
     def update_submodules(self, ud, d):
         # We have to convert bare -> full repo, do the submodule bit, then convert back
         tmpclonedir = ud.clonedir + ".tmp"
@@ -54,6 +101,7 @@ class GitSM(Git):
         runfetchcmd(ud.basecmd + " reset --hard", d)
         runfetchcmd(ud.basecmd + " submodule init", d)
         runfetchcmd(ud.basecmd + " submodule update", d)
+        self._set_relative_paths(tmpclonedir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d)
         os.rename(gitdir, ud.clonedir,)
         bb.utils.remove(tmpclonedir, True)
Signature · 1 line
1.8.5.3
#2 lore raw

PATCH 2/4 bitbake-selftest: enable specifying tests to run on command line

In reply to Paul Eggleton

If you are just trying to fix one test at a time, it can be useful to be able to specify an individual test(s) rather than running them all:

bitbake-selftest bb.tests.codeparser bb.tests.cow

You can even specify the test class or function to run, e.g.:

bitbake-selftest bb.tests.fetch.URITest bitbake-selftest bb.tests.fetch.FetcherNetworkTest.test_fetch

---
 bin/bitbake-selftest | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
bin/bitbake-selftest +16 -5
index 48a58fe..8c55f7b 100755
@@ -25,13 +25,24 @@ try:
 except RuntimeError as exc:
     sys.exit(str(exc))
 
-tests = ["bb.tests.codeparser", 
-         "bb.tests.cow",
-         "bb.tests.data",
-         "bb.tests.fetch",
-         "bb.tests.utils"]
+def usage():
+    print('usage: %s [testname1 [testname2]...]')
+
+if len(sys.argv) > 1:
+    if '--help' in sys.argv[1:]:
+        usage()
+        sys.exit(0)
+
+    tests = sys.argv[1:]
+else:
+    tests = ["bb.tests.codeparser",
+             "bb.tests.cow",
+             "bb.tests.data",
+             "bb.tests.fetch",
+             "bb.tests.utils"]
 
 for t in tests:
+    t = '.'.join(t.split('.')[:3])
     __import__(t)
 
 unittest.main(argv=["bitbake-selftest"] + tests)
Signature · 1 line
1.8.5.3
#3 lore raw

PATCH 3/4 tests: add missing import

In reply to Paul Eggleton

This was found when trying to run the data tests individually.

---
 lib/bb/tests/data.py | 1 +
 1 file changed, 1 insertion(+)
lib/bb/tests/data.py +1 -0
index f281a35..6ec2736 100644
@@ -23,6 +23,7 @@
 import unittest
 import bb
 import bb.data
+import bb.parse
 
 class DataExpansions(unittest.TestCase):
     def setUp(self):
Signature · 1 line
1.8.5.3
#4 lore raw

PATCH 4/4 tests: add test for gitsm fetcher

In reply to Paul Eggleton

Use a newly created "git-submodule-test" repo on git.yoctoproject.org which currently contains one submodule (the bitbake repository).

---
 lib/bb/tests/fetch.py | 7 +++++++
 1 file changed, 7 insertions(+)
lib/bb/tests/fetch.py +7 -0
index deb1d37..156da83 100644
@@ -417,6 +417,13 @@ class FetcherNetworkTest(FetcherTest):
             self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
             self.gitfetcher(dummyurl, dummyurl)
 
+        def test_git_submodule(self):
+            fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
+            fetcher.download()
+            # Previous cwd has been deleted
+            os.chdir(os.path.dirname(self.unpackdir))
+            fetcher.unpack(self.unpackdir)
+
 class URLHandle(unittest.TestCase):
 
     datatable = {
Signature · 1 line
1.8.5.3