gitsm fetcher: Fix recurse submodule error
PATCH 1/1 gitsm fetcher: Fix recurse submodule error
Due to an error in some optimization work, recursive submodules may not always unpack. Resolve this issue and add a specific test case to catch any new regressions.
(already in master-next, but I wanted to make sure others saw the patch as well..)
Mark Hatle (1):
gitsm: The fetcher did not process some recursive submodules properly.
lib/bb/fetch2/gitsm.py | 4 +++- lib/bb/tests/fetch.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-)
Signature · 1 line
1.8.3.1
PATCH 1/1 gitsm: The fetcher did not process some recursive submodules properly.
Move the submodule processing outside of the if statement to avoid any optimizations that may happen.
Update the test cases to include the additional case, and split the other test cases into individual tests to make it easier to figure out what the failure may be.
--- lib/bb/fetch2/gitsm.py | 4 +++- lib/bb/tests/fetch.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-)
lib/bb/fetch2/gitsm.py +3 -1
@@ -198,6 +198,8 @@ class GitSM(Git): Git.unpack(self, ud, destdir, d) - if not ud.bareclone and self.process_submodules(ud, ud.destdir, unpack_submodules, d): + ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) + + if not ud.bareclone and ret: # Run submodule update, this sets up the directories -- without touching the config runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
lib/bb/tests/fetch.py +20 -0
@@ -912,6 +912,7 @@ class FetcherNetworkTest(FetcherTest): if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') + def test_git_submodule_dbus_broker(self): # The following external repositories have show failures in fetch and unpack operations # We want to avoid regressions! url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
@@ -928,16 +929,35 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') + def test_git_submodule_CLI11(self): url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted os.chdir(os.path.dirname(self.unpackdir)) fetcher.unpack(self.unpackdir) + + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') + def test_git_submodule_aktualizr(self): + url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # Previous cwd has been deleted + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') + class TrustedNetworksTest(FetcherTest): def test_trusted_network(self): # Ensure trusted_network returns False when the host IS in the list.
Signature · 1 line
1.8.3.1
Re: [PATCH] gitsm: The fetcher did not process some recursive submodules properly.
Quoted 8 lines · Mark Hatle
Move the submodule processing outside of the if statement to avoid any optimizations that may happen.
Update the test cases to include the additional case, and split the other test cases into individual tests to make it easier to figure out what the failure may be.
Signed-off-by Mark Hatle <[email protected]>
I can confirm that this resolves the issue we have been seeing:
Thanks Mark for looking into this!
--
Stefan
Quoted 99 lines
--- lib/bb/fetch2/gitsm.py | 4 +++- lib/bb/tests/fetch.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-)lib/bb/fetch2/gitsm.py +0
-1@@ -198,6 +198,8 @@ class GitSM(Git): Git.unpack(self, ud, destdir, d) - if not ud.bareclone and self.process_submodules(ud,ud.destdir,
unpack_submodules, d):lib/bb/fetch2/gitsm.py +3
-0+ ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) + + if not ud.bareclone and ret: # Run submodule update, this sets up the directories --without touching the config
runfetchcmd("%s submodule update --recursive --no-fetch"% (ud.basecmd), d, quiet=True, workdir=ud.destdir)
lib/bb/tests/fetch.py +0
-0@@ -912,6 +912,7 @@ class FetcherNetworkTest(FetcherTest): if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): self.assertTrue(os.path.exists(os.path.join(repo_path,'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
+ def
test_git_submodule_dbus_broker(self):
# The following external repositories have show failures in
fetch and unpack operations
# We want to avoid regressions!
url =
"gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"lib/bb/tests/fetch.py +0
-0@@ -928,16 +929,35 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path,'.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')+ def
test_git_submodule_CLI11(self):
url =
"gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
fetcher =bb.fetch.Fetch([url], self.d)
fetcher.download()
# Previous cwd has been deleted os.chdir(os.path.dirname(self.unpackdir))fetcher.unpack(self.unpackdir)lib/bb/tests/fetch.py +2
-0+ + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') self.assertTrue(os.path.exists(os.path.join(repo_path,'.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')lib/bb/tests/fetch.py +2
-0+ def test_git_submodule_aktualizr(self): + url ="gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
lib/bb/tests/fetch.py +8
-0+ fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # Previous cwd has been deleted + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') + self.assertTrue(os.path.exists(os.path.join(repo_path,'.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path,
'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
+ self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
+ self.assertTrue(os.path.exists(os.path.join(repo_path,
'.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
+
class TrustedNetworksTest(FetcherTest):
deftest_trusted_network(self):
# Ensuretrusted_networkreturns False when the host IS in the list.