bitbake-user-manual: Remove unnecessary \n from a PREMIRRORS example
PATCH 1/9 bitbake-user-manual: Remove unnecessary \n from a PREMIRRORS example
Since commit 044fb04d (fetch2: Allow whitespace only mirror entries) there is no need to separate the entries in PREMIRRORS with "\n".
--- .../bitbake-user-manual-ref-variables.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +4 -4
@@ -1179,10 +1179,10 @@ overview of their function and contents. your configuration:: PREMIRRORS:prepend = "\ - git://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ - ftp://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ - http://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n \ - https://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \n" + git://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ + ftp://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ + http://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \ + https://.*/.* http://downloads.yoctoproject.org/mirror/sources/" These changes cause the build system to intercept Git, FTP, HTTP, and HTTPS requests and direct them to the ``http://`` sources mirror. You can
PATCH 3/9 tests/fetch: Skip the crate tests if network tests are disabled
Both test_crate_url and test_crate_url_multi require Internet access.
--- bitbake/lib/bb/tests/fetch.py | 2 ++ 1 file changed, 2 insertions(+)
bitbake/lib/bb/tests/fetch.py +2 -0
@@ -2261,6 +2261,7 @@ class GitURLWithSpacesTest(FetcherTest): self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) class CrateTest(FetcherTest): + @skipIfNoNetwork() def test_crate_url(self): uri = "crate://crates.io/glob/0.2.11"
@@ -2277,6 +2278,7 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + @skipIfNoNetwork() def test_crate_url_multi(self): uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35"
PATCH 2/9 tests/fetch: Remove unnecessary \n from mirror variables
In reply to Peter KjellerstedtSince commit 044fb04d (fetch2: Allow whitespace only mirror entries) there is no need to separate the entries in mirror variables with "\n".
--- bitbake/lib/bb/tests/fetch.py | 46 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-)
bitbake/lib/bb/tests/fetch.py +23 -23
@@ -442,10 +442,10 @@ class MirrorUriTest(FetcherTest): #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", } - mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \ - "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \ - "https://.*/.* file:///someotherpath/downloads/ \n" \ - "http://.*/.* file:///someotherpath/downloads/ \n" + mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ + "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ + "https://.*/.* file:///someotherpath/downloads/ " \ + "http://.*/.* file:///someotherpath/downloads/" def test_urireplace(self): for k, v in self.replaceuris.items():
@@ -470,8 +470,8 @@ class MirrorUriTest(FetcherTest): def test_mirror_of_mirror(self): # Test if mirror of a mirror works - mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n" - mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n" + mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" + mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) mirrors = bb.fetch2.mirror_from_string(mirrorvar) uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
@@ -480,8 +480,8 @@ class MirrorUriTest(FetcherTest): 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) - recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \ - "https://.*/[^/]* https://BBBB/B/B/B/ \n" + recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ + "https://.*/[^/]* https://BBBB/B/B/B/" def test_recursive(self): fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
@@ -503,7 +503,7 @@ class GitDownloadDirectoryNamingTest(FetcherTest): self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') def setup_mirror_rewrite(self): - self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) @skipIfNoNetwork() def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -552,7 +552,7 @@ class TarballNamingTest(FetcherTest): self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') def setup_mirror_rewrite(self): - self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) @skipIfNoNetwork() def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
@@ -587,7 +587,7 @@ class GitShallowTarballNamingTest(FetcherTest): self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') def setup_mirror_rewrite(self): - self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) @skipIfNoNetwork() def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -867,14 +867,14 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch_mirror_of_mirror(self): - self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) @skipIfNoNetwork() def test_fetch_file_mirror_of_mirror(self): - self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) os.mkdir(self.dldir + "/some2where") fetcher.download()
@@ -981,13 +981,13 @@ class FetcherNetworkTest(FetcherTest): def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): recipeurl = "git://git.openembedded.org/bitbake" mirrorurl = "git://someserver.org/bitbake" - self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") self.gitfetcher(recipeurl, mirrorurl) @skipIfNoNetwork() def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): recipeurl = "git://someserver.org/bitbake" - self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") self.gitfetcher(recipeurl, recipeurl) @skipIfNoNetwork()
@@ -997,7 +997,7 @@ class FetcherNetworkTest(FetcherTest): self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") os.chdir(self.tempdir) bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True) - self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir)) + self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) self.gitfetcher(recipeurl, recipeurl) @skipIfNoNetwork()
@@ -1852,7 +1852,7 @@ class GitShallowTest(FetcherTest): # Set up the mirror mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.rename(self.dldir, mirrordir) - self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) # Fetch from the mirror bb.utils.remove(self.dldir, recurse=True)
@@ -1967,7 +1967,7 @@ class GitShallowTest(FetcherTest): # Set up the mirror mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.mkdirhier(mirrordir) - self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) bb.utils.rename(os.path.join(self.dldir, mirrortarball), os.path.join(mirrordir, mirrortarball))
@@ -2349,7 +2349,7 @@ class NPMTest(FetcherTest): mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.mkdirhier(mirrordir) os.replace(ud.localpath, os.path.join(mirrordir, pkgname)) - self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') # Fetch again self.assertFalse(os.path.exists(ud.localpath))
@@ -2370,7 +2370,7 @@ class NPMTest(FetcherTest): bb.utils.mkdirhier(mirrordir) mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) os.replace(ud.localpath, mirrorfilename) - self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s\n' % mirrorfilename) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') # Fetch again self.assertFalse(os.path.exists(ud.localpath))
@@ -2390,7 +2390,7 @@ class NPMTest(FetcherTest): mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.mkdirhier(mirrordir) os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) - self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) # Update the resolved url to an invalid url with open(ud.resolvefile, 'r') as f: url = f.read()
@@ -2706,7 +2706,7 @@ class NPMTest(FetcherTest): mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.mkdirhier(mirrordir) os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) - self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') # Fetch again self.assertFalse(os.path.exists(ud.localpath))
@@ -2735,7 +2735,7 @@ class NPMTest(FetcherTest): mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.mkdirhier(mirrordir) os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) - self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) # Fetch again with invalid url self.assertFalse(os.path.exists(ud.localpath)) swfile = self.create_shrinkwrap_file({
PATCH 4/9 tests/fetch: Improve the verbose messages for skipped tests
In reply to Peter KjellerstedtThis changes:
test_foo .. skipped 'Network tests being skipped' test_foo .. skipped 'npm not installed, tests being skipped'
to:
test_foo .. skipped 'network test'
test_foo .. skipped 'npm not installed'
to avoid double "skipped" in the output.
--- bitbake/lib/bb/tests/fetch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
bitbake/lib/bb/tests/fetch.py +2 -2
@@ -18,7 +18,7 @@ from bb.tests.support.httpserver import HTTPService def skipIfNoNetwork(): if os.environ.get("BB_SKIP_NETTESTS") == "yes": - return unittest.skip("Network tests being skipped") + return unittest.skip("network test") return lambda f: f class URITest(unittest.TestCase):
@@ -2301,7 +2301,7 @@ class NPMTest(FetcherTest): def skipIfNoNpm(): import shutil if not shutil.which('npm'): - return unittest.skip('npm not installed, tests being skipped') + return unittest.skip('npm not installed') return lambda f: f @skipIfNoNpm()
PATCH 5/9 tests/fetch: Unify how git commands are run
In reply to Peter KjellerstedtThis unifies the three separate implementations of git() from GitMakeShallowTest, GitShallowTest and GitLfsTest into one implementation in FetcherTest. It also makes use of this in FetcherLocalTest and FetcherNetworkTest.
--- bitbake/lib/bb/tests/fetch.py | 73 +++++++++++++---------------------- 1 file changed, 27 insertions(+), 46 deletions(-)
bitbake/lib/bb/tests/fetch.py +27 -46
@@ -393,6 +393,15 @@ class FetcherTest(unittest.TestCase): bb.process.run('chmod u+rw -R %s' % self.tempdir) bb.utils.prunedir(self.tempdir) + def git(self, cmd, cwd=None): + if isinstance(cmd, str): + cmd = 'git ' + cmd + else: + cmd = ['git'] + cmd + if cwd is None: + cwd = self.gitdir + return bb.process.run(cmd, cwd=cwd)[0] + class MirrorUriTest(FetcherTest): replaceuris = {
@@ -699,28 +708,24 @@ class FetcherLocalTest(FetcherTest): # Create dummy local Git repo src_dir = tempfile.mkdtemp(dir=self.tempdir, prefix='gitfetch_localusehead_') - src_dir = os.path.abspath(src_dir) - bb.process.run("git init", cwd=src_dir) - bb.process.run("git config user.email '[email protected]'", cwd=src_dir) - bb.process.run("git config user.name 'Your Name'", cwd=src_dir) - bb.process.run("git commit --allow-empty -m'Dummy commit'", - cwd=src_dir) + self.gitdir = os.path.abspath(src_dir) + self.git('init') + self.git('config user.email "[email protected]"') + self.git('config user.name "Your Name"') + self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) # Use other branch than master - bb.process.run("git checkout -b my-devel", cwd=src_dir) - bb.process.run("git commit --allow-empty -m'Dummy commit 2'", - cwd=src_dir) - stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir) - orig_rev = stdout[0].strip() + self.git(['checkout', '-b', 'my-devel']) + self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) + orig_rev = self.git(['rev-parse', 'HEAD']).strip() # Fetch and check revision self.d.setVar("SRCREV", "AUTOINC") - url = "git://" + src_dir + ";protocol=file;" + suffix + url = "git://" + self.gitdir + ";protocol=file;" + suffix fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() fetcher.unpack(self.unpackdir) - stdout = bb.process.run("git rev-parse HEAD", - cwd=os.path.join(self.unpackdir, 'git')) - unpack_rev = stdout[0].strip() + unpack_rev = self.git(['rev-parse', 'HEAD'], + cwd=os.path.join(self.unpackdir, 'git')).strip() self.assertEqual(orig_rev, unpack_rev) def test_local_gitfetch_usehead(self):
@@ -920,7 +925,8 @@ class FetcherNetworkTest(FetcherTest): def gitfetcher(self, url1, url2): def checkrevision(self, fetcher): fetcher.unpack(self.unpackdir) - revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip() + revision = self.git(['rev-parse', 'HEAD'], + cwd=os.path.join(self.unpackdir, 'git')).strip() self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
@@ -996,7 +1002,7 @@ class FetcherNetworkTest(FetcherTest): recipeurl = "git://someserver.org/bitbake" self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") os.chdir(self.tempdir) - bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True) + self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) self.gitfetcher(recipeurl, recipeurl)
@@ -1437,9 +1443,9 @@ class GitMakeShallowTest(FetcherTest): FetcherTest.setUp(self) self.gitdir = os.path.join(self.tempdir, 'gitshallow') bb.utils.mkdirhier(self.gitdir) - bb.process.run('git init', cwd=self.gitdir) - bb.process.run('git config user.email "[email protected]"', cwd=self.gitdir) - bb.process.run('git config user.name "Your Name"', cwd=self.gitdir) + self.git('init') + self.git('config user.email "[email protected]"') + self.git('config user.name "Your Name"') def assertRefs(self, expected_refs): actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1453,13 +1459,6 @@ class GitMakeShallowTest(FetcherTest): actual_count = len(revs.splitlines()) self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) - def git(self, cmd): - if isinstance(cmd, str): - cmd = 'git ' + cmd - else: - cmd = ['git'] + cmd - return bb.process.run(cmd, cwd=self.gitdir)[0] - def make_shallow(self, args=None): if args is None: args = ['HEAD']
@@ -1595,15 +1594,6 @@ class GitShallowTest(FetcherTest): actual_count = len(revs.splitlines()) self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) - def git(self, cmd, cwd=None): - if isinstance(cmd, str): - cmd = 'git ' + cmd - else: - cmd = ['git'] + cmd - if cwd is None: - cwd = self.gitdir - return bb.process.run(cmd, cwd=cwd)[0] - def add_empty_file(self, path, cwd=None, msg=None): if msg is None: msg = path
@@ -2140,7 +2130,7 @@ class GitLfsTest(FetcherTest): self.gitdir = os.path.join(self.tempdir, 'git') self.srcdir = os.path.join(self.tempdir, 'gitsource') - + self.d.setVar('WORKDIR', self.tempdir) self.d.setVar('S', self.gitdir) self.d.delVar('PREMIRRORS')
@@ -2158,15 +2148,6 @@ class GitLfsTest(FetcherTest): self.git(['add', '.gitattributes'], cwd=self.srcdir) self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) - def git(self, cmd, cwd=None): - if isinstance(cmd, str): - cmd = 'git ' + cmd - else: - cmd = ['git'] + cmd - if cwd is None: - cwd = self.gitdir - return bb.process.run(cmd, cwd=cwd)[0] - def fetch(self, uri=None, download=True): uris = self.d.getVar('SRC_URI').split() uri = uris[0]
PATCH 6/9 tests/fetch: Only set the Git user name/email if they are not already set
In reply to Peter KjellerstedtIf they are set, assume they are correct.
--- bitbake/lib/bb/tests/fetch.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-)
bitbake/lib/bb/tests/fetch.py +13 -18
@@ -402,6 +402,13 @@ class FetcherTest(unittest.TestCase): cwd = self.gitdir return bb.process.run(cmd, cwd=cwd)[0] + def git_init(self, cwd=None): + self.git('init', cwd=cwd) + if not self.git(['config', 'user.email'], cwd=cwd): + self.git(['config', 'user.email', '[email protected]'], cwd=cwd) + if not self.git(['config', 'user.name'], cwd=cwd): + self.git(['config', 'user.name', 'Your Name'], cwd=cwd) + class MirrorUriTest(FetcherTest): replaceuris = {
@@ -709,9 +716,7 @@ class FetcherLocalTest(FetcherTest): src_dir = tempfile.mkdtemp(dir=self.tempdir, prefix='gitfetch_localusehead_') self.gitdir = os.path.abspath(src_dir) - self.git('init') - self.git('config user.email "[email protected]"') - self.git('config user.name "Your Name"') + self.git_init() self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) # Use other branch than master self.git(['checkout', '-b', 'my-devel'])
@@ -1443,9 +1448,7 @@ class GitMakeShallowTest(FetcherTest): FetcherTest.setUp(self) self.gitdir = os.path.join(self.tempdir, 'gitshallow') bb.utils.mkdirhier(self.gitdir) - self.git('init') - self.git('config user.email "[email protected]"') - self.git('config user.name "Your Name"') + self.git_init() def assertRefs(self, expected_refs): actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1561,9 +1564,7 @@ class GitShallowTest(FetcherTest): self.srcdir = os.path.join(self.tempdir, 'gitsource') bb.utils.mkdirhier(self.srcdir) - self.git('init', cwd=self.srcdir) - self.git('config user.email "[email protected]"', cwd=self.srcdir) - self.git('config user.name "Your Name"', cwd=self.srcdir) + self.git_init(cwd=self.srcdir) self.d.setVar('WORKDIR', self.tempdir) self.d.setVar('S', self.gitdir) self.d.delVar('PREMIRRORS')
@@ -1788,9 +1789,7 @@ class GitShallowTest(FetcherTest): smdir = os.path.join(self.tempdir, 'gitsubmodule') bb.utils.mkdirhier(smdir) - self.git('init', cwd=smdir) - self.git('config user.email "[email protected]"', cwd=smdir) - self.git('config user.name "Your Name"', cwd=smdir) + self.git_init(cwd=smdir) # Make this look like it was cloned from a remote... self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1820,9 +1819,7 @@ class GitShallowTest(FetcherTest): smdir = os.path.join(self.tempdir, 'gitsubmodule') bb.utils.mkdirhier(smdir) - self.git('init', cwd=smdir) - self.git('config user.email "[email protected]"', cwd=smdir) - self.git('config user.name "Your Name"', cwd=smdir) + self.git_init(cwd=smdir) # Make this look like it was cloned from a remote... self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -2140,9 +2137,7 @@ class GitLfsTest(FetcherTest): self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') bb.utils.mkdirhier(self.srcdir) - self.git('init', cwd=self.srcdir) - self.git('config user.email "[email protected]"', cwd=self.srcdir) - self.git('config user.name "Your Name"', cwd=self.srcdir) + self.git_init(cwd=self.srcdir) with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: attrs.write('*.mp3 filter=lfs -text') self.git(['add', '.gitattributes'], cwd=self.srcdir)
PATCH 7/9 tests/fetch: Make test_npm_premirrors work with the current fetcher
In reply to Peter KjellerstedtThere are two totally opposite use cases for how a premirror is expected to behave in combination with specifying a downloadfilename= parameter in the SRC_URI. On the one hand there is the expectation that it works like any other mirror, which means the premirror is expected to contain a file with the original name specified in the SRC_URI. On the other hand there is the expectation that one can use the artefacts downloaded by bitbake in ${DL_DIR} as a premirror, in which case it is expected to contain a file with the name from the downloadfilename= parameter.
The latter case has been how downloaded files have been handled until commit 8a3ff9f3 (fetch2: fix premirror URI when downloadfilename defined), where the fetcher was changed to store files as per the first case. This is also when the test_npm_premirrors test case was modified in commit 5ba191a0 (tests/fetch: add and fix npm tests) to expect the first case.
The above change was later reverted in commit 96c30007 (fetch2: fix downloadfilename issue with premirror). However the test_npm_premirrors test case was not updated to match, and has been failing ever since. This has probably gone unnoticed because the npm related test cases require that npm is installed on the host.
This commit updates test_npm_premirrors to expect that premirrors use the filenames specified by downloadfilename= as this matches the current fetcher implementation and also is the most likely use case for premirrors. It also tries to mimic how one typically might setup the premirror directory by simply copying the download directory.
--- bitbake/lib/bb/tests/fetch.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-)
bitbake/lib/bb/tests/fetch.py +14 -5
@@ -2322,15 +2322,24 @@ class NPMTest(FetcherTest): ud = fetcher.ud[fetcher.urls[0]] fetcher.download() self.assertTrue(os.path.exists(ud.localpath)) - # Setup the mirror - pkgname = os.path.basename(ud.proxy.urls[0].split(';')[0]) + + # Setup the mirror by renaming the download directory mirrordir = os.path.join(self.tempdir, 'mirror') - bb.utils.mkdirhier(mirrordir) - os.replace(ud.localpath, os.path.join(mirrordir, pkgname)) - self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) + bb.utils.rename(self.dldir, mirrordir) + os.mkdir(self.dldir) + + # Configure the premirror to be used + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') + # Fetch again self.assertFalse(os.path.exists(ud.localpath)) + # The npm fetcher doesn't handle that the .resolved file disappears + # while the fetcher object exists, which it does when we rename the + # download directory to "mirror" above. Thus we need a new fetcher to go + # with the now empty download directory. + fetcher = bb.fetch.Fetch([url], self.d) + ud = fetcher.ud[fetcher.urls[0]] fetcher.download() self.assertTrue(os.path.exists(ud.localpath))
PATCH 8/9 fetch2: Correct handling of replacing the basename in URIs
In reply to Peter KjellerstedtThe solution implementated in commit 96c30007 (fetch2: fix downloadfilename issue with premirror) missed two corner cases. The first is if the basename of the original URI also appears somewhere else in the replacement URI, in which case it would also be replaced. The second is if the basename of the original URI partially matches the basename of the replacement URI, in which case the inital part of the basename from the replacement URI would be left behind.
The second case caused test_npm_premirrors_with_specified_filename to fail.
The solution is to prefix the basename with a slash when matching to avoid partial matches, and only replace the basename at the end of the URI.
This also adds two test cases that test for these problems. Before this they would give the following errors:
- ['file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz']
+ ['file:///mirror/some-example-1.0.0.tgz/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz']
? +++++ ++++++++++
- ['file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz']
+ ['file:///mirror/some-some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz']
? +++++
--- bitbake/lib/bb/fetch2/__init__.py | 11 +++++---- bitbake/lib/bb/tests/fetch.py | 39 +++++++++++++++++-------------- 2 files changed, 28 insertions(+), 22 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +7 -4
@@ -473,10 +473,13 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): basename = os.path.basename(ud.localpath) if basename: uri_basename = os.path.basename(uri_decoded[loc]) - if uri_basename and basename != uri_basename and result_decoded[loc].endswith(uri_basename): - result_decoded[loc] = result_decoded[loc].replace(uri_basename, basename) - elif not result_decoded[loc].endswith(basename): - result_decoded[loc] = os.path.join(result_decoded[loc], basename) + # Prefix with a slash as a sentinel in case + # result_decoded[loc] does not contain one. + path = "/" + result_decoded[loc] + if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename): + result_decoded[loc] = path[1:-len(uri_basename)] + basename + elif not path.endswith("/" + basename): + result_decoded[loc] = os.path.join(path[1:], basename) else: return None result = encodeurl(result_decoded)
bitbake/lib/bb/tests/fetch.py +21 -18
@@ -412,34 +412,34 @@ class FetcherTest(unittest.TestCase): class MirrorUriTest(FetcherTest): replaceuris = { - ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", - ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") - : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") + : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", - ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", - ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", - ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") + ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", - ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") + ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", - ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") - : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", - ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") - : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") + : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") : "http://somewhere2.org/somefile_1.2.3.tar.gz", ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
@@ -450,8 +450,11 @@ class MirrorUriTest(FetcherTest): : "git://[email protected]/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", - ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", + ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") + : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", + ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") + : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", #Renaming files doesn't work #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
PATCH 9/9 fetch2: npm: Put all downloaded files in the npm2 directory
In reply to Peter KjellerstedtPreviously npm files that specify downloadfilename= in the SRC_URI would be downloaded to the root of ${DL_DIR} rather than in the ${DL_DIR}/npm2 directory where all other npm files are downloaded. This should make it simpler when setting up and configuring a premirror with the downloaded npm packages.
--- bitbake/lib/bb/fetch2/npm.py | 10 +++++++--- bitbake/lib/bb/fetch2/npmsw.py | 2 +- bitbake/lib/bb/tests/fetch.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-)
bitbake/lib/bb/fetch2/npm.py +7 -3
@@ -52,9 +52,13 @@ def npm_filename(package, version): """Get the filename of a npm package""" return npm_package(package) + "-" + version + ".tgz" -def npm_localfile(package, version): +def npm_localfile(package, version=None): """Get the local filename of a npm package""" - return os.path.join("npm2", npm_filename(package, version)) + if version is not None: + filename = npm_filename(package, version) + else: + filename = package + return os.path.join("npm2", filename) def npm_integrity(integrity): """
@@ -157,7 +161,7 @@ class Npm(FetchMethod): # Using the 'downloadfilename' parameter as local filename # or the npm package name. if "downloadfilename" in ud.parm: - ud.localfile = d.expand(ud.parm["downloadfilename"]) + ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"])) else: ud.localfile = npm_localfile(ud.package, ud.version)
bitbake/lib/bb/fetch2/npmsw.py +1 -1
@@ -117,7 +117,7 @@ class NpmShrinkWrap(FetchMethod): # Handle http tarball sources elif version.startswith("http") and integrity: - localfile = os.path.join("npm2", os.path.basename(version)) + localfile = npm_localfile(os.path.basename(version)) uri = URI(version) uri.params["downloadfilename"] = localfile
bitbake/lib/bb/tests/fetch.py +1 -1
@@ -2399,7 +2399,7 @@ class NPMTest(FetcherTest): url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() - self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) fetcher.unpack(self.unpackdir) unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))