Bitbake fetcher mirror handling overhaul
cover Bitbake fetcher mirror handling overhaul
The following commits overhaul bitbake's mirror handling in the fetcher bring various correctness fixes and improvements.
Richard Purdie (14):
fetch2/__init__: Make it clearer when uri_replace doesn't return a
match
fetch2: Fix error handling in uri_replace()
fetch2: Only cache data if fn is set, its pointless caching it
against a None value
fetch2: Ensure when downloading we are consistently in the same
directory
fetch2: Split try_mirrors into two parts
fetch2: Explicitly check for mirror tarballs in mirror handling code
fetch2: Improve mirror looping to consider more cases
fetch2: Simplify some looping constructs in uri_replace()
fetch2: Remove basestring test and simplify uri_replace
fetch2: Add parameter handling to uri_replace()
fetch2: uri_replace() remove what amounts to a null operation and add
some comments
fetch2: uri_replace() improve mirrortarball handling
fetch2: uri_replace() only consider ud.localpath where its a file test/fetch: Add in unit tests for uri_replace() and git premirrors
bitbake/lib/bb/fetch2/__init__.py | 203 ++++++++++++++++++++++--------------- bitbake/lib/bb/tests/fetch.py | 55 ++++++++++- 2 files changed, 175 insertions(+), 83 deletions(-)
Signature · 1 line
1.7.5.4
PATCH 2/14 fetch2: Fix error handling in uri_replace()
--- bitbake/lib/bb/fetch2/__init__.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -1
@@ -184,7 +184,8 @@ def encodeurl(decoded): def uri_replace(ud, uri_find, uri_replace, d): if not ud.url or not uri_find or not uri_replace: - logger.debug(1, "uri_replace: passed an undefined value, not replacing") + logger.error("uri_replace: passed an undefined value, not replacing") + return None uri_decoded = list(decodeurl(ud.url)) uri_find_decoded = list(decodeurl(uri_find)) uri_replace_decoded = list(decodeurl(uri_replace))
Signature · 1 line
1.7.5.4
PATCH 3/14 fetch2: Only cache data if fn is set, its pointless caching it against a None value
In reply to Richard Purdie--- bitbake/lib/bb/fetch2/__init__.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -2
@@ -1028,7 +1028,7 @@ class Fetch(object): self.ud = {} fn = d.getVar('FILE', True) - if cache and fn in urldata_cache: + if cache and fn and fn in urldata_cache: self.ud = urldata_cache[fn] for url in urls:
@@ -1040,7 +1040,7 @@ class Fetch(object): self.ud[url] = None pass - if cache: + if fn and cache: urldata_cache[fn] = self.ud def localpath(self, url):
Signature · 1 line
1.7.5.4
PATCH 1/14 fetch2/__init__: Make it clearer when uri_replace doesn't return a match
In reply to Richard Purdie--- bitbake/lib/bb/fetch2/__init__.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +4 -2
@@ -210,8 +210,10 @@ def uri_replace(ud, uri_find, uri_replace, d): if basename and not result_decoded[loc].endswith(basename): result_decoded[loc] = os.path.join(result_decoded[loc], basename) else: - return ud.url + return None result = encodeurl(result_decoded) + if result == ud.url: + return None logger.debug(2, "For url %s returning %s" % (ud.url, result)) return result
@@ -477,7 +479,7 @@ def try_mirrors(d, origud, mirrors, check = False): except ValueError: continue newuri = uri_replace(origud, find, replace, ld) - if newuri == origud.url: + if not newuri: continue try: ud = FetchData(newuri, ld)
Signature · 1 line
1.7.5.4
PATCH 13/14 fetch2: uri_replace() only consider ud.localpath where its a file
In reply to Richard PurdieUsing ud.localpath as a basename when it points at a directory causes problems. The supports_checksum() method gives a good indication of whether ud.localpath can be used in the way we need.
--- bitbake/lib/bb/fetch2/__init__.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +1 -1
@@ -217,7 +217,7 @@ def uri_replace(ud, uri_find, uri_replace, d): basename = os.path.basename(ud.mirrortarball) # Kill parameters, they make no sense for mirror tarballs uri_decoded[5] = {} - elif ud.localpath: + elif ud.localpath and ud.method.supports_checksum(ud): basename = os.path.basename(ud.localpath) if basename and not result_decoded[loc].endswith(basename): result_decoded[loc] = os.path.join(result_decoded[loc], basename)
Signature · 1 line
1.7.5.4
PATCH 6/14 fetch2: Explicitly check for mirror tarballs in mirror handling code
In reply to Richard PurdieWith support for things like git:// -> git:// urls, we need to be more explicity about the mirrortarball check since we need to fall through to the following code in other cases.
--- bitbake/lib/bb/fetch2/__init__.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -1
@@ -490,7 +490,8 @@ def try_mirror_url(newuri, origud, ud, ld, check = False): # We may be obtaining a mirror tarball which needs further processing by the real fetcher # If that tarball is a local file:// we need to provide a symlink to it dldir = ld.getVar("DL_DIR", True) - if os.path.basename(ud.localpath) != os.path.basename(origud.localpath): + if os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \ + and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): open(ud.donestamp, 'w').close() dest = os.path.join(dldir, os.path.basename(ud.localpath)) if not os.path.exists(dest):
Signature · 1 line
1.7.5.4
PATCH 12/14 fetch2: uri_replace() improve mirrortarball handling
In reply to Richard PurdieWe only consider mirror tarballs when the source and target urls are of differing types. We also should clear all url paramters when handling mirror tarballs.
--- bitbake/lib/bb/fetch2/__init__.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +4 -1
@@ -212,8 +212,11 @@ def uri_replace(ud, uri_find, uri_replace, d): if loc == 2: # Handle path manipulations basename = None - if ud.mirrortarball: + if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball: + # If the source and destination url types differ, must be a mirrortarball mapping basename = os.path.basename(ud.mirrortarball) + # Kill parameters, they make no sense for mirror tarballs + uri_decoded[5] = {} elif ud.localpath: basename = os.path.basename(ud.localpath) if basename and not result_decoded[loc].endswith(basename):
Signature · 1 line
1.7.5.4
PATCH 8/14 fetch2: Simplify some looping constructs in uri_replace()
In reply to Richard Purdie--- bitbake/lib/bb/fetch2/__init__.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -3
@@ -191,8 +191,7 @@ def uri_replace(ud, uri_find, uri_replace, d): uri_replace_decoded = list(decodeurl(uri_replace)) logger.debug(2, "For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) result_decoded = ['', '', '', '', '', {}] - for i in uri_find_decoded: - loc = uri_find_decoded.index(i) + for loc, i in enumerate(uri_find_decoded): result_decoded[loc] = uri_decoded[loc] if isinstance(i, basestring): if (re.match(i, uri_decoded[loc])):
@@ -200,7 +199,7 @@ def uri_replace(ud, uri_find, uri_replace, d): result_decoded[loc] = "" else: result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) - if uri_find_decoded.index(i) == 2: + if loc == 2: basename = None if ud.mirrortarball: basename = os.path.basename(ud.mirrortarball)
Signature · 1 line
1.7.5.4
PATCH 10/14 fetch2: Add parameter handling to uri_replace()
In reply to Richard PurdieThis means that parameters in the source expression are used as part of the match. Parameters in the destination are used explicitly in the final url.
--- bitbake/lib/bb/fetch2/__init__.py | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +9 -1
@@ -194,7 +194,15 @@ def uri_replace(ud, uri_find, uri_replace, d): for loc, i in enumerate(uri_find_decoded): result_decoded[loc] = uri_decoded[loc] if loc == 5: - continue + # Handle URL parameters + if i: + # Any specified URL parameters must match + for k in uri_replace_decoded[loc]: + if uri_decoded[loc][k] != uri_replace_decoded[loc][k]: + return None + # Overwrite any specified replacement parameters + for k in uri_replace_decoded[loc]: + result_decoded[loc][k] = uri_replace_decoded[loc][k] elif (re.match(i, uri_decoded[loc])): if not uri_replace_decoded[loc]: result_decoded[loc] = ""
Signature · 1 line
1.7.5.4
PATCH 11/14 fetch2: uri_replace() remove what amounts to a null operation and add some comments
In reply to Richard Purdie--- bitbake/lib/bb/fetch2/__init__.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -2
@@ -207,15 +207,15 @@ def uri_replace(ud, uri_find, uri_replace, d): if not uri_replace_decoded[loc]: result_decoded[loc] = "" else: + #bb.note("%s %s %s" % (i, uri_replace_decoded[loc], uri_decoded[loc])) result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) if loc == 2: + # Handle path manipulations basename = None if ud.mirrortarball: basename = os.path.basename(ud.mirrortarball) elif ud.localpath: basename = os.path.basename(ud.localpath) - if basename and result_decoded[loc].endswith("/"): - result_decoded[loc] = os.path.dirname(result_decoded[loc]) if basename and not result_decoded[loc].endswith(basename): result_decoded[loc] = os.path.join(result_decoded[loc], basename) else:
Signature · 1 line
1.7.5.4
PATCH 4/14 fetch2: Ensure when downloading we are consistently in the same directory
In reply to Richard PurdieThis assists with build reproducuility. It also avoids errors if cwd happens not to exist when we call into the fetcher. That situation would be unusual but I hit it with the unit tests.
--- bitbake/lib/bb/fetch2/__init__.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +5 -1
@@ -486,6 +486,8 @@ def try_mirrors(d, origud, mirrors, check = False): ud = FetchData(newuri, ld) ud.setup_localpath(ld) + os.chdir(ld.getVar("DL_DIR", True)) + if check: found = ud.method.checkstatus(newuri, ud, ld) if found:
@@ -1094,6 +1096,8 @@ class Fetch(object): if premirroronly: self.d.setVar("BB_NO_NETWORK", "1") + os.chdir(self.d.getVar("DL_DIR", True)) + firsterr = None if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(u, ud, self.d)): try:
@@ -1155,7 +1159,7 @@ class Fetch(object): except: # Finally, try checking uri, u, from MIRRORS mirrors = mirror_from_string(self.d.getVar('MIRRORS', True)) - ret = try_mirrors (self.d, ud, mirrors, True) + ret = try_mirrors(self.d, ud, mirrors, True) if not ret: raise FetchError("URL %s doesn't work" % u, u)
Signature · 1 line
1.7.5.4
PATCH 7/14 fetch2: Improve mirror looping to consider more cases
In reply to Richard PurdieCurrently we only consider one pass through the mirror list. This doesn't catch cases where for example you might want to setup a mirror of a mirror and allow multiple redirection. There is no reason we can't support this and the patch loops through the list recursively now.
As a safeguard, it will stop if any duplicate urls are found, hence avoiding circular dependency looping.
--- bitbake/lib/bb/fetch2/__init__.py | 41 ++++++++++++++++++++++++++---------- 1 files changed, 29 insertions(+), 12 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +29 -12
@@ -464,6 +464,30 @@ def check_network_access(d, info = "", url = None): else: logger.debug(1, "Fetcher accessed the network with the command %s" % info) +def build_mirroruris(origud, mirrors, ld): + uris = [] + uds = [] + + def adduri(uri, ud, uris, uds): + for line in mirrors: + try: + (find, replace) = line + except ValueError: + continue + newuri = uri_replace(ud, find, replace, ld) + if not newuri or newuri in uris or uri == origud.url: + continue + uris.append(newuri) + ud = FetchData(newuri, ld) + ud.setup_localpath(ld) + uds.append(ud) + + adduri(newuri, ud, uris, uds) + + adduri(None, origud, uris, uds) + + return uris, uds + def try_mirror_url(newuri, origud, ud, ld, check = False): # Return of None or a value means we're finished # False means try another url
@@ -529,18 +553,11 @@ def try_mirrors(d, origud, mirrors, check = False): mirrors is the list of mirrors we're going to try """ ld = d.createCopy() - for line in mirrors: - try: - (find, replace) = line - except ValueError: - continue - newuri = uri_replace(origud, find, replace, ld) - if not newuri: - continue - ud = FetchData(newuri, ld) - ud.setup_localpath(ld) - - ret = try_mirror_url(newuri, origud, ud, ld, check) + + uris, uds = build_mirroruris(origud, mirrors, ld) + + for index, uri in enumerate(uris): + ret = try_mirror_url(uri, origud, uds[index], ld, check) if ret != False: return ret return None
Signature · 1 line
1.7.5.4
PATCH 5/14 fetch2: Split try_mirrors into two parts
In reply to Richard PurdieThere are no functionality changes in this change
--- bitbake/lib/bb/fetch2/__init__.py | 111 +++++++++++++++++++----------------- 1 files changed, 59 insertions(+), 52 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +59 -52
@@ -464,6 +464,60 @@ def check_network_access(d, info = "", url = None): else: logger.debug(1, "Fetcher accessed the network with the command %s" % info) +def try_mirror_url(newuri, origud, ud, ld, check = False): + # Return of None or a value means we're finished + # False means try another url + try: + if check: + found = ud.method.checkstatus(newuri, ud, ld) + if found: + return found + return False + + os.chdir(ld.getVar("DL_DIR", True)) + + if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld): + ud.method.download(newuri, ud, ld) + if hasattr(ud.method,"build_mirror_data"): + ud.method.build_mirror_data(newuri, ud, ld) + + if not ud.localpath or not os.path.exists(ud.localpath): + return False + + if ud.localpath == origud.localpath: + return ud.localpath + + # We may be obtaining a mirror tarball which needs further processing by the real fetcher + # If that tarball is a local file:// we need to provide a symlink to it + dldir = ld.getVar("DL_DIR", True) + if os.path.basename(ud.localpath) != os.path.basename(origud.localpath): + open(ud.donestamp, 'w').close() + dest = os.path.join(dldir, os.path.basename(ud.localpath)) + if not os.path.exists(dest): + os.symlink(ud.localpath, dest) + return None + # Otherwise the result is a local file:// and we symlink to it + if not os.path.exists(origud.localpath): + os.symlink(ud.localpath, origud.localpath) + update_stamp(newuri, origud, ld) + return ud.localpath + + except bb.fetch2.NetworkAccess: + raise + + except bb.fetch2.BBFetchException as e: + if isinstance(e, ChecksumError): + logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (newuri, origud.url)) + logger.warn(str(e)) + else: + logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) + logger.debug(1, str(e)) + try: + ud.method.clean(ud, ld) + except UnboundLocalError: + pass + return False + def try_mirrors(d, origud, mirrors, check = False): """ Try to use a mirrored version of the sources.
@@ -482,59 +536,12 @@ def try_mirrors(d, origud, mirrors, check = False): newuri = uri_replace(origud, find, replace, ld) if not newuri: continue - try: - ud = FetchData(newuri, ld) - ud.setup_localpath(ld) - - os.chdir(ld.getVar("DL_DIR", True)) - - if check: - found = ud.method.checkstatus(newuri, ud, ld) - if found: - return found - continue + ud = FetchData(newuri, ld) + ud.setup_localpath(ld) - if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld): - ud.method.download(newuri, ud, ld) - if hasattr(ud.method,"build_mirror_data"): - ud.method.build_mirror_data(newuri, ud, ld) - - if not ud.localpath or not os.path.exists(ud.localpath): - continue - - if ud.localpath == origud.localpath: - return ud.localpath - - # We may be obtaining a mirror tarball which needs further processing by the real fetcher - # If that tarball is a local file:// we need to provide a symlink to it - dldir = ld.getVar("DL_DIR", True) - if os.path.basename(ud.localpath) != os.path.basename(origud.localpath): - open(ud.donestamp, 'w').close() - dest = os.path.join(dldir, os.path.basename(ud.localpath)) - if not os.path.exists(dest): - os.symlink(ud.localpath, dest) - return None - # Otherwise the result is a local file:// and we symlink to it - if not os.path.exists(origud.localpath): - os.symlink(ud.localpath, origud.localpath) - update_stamp(newuri, origud, ld) - return ud.localpath - - except bb.fetch2.NetworkAccess: - raise - - except bb.fetch2.BBFetchException as e: - if isinstance(e, ChecksumError): - logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (newuri, origud.url)) - logger.warn(str(e)) - else: - logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) - logger.debug(1, str(e)) - try: - ud.method.clean(ud, ld) - except UnboundLocalError: - pass - continue + ret = try_mirror_url(newuri, origud, ud, ld, check) + if ret != False: + return ret return None def srcrev_internal_helper(ud, d, name):
Signature · 1 line
1.7.5.4
PATCH 9/14 fetch2: Remove basestring test and simplify uri_replace
In reply to Richard Purdie--- bitbake/lib/bb/fetch2/__init__.py | 35 ++++++++++++++++++----------------- 1 files changed, 18 insertions(+), 17 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +18 -17
@@ -193,24 +193,25 @@ def uri_replace(ud, uri_find, uri_replace, d): result_decoded = ['', '', '', '', '', {}] for loc, i in enumerate(uri_find_decoded): result_decoded[loc] = uri_decoded[loc] - if isinstance(i, basestring): - if (re.match(i, uri_decoded[loc])): - if not uri_replace_decoded[loc]: - result_decoded[loc] = "" - else: - result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) - if loc == 2: - basename = None - if ud.mirrortarball: - basename = os.path.basename(ud.mirrortarball) - elif ud.localpath: - basename = os.path.basename(ud.localpath) - if basename and result_decoded[loc].endswith("/"): - result_decoded[loc] = os.path.dirname(result_decoded[loc]) - if basename and not result_decoded[loc].endswith(basename): - result_decoded[loc] = os.path.join(result_decoded[loc], basename) + if loc == 5: + continue + elif (re.match(i, uri_decoded[loc])): + if not uri_replace_decoded[loc]: + result_decoded[loc] = "" else: - return None + result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) + if loc == 2: + basename = None + if ud.mirrortarball: + basename = os.path.basename(ud.mirrortarball) + elif ud.localpath: + basename = os.path.basename(ud.localpath) + if basename and result_decoded[loc].endswith("/"): + result_decoded[loc] = os.path.dirname(result_decoded[loc]) + if basename and not result_decoded[loc].endswith(basename): + result_decoded[loc] = os.path.join(result_decoded[loc], basename) + else: + return None result = encodeurl(result_decoded) if result == ud.url: return None
Signature · 1 line
1.7.5.4
PATCH 14/14 test/fetch: Add in unit tests for uri_replace() and git premirrors
In reply to Richard Purdie--- bitbake/lib/bb/tests/fetch.py | 55 ++++++++++++++++++++++++++++++++++++++-- 1 files changed, 52 insertions(+), 3 deletions(-)
bitbake/lib/bb/tests/fetch.py +52 -3
@@ -26,6 +26,32 @@ import bb class FetcherTest(unittest.TestCase): + replaceuris = { + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") + : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", + ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") + : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") + : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", + ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") + : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") + : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") + : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") + : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", + ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") + : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", + ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") + : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2" + #Renaming files doesn't work + #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" + #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", + } + def setUp(self): self.d = bb.data.init() self.tempdir = tempfile.mkdtemp()
@@ -64,7 +90,7 @@ class FetcherTest(unittest.TestCase): fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) - def test_gitfetch(self): + def gitfetcher(self, url1, url2): def checkrevision(self, fetcher): fetcher.unpack(self.unpackdir) revision = subprocess.check_output("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git").strip()
@@ -72,17 +98,40 @@ class FetcherTest(unittest.TestCase): self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") - fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake"], self.d) + fetcher = bb.fetch.Fetch([url1], self.d) fetcher.download() checkrevision(self, fetcher) # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works bb.utils.prunedir(self.dldir + "/git2/") bb.utils.prunedir(self.unpackdir) self.d.setVar("BB_NO_NETWORK", "1") - fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake"], self.d) + fetcher = bb.fetch.Fetch([url2], self.d) fetcher.download() checkrevision(self, fetcher) + def test_gitfetch(self): + url1 = url2 = "git://git.openembedded.org/bitbake" + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror(self): + url1 = "git://git.openembedded.org/bitbake" + url2 = "git://someserver.org/bitbake" + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.gitfetcher(url1, url2) + + def test_gitfetch_premirror2(self): + url1 = url2 = "git://someserver.org/bitbake" + self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n") + self.gitfetcher(url1, url2) + + def test_urireplace(self): + for k, v in self.replaceuris.items(): + ud = bb.fetch.FetchData(k[0], self.d) + ud.setup_localpath(self.d) + newuris = bb.fetch2.uri_replace(ud, k[1], k[2], self.d) + self.assertEqual(newuris, v) + + class URLHandle(unittest.TestCase): datatable = {
Signature · 1 line
1.7.5.4