Skip to contentSkip to conversation

Bitbake fetcher mirror handling overhaul

PATCH New Richard Purdie 1 participant 15 messages Jun 20, 2012

lore
Share & tools
lore
#0 lore raw

cover Bitbake fetcher mirror handling overhaul

The following commits overhaul bitbake's mirror handling in the fetcher bring various correctness fixes and improvements.

Richard Purdie (14):
fetch2/__init__: Make it clearer when uri_replace doesn't return a
match
fetch2: Fix error handling in uri_replace()
fetch2: Only cache data if fn is set, its pointless caching it
against a None value
fetch2: Ensure when downloading we are consistently in the same
directory
fetch2: Split try_mirrors into two parts
fetch2: Explicitly check for mirror tarballs in mirror handling code
fetch2: Improve mirror looping to consider more cases
fetch2: Simplify some looping constructs in uri_replace()
fetch2: Remove basestring test and simplify uri_replace
fetch2: Add parameter handling to uri_replace()
fetch2: uri_replace() remove what amounts to a null operation and add
some comments
fetch2: uri_replace() improve mirrortarball handling
fetch2: uri_replace() only consider ud.localpath where its a file test/fetch: Add in unit tests for uri_replace() and git premirrors

 bitbake/lib/bb/fetch2/__init__.py |  203 ++++++++++++++++++++++---------------
 bitbake/lib/bb/tests/fetch.py     |   55 ++++++++++-
 2 files changed, 175 insertions(+), 83 deletions(-)
Signature · 1 line
1.7.5.4
#2 lore raw

PATCH 2/14 fetch2: Fix error handling in uri_replace()

---
 bitbake/lib/bb/fetch2/__init__.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -1
index 73ac73a..067d60c 100644
@@ -184,7 +184,8 @@ def encodeurl(decoded):
 
 def uri_replace(ud, uri_find, uri_replace, d):
     if not ud.url or not uri_find or not uri_replace:
-        logger.debug(1, "uri_replace: passed an undefined value, not replacing")
+        logger.error("uri_replace: passed an undefined value, not replacing")
+        return None
     uri_decoded = list(decodeurl(ud.url))
     uri_find_decoded = list(decodeurl(uri_find))
     uri_replace_decoded = list(decodeurl(uri_replace))
Signature · 1 line
1.7.5.4
#3 lore raw

PATCH 3/14 fetch2: Only cache data if fn is set, its pointless caching it against a None value

In reply to Richard Purdie
---
 bitbake/lib/bb/fetch2/__init__.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -2
index 067d60c..2eeec3d 100644
@@ -1028,7 +1028,7 @@ class Fetch(object):
         self.ud = {}
 
         fn = d.getVar('FILE', True)
-        if cache and fn in urldata_cache:
+        if cache and fn and fn in urldata_cache:
             self.ud = urldata_cache[fn]
 
         for url in urls:
@@ -1040,7 +1040,7 @@ class Fetch(object):
                         self.ud[url] = None
                         pass
 
-        if cache:
+        if fn and cache:
             urldata_cache[fn] = self.ud
 
     def localpath(self, url):
Signature · 1 line
1.7.5.4
#1 lore raw

PATCH 1/14 fetch2/__init__: Make it clearer when uri_replace doesn't return a match

In reply to Richard Purdie
---
 bitbake/lib/bb/fetch2/__init__.py |    6 ++++--
 1 files changed, 4 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +4 -2
index 324eef2..73ac73a 100644
@@ -210,8 +210,10 @@ def uri_replace(ud, uri_find, uri_replace, d):
                     if basename and not result_decoded[loc].endswith(basename):
                         result_decoded[loc] = os.path.join(result_decoded[loc], basename)
             else:
-                return ud.url
+                return None
     result = encodeurl(result_decoded)
+    if result == ud.url:
+        return None
     logger.debug(2, "For url %s returning %s" % (ud.url, result))
     return result
 
@@ -477,7 +479,7 @@ def try_mirrors(d, origud, mirrors, check = False):
         except ValueError:
             continue
         newuri = uri_replace(origud, find, replace, ld)
-        if newuri == origud.url:
+        if not newuri:
             continue
         try:
             ud = FetchData(newuri, ld)
Signature · 1 line
1.7.5.4
#13 lore raw

PATCH 13/14 fetch2: uri_replace() only consider ud.localpath where its a file

In reply to Richard Purdie

Using ud.localpath as a basename when it points at a directory causes problems. The supports_checksum() method gives a good indication of whether ud.localpath can be used in the way we need.

---
 bitbake/lib/bb/fetch2/__init__.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +1 -1
index 63e2e48..e222fdb 100644
@@ -217,7 +217,7 @@ def uri_replace(ud, uri_find, uri_replace, d):
                     basename = os.path.basename(ud.mirrortarball)
                     # Kill parameters, they make no sense for mirror tarballs
                     uri_decoded[5] = {}
-                elif ud.localpath:
+                elif ud.localpath and ud.method.supports_checksum(ud):
                     basename = os.path.basename(ud.localpath)
                 if basename and not result_decoded[loc].endswith(basename):
                     result_decoded[loc] = os.path.join(result_decoded[loc], basename)
Signature · 1 line
1.7.5.4
#6 lore raw

PATCH 6/14 fetch2: Explicitly check for mirror tarballs in mirror handling code

In reply to Richard Purdie

With support for things like git:// -> git:// urls, we need to be more explicity about the mirrortarball check since we need to fall through to the following code in other cases.

---
 bitbake/lib/bb/fetch2/__init__.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -1
index e4af951..a2e0beb 100644
@@ -490,7 +490,8 @@ def try_mirror_url(newuri, origud, ud, ld, check = False):
         # We may be obtaining a mirror tarball which needs further processing by the real fetcher
         # If that tarball is a local file:// we need to provide a symlink to it
         dldir = ld.getVar("DL_DIR", True)
-        if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+        if os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
+                and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
             open(ud.donestamp, 'w').close()
             dest = os.path.join(dldir, os.path.basename(ud.localpath))
             if not os.path.exists(dest):
Signature · 1 line
1.7.5.4
#12 lore raw

PATCH 12/14 fetch2: uri_replace() improve mirrortarball handling

In reply to Richard Purdie

We only consider mirror tarballs when the source and target urls are of differing types. We also should clear all url paramters when handling mirror tarballs.

---
 bitbake/lib/bb/fetch2/__init__.py |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +4 -1
index 945db7b..63e2e48 100644
@@ -212,8 +212,11 @@ def uri_replace(ud, uri_find, uri_replace, d):
             if loc == 2:
                 # Handle path manipulations
                 basename = None
-                if ud.mirrortarball:
+                if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball:
+                    # If the source and destination url types differ, must be a mirrortarball mapping
                     basename = os.path.basename(ud.mirrortarball)
+                    # Kill parameters, they make no sense for mirror tarballs
+                    uri_decoded[5] = {}
                 elif ud.localpath:
                     basename = os.path.basename(ud.localpath)
                 if basename and not result_decoded[loc].endswith(basename):
Signature · 1 line
1.7.5.4
#8 lore raw

PATCH 8/14 fetch2: Simplify some looping constructs in uri_replace()

In reply to Richard Purdie
---
 bitbake/lib/bb/fetch2/__init__.py |    5 ++---
 1 files changed, 2 insertions(+), 3 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -3
index 5838436..fe0c6a7 100644
@@ -191,8 +191,7 @@ def uri_replace(ud, uri_find, uri_replace, d):
     uri_replace_decoded = list(decodeurl(uri_replace))
     logger.debug(2, "For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
     result_decoded = ['', '', '', '', '', {}]
-    for i in uri_find_decoded:
-        loc = uri_find_decoded.index(i)
+    for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
         if isinstance(i, basestring):
             if (re.match(i, uri_decoded[loc])):
@@ -200,7 +199,7 @@ def uri_replace(ud, uri_find, uri_replace, d):
                     result_decoded[loc] = ""    
                 else:
                     result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
-                if uri_find_decoded.index(i) == 2:
+                if loc == 2:
                     basename = None
                     if ud.mirrortarball:
                         basename = os.path.basename(ud.mirrortarball)
Signature · 1 line
1.7.5.4
#10 lore raw

PATCH 10/14 fetch2: Add parameter handling to uri_replace()

In reply to Richard Purdie

This means that parameters in the source expression are used as part of the match. Parameters in the destination are used explicitly in the final url.

---
 bitbake/lib/bb/fetch2/__init__.py |   10 +++++++++-
 1 files changed, 9 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +9 -1
index 7045e1b..37f13e2 100644
@@ -194,7 +194,15 @@ def uri_replace(ud, uri_find, uri_replace, d):
     for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
         if loc == 5:
-            continue
+            # Handle URL parameters
+            if i:
+                # Any specified URL parameters must match
+                for k in uri_replace_decoded[loc]:
+                    if uri_decoded[loc][k] != uri_replace_decoded[loc][k]:
+                        return None
+            # Overwrite any specified replacement parameters
+            for k in uri_replace_decoded[loc]:
+                result_decoded[loc][k] = uri_replace_decoded[loc][k]
         elif (re.match(i, uri_decoded[loc])):
             if not uri_replace_decoded[loc]:
                 result_decoded[loc] = ""    
Signature · 1 line
1.7.5.4
#11 lore raw

PATCH 11/14 fetch2: uri_replace() remove what amounts to a null operation and add some comments

In reply to Richard Purdie
---
 bitbake/lib/bb/fetch2/__init__.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +2 -2
index 37f13e2..945db7b 100644
@@ -207,15 +207,15 @@ def uri_replace(ud, uri_find, uri_replace, d):
             if not uri_replace_decoded[loc]:
                 result_decoded[loc] = ""    
             else:
+                #bb.note("%s %s %s" % (i, uri_replace_decoded[loc], uri_decoded[loc]))
                 result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
             if loc == 2:
+                # Handle path manipulations
                 basename = None
                 if ud.mirrortarball:
                     basename = os.path.basename(ud.mirrortarball)
                 elif ud.localpath:
                     basename = os.path.basename(ud.localpath)
-                if basename and result_decoded[loc].endswith("/"):
-                    result_decoded[loc] = os.path.dirname(result_decoded[loc])
                 if basename and not result_decoded[loc].endswith(basename):
                     result_decoded[loc] = os.path.join(result_decoded[loc], basename)
         else:
Signature · 1 line
1.7.5.4
#4 lore raw

PATCH 4/14 fetch2: Ensure when downloading we are consistently in the same directory

In reply to Richard Purdie

This assists with build reproducuility. It also avoids errors if cwd happens not to exist when we call into the fetcher. That situation would be unusual but I hit it with the unit tests.

---
 bitbake/lib/bb/fetch2/__init__.py |    6 +++++-
 1 files changed, 5 insertions(+), 1 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +5 -1
index 2eeec3d..b09753f 100644
@@ -486,6 +486,8 @@ def try_mirrors(d, origud, mirrors, check = False):
             ud = FetchData(newuri, ld)
             ud.setup_localpath(ld)
 
+            os.chdir(ld.getVar("DL_DIR", True))
+
             if check:
                 found = ud.method.checkstatus(newuri, ud, ld)
                 if found:
@@ -1094,6 +1096,8 @@ class Fetch(object):
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
 
+                os.chdir(self.d.getVar("DL_DIR", True))
+
                 firsterr = None
                 if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(u, ud, self.d)):
                     try:
@@ -1155,7 +1159,7 @@ class Fetch(object):
                 except:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
-                    ret = try_mirrors (self.d, ud, mirrors, True)
+                    ret = try_mirrors(self.d, ud, mirrors, True)
 
             if not ret:
                 raise FetchError("URL %s doesn't work" % u, u)
Signature · 1 line
1.7.5.4
#7 lore raw

PATCH 7/14 fetch2: Improve mirror looping to consider more cases

In reply to Richard Purdie

Currently we only consider one pass through the mirror list. This doesn't catch cases where for example you might want to setup a mirror of a mirror and allow multiple redirection. There is no reason we can't support this and the patch loops through the list recursively now.

As a safeguard, it will stop if any duplicate urls are found, hence avoiding circular dependency looping.

---
 bitbake/lib/bb/fetch2/__init__.py |   41 ++++++++++++++++++++++++++----------
 1 files changed, 29 insertions(+), 12 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +29 -12
index a2e0beb..5838436 100644
@@ -464,6 +464,30 @@ def check_network_access(d, info = "", url = None):
     else:
         logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
+def build_mirroruris(origud, mirrors, ld):
+    uris = []
+    uds = []
+
+    def adduri(uri, ud, uris, uds):
+        for line in mirrors:
+            try:
+                (find, replace) = line
+            except ValueError:
+                continue
+            newuri = uri_replace(ud, find, replace, ld)
+            if not newuri or newuri in uris or uri == origud.url:
+                continue
+            uris.append(newuri)
+            ud = FetchData(newuri, ld)
+            ud.setup_localpath(ld)
+            uds.append(ud)
+
+            adduri(newuri, ud, uris, uds)
+
+    adduri(None, origud, uris, uds)
+
+    return uris, uds
+
 def try_mirror_url(newuri, origud, ud, ld, check = False):
     # Return of None or a value means we're finished
     # False means try another url
@@ -529,18 +553,11 @@ def try_mirrors(d, origud, mirrors, check = False):
     mirrors is the list of mirrors we're going to try
     """
     ld = d.createCopy()
-    for line in mirrors:
-        try:
-            (find, replace) = line
-        except ValueError:
-            continue
-        newuri = uri_replace(origud, find, replace, ld)
-        if not newuri:
-            continue
-        ud = FetchData(newuri, ld)
-        ud.setup_localpath(ld)
-
-        ret = try_mirror_url(newuri, origud, ud, ld, check)
+
+    uris, uds = build_mirroruris(origud, mirrors, ld)
+
+    for index, uri in enumerate(uris):
+        ret = try_mirror_url(uri, origud, uds[index], ld, check)
         if ret != False:
             return ret
     return None
Signature · 1 line
1.7.5.4
#5 lore raw

PATCH 5/14 fetch2: Split try_mirrors into two parts

In reply to Richard Purdie

There are no functionality changes in this change

---
 bitbake/lib/bb/fetch2/__init__.py |  111 +++++++++++++++++++-----------------
 1 files changed, 59 insertions(+), 52 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +59 -52
index b09753f..e4af951 100644
@@ -464,6 +464,60 @@ def check_network_access(d, info = "", url = None):
     else:
         logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
+def try_mirror_url(newuri, origud, ud, ld, check = False):
+    # Return of None or a value means we're finished
+    # False means try another url
+    try:
+        if check:
+            found = ud.method.checkstatus(newuri, ud, ld)
+            if found:
+                return found
+            return False
+
+        os.chdir(ld.getVar("DL_DIR", True))
+
+        if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld):
+            ud.method.download(newuri, ud, ld)
+            if hasattr(ud.method,"build_mirror_data"):
+                ud.method.build_mirror_data(newuri, ud, ld)
+
+        if not ud.localpath or not os.path.exists(ud.localpath):
+            return False
+
+        if ud.localpath == origud.localpath:
+            return ud.localpath
+
+        # We may be obtaining a mirror tarball which needs further processing by the real fetcher
+        # If that tarball is a local file:// we need to provide a symlink to it
+        dldir = ld.getVar("DL_DIR", True)
+        if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+            open(ud.donestamp, 'w').close()
+            dest = os.path.join(dldir, os.path.basename(ud.localpath))
+            if not os.path.exists(dest):
+                os.symlink(ud.localpath, dest)
+            return None
+        # Otherwise the result is a local file:// and we symlink to it
+        if not os.path.exists(origud.localpath):
+             os.symlink(ud.localpath, origud.localpath)
+        update_stamp(newuri, origud, ld)
+        return ud.localpath
+
+    except bb.fetch2.NetworkAccess:
+        raise
+
+    except bb.fetch2.BBFetchException as e:
+        if isinstance(e, ChecksumError):
+            logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (newuri, origud.url))
+            logger.warn(str(e))
+        else:
+            logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+            logger.debug(1, str(e))
+        try:
+            ud.method.clean(ud, ld)
+        except UnboundLocalError:
+            pass
+        return False
+
 def try_mirrors(d, origud, mirrors, check = False):
     """
     Try to use a mirrored version of the sources.
@@ -482,59 +536,12 @@ def try_mirrors(d, origud, mirrors, check = False):
         newuri = uri_replace(origud, find, replace, ld)
         if not newuri:
             continue
-        try:
-            ud = FetchData(newuri, ld)
-            ud.setup_localpath(ld)
-
-            os.chdir(ld.getVar("DL_DIR", True))
-
-            if check:
-                found = ud.method.checkstatus(newuri, ud, ld)
-                if found:
-                    return found
-                continue
+        ud = FetchData(newuri, ld)
+        ud.setup_localpath(ld)
 
-            if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld):
-                ud.method.download(newuri, ud, ld)
-                if hasattr(ud.method,"build_mirror_data"):
-                    ud.method.build_mirror_data(newuri, ud, ld)
-
-            if not ud.localpath or not os.path.exists(ud.localpath):
-                continue
-
-            if ud.localpath == origud.localpath:
-                return ud.localpath
-
-            # We may be obtaining a mirror tarball which needs further processing by the real fetcher
-            # If that tarball is a local file:// we need to provide a symlink to it
-            dldir = ld.getVar("DL_DIR", True)
-            if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
-                open(ud.donestamp, 'w').close()
-                dest = os.path.join(dldir, os.path.basename(ud.localpath))
-                if not os.path.exists(dest):
-                    os.symlink(ud.localpath, dest)
-                return None
-            # Otherwise the result is a local file:// and we symlink to it
-            if not os.path.exists(origud.localpath):
-                 os.symlink(ud.localpath, origud.localpath)
-            update_stamp(newuri, origud, ld)
-            return ud.localpath
-
-        except bb.fetch2.NetworkAccess:
-            raise
-
-        except bb.fetch2.BBFetchException as e:
-            if isinstance(e, ChecksumError):
-                logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (newuri, origud.url))
-                logger.warn(str(e))
-            else:
-                logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
-                logger.debug(1, str(e))
-            try:
-                ud.method.clean(ud, ld)
-            except UnboundLocalError:
-                pass
-            continue
+        ret = try_mirror_url(newuri, origud, ud, ld, check)
+        if ret != False:
+            return ret
     return None
 
 def srcrev_internal_helper(ud, d, name):
Signature · 1 line
1.7.5.4
#9 lore raw

PATCH 9/14 fetch2: Remove basestring test and simplify uri_replace

In reply to Richard Purdie
---
 bitbake/lib/bb/fetch2/__init__.py |   35 ++++++++++++++++++-----------------
 1 files changed, 18 insertions(+), 17 deletions(-)
bitbake/lib/bb/fetch2/__init__.py +18 -17
index fe0c6a7..7045e1b 100644
@@ -193,24 +193,25 @@ def uri_replace(ud, uri_find, uri_replace, d):
     result_decoded = ['', '', '', '', '', {}]
     for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
-        if isinstance(i, basestring):
-            if (re.match(i, uri_decoded[loc])):
-                if not uri_replace_decoded[loc]:
-                    result_decoded[loc] = ""    
-                else:
-                    result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
-                if loc == 2:
-                    basename = None
-                    if ud.mirrortarball:
-                        basename = os.path.basename(ud.mirrortarball)
-                    elif ud.localpath:
-                        basename = os.path.basename(ud.localpath)
-                    if basename and result_decoded[loc].endswith("/"):
-                        result_decoded[loc] = os.path.dirname(result_decoded[loc])
-                    if basename and not result_decoded[loc].endswith(basename):
-                        result_decoded[loc] = os.path.join(result_decoded[loc], basename)
+        if loc == 5:
+            continue
+        elif (re.match(i, uri_decoded[loc])):
+            if not uri_replace_decoded[loc]:
+                result_decoded[loc] = ""    
             else:
-                return None
+                result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
+            if loc == 2:
+                basename = None
+                if ud.mirrortarball:
+                    basename = os.path.basename(ud.mirrortarball)
+                elif ud.localpath:
+                    basename = os.path.basename(ud.localpath)
+                if basename and result_decoded[loc].endswith("/"):
+                    result_decoded[loc] = os.path.dirname(result_decoded[loc])
+                if basename and not result_decoded[loc].endswith(basename):
+                    result_decoded[loc] = os.path.join(result_decoded[loc], basename)
+        else:
+            return None
     result = encodeurl(result_decoded)
     if result == ud.url:
         return None
Signature · 1 line
1.7.5.4
#14 lore raw

PATCH 14/14 test/fetch: Add in unit tests for uri_replace() and git premirrors

In reply to Richard Purdie
---
 bitbake/lib/bb/tests/fetch.py |   55 ++++++++++++++++++++++++++++++++++++++--
 1 files changed, 52 insertions(+), 3 deletions(-)
bitbake/lib/bb/tests/fetch.py +52 -3
index 42af883..d890ec9 100644
@@ -26,6 +26,32 @@ import bb
 
 class FetcherTest(unittest.TestCase):
 
+    replaceuris = {
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 
+            : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 
+            : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
+            : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
+        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 
+            : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
+        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 
+            : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
+        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 
+            : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
+        ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 
+            : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
+        ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
+            : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2"
+        #Renaming files doesn't work
+        #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
+        #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
+    }
+
     def setUp(self):
         self.d = bb.data.init()
         self.tempdir = tempfile.mkdtemp()
@@ -64,7 +90,7 @@ class FetcherTest(unittest.TestCase):
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
-    def test_gitfetch(self):
+    def gitfetcher(self, url1, url2):
         def checkrevision(self, fetcher):
             fetcher.unpack(self.unpackdir)
             revision = subprocess.check_output("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git").strip()
@@ -72,17 +98,40 @@ class FetcherTest(unittest.TestCase):
 
         self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
         self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
-        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake"], self.d)
+        fetcher = bb.fetch.Fetch([url1], self.d)
         fetcher.download()
         checkrevision(self, fetcher)
         # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
         bb.utils.prunedir(self.dldir + "/git2/")
         bb.utils.prunedir(self.unpackdir)
         self.d.setVar("BB_NO_NETWORK", "1")
-        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake"], self.d)
+        fetcher = bb.fetch.Fetch([url2], self.d)
         fetcher.download()
         checkrevision(self, fetcher)
 
+    def test_gitfetch(self):
+        url1 = url2 = "git://git.openembedded.org/bitbake"
+        self.gitfetcher(url1, url2)
+
+    def test_gitfetch_premirror(self):
+        url1 = "git://git.openembedded.org/bitbake"
+        url2 = "git://someserver.org/bitbake"
+        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
+        self.gitfetcher(url1, url2)
+
+    def test_gitfetch_premirror2(self):
+        url1 = url2 = "git://someserver.org/bitbake"
+        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
+        self.gitfetcher(url1, url2)
+
+    def test_urireplace(self):
+        for k, v in self.replaceuris.items():
+            ud = bb.fetch.FetchData(k[0], self.d)
+            ud.setup_localpath(self.d)
+            newuris = bb.fetch2.uri_replace(ud, k[1], k[2], self.d)
+            self.assertEqual(newuris, v)
+
+
 class URLHandle(unittest.TestCase):
 
     datatable = {
Signature · 1 line
1.7.5.4