fetcher overhaul - api reorg and git optimization V2

index c621457..e85f1da 100644
@@ -22,6 +22,7 @@ BitBake 'Fetch' git implementation
 
 import os
 import bb
+import bb.persist_data
 from   bb    import data
 from   bb.fetch2 import Fetch
 from   bb.fetch2 import runfetchcmd

@@ -116,6 +117,7 @@ class Git(Fetch):
 
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
+
         coname = '%s' % (ud.tag)
         codir = os.path.join(ud.clonedir, coname)
 

@@ -205,11 +207,19 @@ class Git(Fetch):
         output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
         return output.split()[0] != "0"
 
-    def _revision_key(self, url, ud, d):
+    def _revision_key(self, url, ud, d, branch=False):
         """
         Return a unique key for the url
         """
-        return "git:" + ud.host + ud.path.replace('/', '.') + ud.branch
+        key = 'git:' + ud.host + ud.path.replace('/', '.')
+        if branch:
+            return key + ud.branch
+        else:
+            return key
+
+    def generate_revision_key(self, url, ud, d, branch=False):
+        key = self._revision_key(url, ud, d, branch)
+        return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
 
     def _latest_revision(self, url, ud, d):
         """

@@ -227,6 +237,74 @@ class Git(Fetch):
             raise bb.fetch2.FetchError("Fetch command %s gave empty output\n" % (cmd))
         return output.split()[0]
 
+    def latest_revision(self, url, ud, d):
+        """
+        Look in the cache for the latest revision, if not present ask the SCM.
+        """
+        persisted = bb.persist_data.persist(d)
+        revs = persisted['BB_URI_HEADREVS']
+
+        key = self.generate_revision_key(url, ud, d, branch=True)
+        rev = revs[key]
+        if rev is None:
+            # Compatibility with old key format, no branch included
+            oldkey = self.generate_revision_key(url, ud, d, branch=False)
+            rev = revs[oldkey]
+            if rev is not None:
+                del revs[oldkey]
+            else:
+                rev = self._latest_revision(url, ud, d)
+            revs[key] = rev
+
+        return str(rev)
+
+    def sortable_revision(self, url, ud, d):
+        """
+
+        """
+        pd = bb.persist_data.persist(d)
+        localcounts = pd['BB_URI_LOCALCOUNT']
+        key = self.generate_revision_key(url, ud, d, branch=True)
+        oldkey = self.generate_revision_key(url, ud, d, branch=False)
+
+        latest_rev = self._build_revision(url, ud, d)
+        last_rev = localcounts[key + '_rev']
+        if last_rev is None:
+            last_rev = localcounts[oldkey + '_rev']
+            if last_rev is not None:
+                del localcounts[oldkey + '_rev']
+                localcounts[key + '_rev'] = last_rev
+
+        uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
+        count = None
+        if uselocalcount:
+            count = Fetch.localcount_internal_helper(ud, d)
+        if count is None:
+            count = localcounts[key + '_count']
+        if count is None:
+            count = localcounts[oldkey + '_count']
+            if count is not None:
+                del localcounts[oldkey + '_count']
+                localcounts[key + '_count'] = count
+
+        if last_rev == latest_rev:
+            return str(count + "+" + latest_rev)
+
+        buildindex_provided = hasattr(self, "_sortable_buildindex")
+        if buildindex_provided:
+            count = self._sortable_buildindex(url, ud, d, latest_rev)
+        if count is None:
+            count = "0"
+        elif uselocalcount or buildindex_provided:
+            count = str(count)
+        else:
+            count = str(int(count) + 1)
+
+        localcounts[key + '_rev'] = latest_rev
+        localcounts[key + '_count'] = count
+
+        return str(count + "+" + latest_rev)
+
     def _build_revision(self, url, ud, d):
         return ud.tag
 

index 751e514..4b4ac47 100644
@@ -259,6 +259,13 @@ def verify_checksum(u, ud, d):
                      ud.sha256_expected, sha256data)
         raise FetchError("%s checksum mismatch." % u)
 
+def subprocess_setup():
+    import signal
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    # SIGPIPE errors are known issues with gzip/bash
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
 def go(d, urls = None):
     """
     Fetch all urls

@@ -629,6 +636,83 @@ class Fetch(object):
         """
         raise NoMethodError("Missing implementation for url")
 
+    def unpack(file, data, url = None):
+        import subprocess
+        if not url:
+            url = "file://%s" % file
+        dots = file.split(".")
+        if dots[-1] in ['gz', 'bz2', 'Z']:
+            efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
+        else:
+            efile = file
+        cmd = None
+        if file.endswith('.tar'):
+            cmd = 'tar x --no-same-owner -f %s' % file
+        elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+            cmd = 'tar xz --no-same-owner -f %s' % file
+        elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
+            cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
+        elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
+            cmd = 'gzip -dc %s > %s' % (file, efile)
+        elif file.endswith('.bz2'):
+            cmd = 'bzip2 -dc %s > %s' % (file, efile)
+        elif file.endswith('.tar.xz'):
+            cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
+        elif file.endswith('.xz'):
+            cmd = 'xz -dc %s > %s' % (file, efile)
+        elif file.endswith('.zip') or file.endswith('.jar'):
+            cmd = 'unzip -q -o'
+            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+            if 'dos' in parm:
+                cmd = '%s -a' % cmd
+            cmd = "%s '%s'" % (cmd, file)
+        elif os.path.isdir(file):
+            filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
+            destdir = "."
+            if file[0:len(filesdir)] == filesdir:
+                destdir = file[len(filesdir):file.rfind('/')]
+                destdir = destdir.strip('/')
+                if len(destdir) < 1:
+                    destdir = "."
+                elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
+                    os.makedirs("%s/%s" % (os.getcwd(), destdir))
+            cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+        else:
+            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+            if not 'patch' in parm:
+                # The "destdir" handling was specifically done for FILESPATH
+                # items.  So, only do so for file:// entries.
+                if type == "file" and path.find("/") != -1:
+                    destdir = path.rsplit("/", 1)[0]
+                else:
+                    destdir = "."
+                bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
+                cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+
+        if not cmd:
+            return True
+
+        dest = os.path.join(os.getcwd(), os.path.basename(file))
+        if os.path.exists(dest):
+            if os.path.samefile(file, dest):
+                return True
+
+        # Change to subdir before executing command
+        save_cwd = os.getcwd();
+        parm = bb.decodeurl(url)[5]
+        if 'subdir' in parm:
+            newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+            bb.mkdirhier(newdir)
+            os.chdir(newdir)
+
+        cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
+        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+
+        os.chdir(save_cwd)
+
+        return ret == 0
+
     def try_premirror(self, url, urldata, d):
         """
         Should premirrors be used?

index 4b4ac47..c5b5272 100644
@@ -636,10 +636,9 @@ class Fetch(object):
         """
         raise NoMethodError("Missing implementation for url")
 
-    def unpack(file, data, url = None):
+    def unpack(self, urldata, rootdir, data):
         import subprocess
-        if not url:
-            url = "file://%s" % file
+        file = urldata.localpath
         dots = file.split(".")
         if dots[-1] in ['gz', 'bz2', 'Z']:
             efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))

@@ -662,8 +661,7 @@ class Fetch(object):
             cmd = 'xz -dc %s > %s' % (file, efile)
         elif file.endswith('.zip') or file.endswith('.jar'):
             cmd = 'unzip -q -o'
-            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-            if 'dos' in parm:
+            if 'dos' in urldata.parm:
                 cmd = '%s -a' % cmd
             cmd = "%s '%s'" % (cmd, file)
         elif os.path.isdir(file):

@@ -674,34 +672,33 @@ class Fetch(object):
                 destdir = destdir.strip('/')
                 if len(destdir) < 1:
                     destdir = "."
-                elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
-                    os.makedirs("%s/%s" % (os.getcwd(), destdir))
-            cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+                elif not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
+                    os.makedirs("%s/%s" % (rootdir, destdir))
+            cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir)
         else:
-            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-            if not 'patch' in parm:
+            if not 'patch' in urldata.parm:
                 # The "destdir" handling was specifically done for FILESPATH
                 # items.  So, only do so for file:// entries.
-                if type == "file" and path.find("/") != -1:
-                    destdir = path.rsplit("/", 1)[0]
+                if urldata.type == "file" and urldata.path.find("/") != -1:
+                    destdir = urldata.path.rsplit("/", 1)[0]
                 else:
                     destdir = "."
-                bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
-                cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+                bb.mkdirhier("%s/%s" % (rootdir, destdir))
+                cmd = 'cp %s %s/%s/' % (file, rootdir, destdir)
 
         if not cmd:
             return True
 
-        dest = os.path.join(os.getcwd(), os.path.basename(file))
+        dest = os.path.join(rootdir, os.path.basename(file))
         if os.path.exists(dest):
             if os.path.samefile(file, dest):
                 return True
 
         # Change to subdir before executing command
         save_cwd = os.getcwd();
-        parm = bb.decodeurl(url)[5]
-        if 'subdir' in parm:
-            newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+        os.chdir(rootdir)
+        if 'subdir' in urldata.parm:
+            newdir = ("%s/%s" % (rootdir, urldata.parm['subdir']))
             bb.mkdirhier(newdir)
             os.chdir(newdir)
 

index 636178c..ca48e57 100644
@@ -268,7 +268,7 @@ def subprocess_setup():
     # SIGPIPE errors are known issues with gzip/bash
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
-def go(d, urls = None):
+def download(d, urls = None):
     """
     Fetch all urls
     init must have previously been called

@@ -298,7 +298,7 @@ def go(d, urls = None):
         if m.forcefetch(u, ud, d) or not localpath:
             # Next try fetching from the original uri, u
             try:
-                m.go(u, ud, d)
+                m.download(u, ud, d)
                 localpath = ud.localpath
             except FetchError:
                 # Remove any incomplete file

@@ -504,7 +504,7 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
                     if found:
                         return found
                 else:
-                    ud.method.go(newuri, ud, ld)
+                    ud.method.download(newuri, ud, ld)
                     return ud.localpath
             except (bb.fetch2.MissingParameterError,
                     bb.fetch2.FetchError,

@@ -631,7 +631,7 @@ class Fetch(object):
         """
         return False
 
-    def go(self, url, urldata, d):
+    def download(self, url, urldata, d):
         """
         Fetch urls
         Assumes localpath was called first

index 97b042b..608ecc7 100644
@@ -79,7 +79,7 @@ class Bzr(Fetch):
 
         return bzrcmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):

index 1570cab..8e72090 100644
@@ -72,7 +72,7 @@ class Cvs(Fetch):
             return True
         return False
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
 
         method = ud.parm.get('method', 'pserver')
         localdir = ud.parm.get('localdir', ud.module)

index e85f1da..c40a19c 100644
@@ -107,7 +107,7 @@ class Git(Fetch):
 
         return True
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         if ud.user:

@@ -320,7 +320,7 @@ class Git(Fetch):
 
         if not os.path.exists(ud.clonedir):
             print("no repo")
-            self.go(None, ud, d)
+            self.download(None, ud, d)
             if not os.path.exists(ud.clonedir):
                 logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
                 return None

@@ -328,7 +328,7 @@ class Git(Fetch):
 
         os.chdir(ud.clonedir)
         if not self._contains_ref(rev, d):
-            self.go(None, ud, d)
+            self.download(None, ud, d)
 
         output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
         os.chdir(cwd)

index 0ba8433..635ecbf 100644
@@ -112,7 +112,7 @@ class Hg(Fetch):
 
         return cmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")

index bcb30df..89fbdf6 100644
@@ -56,7 +56,7 @@ class Local(Fetch):
         # We don't set localfile as for this fetcher the file is already local!
         return newpath
 
-    def go(self, url, urldata, d):
+    def download(self, url, urldata, d):
         """Fetch urls (no-op for Local method)"""
         # no need to fetch local files, we'll deal with them in place.
         return 1

index 06ac5a9..619e2f1 100644
@@ -79,7 +79,7 @@ class Osc(Fetch):
 
         return osccmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """
         Fetch url
         """

index 18b2781..bda0bb8 100644
@@ -121,7 +121,7 @@ class Perforce(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """
         Fetch urls
         """

index 3330957..510ba46 100644
@@ -55,7 +55,7 @@ class Repo(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):

index 8b28322..78f55a6 100644
@@ -74,7 +74,7 @@ class SSH(Fetch):
         lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
         return lpath
 
-    def go(self, url, urldata, d):
+    def download(self, url, urldata, d):
         dldir = data.getVar('DL_DIR', d, 1)
 
         m = __pattern__.match(url)

index 7990ff2..3bb4c38 100644
@@ -57,7 +57,7 @@ class Svk(Fetch):
     def forcefetch(self, url, ud, d):
         return ud.date == "now"
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch urls"""
 
         svkroot = ud.host + ud.path

index 1116795..547c04f 100644
@@ -128,7 +128,7 @@ class Svn(Fetch):
 
         return svncmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")

index cf36cca..91cfafb 100644
@@ -48,7 +48,7 @@ class Wget(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
 
-    def go(self, uri, ud, d, checkonly = False):
+    def download(self, uri, ud, d, checkonly = False):
         """Fetch urls"""
 
         def fetch_uri(uri, ud, d):

@@ -90,4 +90,4 @@ class Wget(Fetch):
 
 
     def checkstatus(self, uri, ud, d):
-        return self.go(uri, ud, d, True)
+        return self.download(uri, ud, d, True)

index f1ffb45..23b17f2 100644
@@ -256,7 +256,13 @@ python base_do_unpack() {
 			continue
 		local = os.path.realpath(local)
 		lf = bb.utils.lockfile(urldata[url].lockfile)
-		ret = oe_unpack_file(local, localdata, url)
+		if bb.fetch.__version__ == "1":
+			ret = oe_unpack_file(local, localdata, url)
+		else:
+			# use bb.fetch2 unpack API
+			ud = urldata[url]
+			rootdir = bb.data.getVar('WORKDIR', localdata, True)
+			ret = ud.method.unpack(ud, rootdir, localdata)
 		bb.utils.unlockfile(lf)
 		if not ret:
 			raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)

index ca48e57..466b62e 100644
@@ -299,6 +299,8 @@ def download(d, urls = None):
             # Next try fetching from the original uri, u
             try:
                 m.download(u, ud, d)
+                if hasattr(m, "build_mirror_data"):
+                    m.build_mirror_data(u, ud, d)
                 localpath = ud.localpath
             except FetchError:
                 # Remove any incomplete file

@@ -505,6 +507,8 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
                         return found
                 else:
                     ud.method.download(newuri, ud, ld)
+                    if hasattr(ud.method,"build_mirror_data"):
+                        ud.method.build_mirror_data(newuri, ud, ld)
                     return ud.localpath
             except (bb.fetch2.MissingParameterError,
                     bb.fetch2.FetchError,

index c40a19c..690bb08 100644
@@ -118,9 +118,6 @@ class Git(Fetch):
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
 
-        coname = '%s' % (ud.tag)
-        codir = os.path.join(ud.clonedir, coname)
-
         # If we have no existing clone and no mirror tarball, try and obtain one
         if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
             try:

@@ -151,7 +148,12 @@ class Git(Fetch):
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
 
+    def build_mirror_data(self, url, ud, d):
         # Generate a mirror tarball if needed
+        coname = '%s' % (ud.tag)
+        codir = os.path.join(ud.clonedir, coname)
+        repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
         os.chdir(ud.clonedir)
         mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
         if mirror_tarballs != "0" or 'fullclone' in ud.parm:

index 438756a..ca9d925 100644
@@ -193,6 +193,22 @@ class Git(Fetch):
         os.chdir(ud.clonedir)
         bb.utils.prunedir(codir)
 
+    def unpack(self, ud, destdir, d):
+        """ unpack the downloaded src to destdir"""
+        subdir = ud.parm.get("subpath", "")
+        if subdir != "":
+            readpathspec = ":%s" % (subdir)
+        else:
+            readpathspec = ""
+
+		if 'fullclone' in ud.parm:
+			runfetchcmd("cp -pPR %s %s" % (os.path.join(ud.clonedir, ".git"), destdir), d)
+		else:
+			os.chdir(ud.clonedir)
+	        runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+		    runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, os.path.join(destdir,"git/")), d)
+        return True
+
     def supports_srcrev(self):
         return True
 

index 07eb77d..49fe502 100644
@@ -32,6 +32,8 @@ import bb
 from   bb import data
 from   bb import persist_data
 
+__version__ = "1"
+
 logger = logging.getLogger("BitBake.Fetch")
 
 class MalformedUrl(Exception):

index c5b5272..636178c 100644
@@ -32,6 +32,8 @@ import bb
 from   bb import data
 from   bb import persist_data
 
+__version__ = "2"
+
 logger = logging.getLogger("BitBake.Fetch")
 
 class MalformedUrl(Exception):

index 690bb08..438756a 100644
@@ -117,14 +117,6 @@ class Git(Fetch):
 
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
-
-        # If we have no existing clone and no mirror tarball, try and obtain one
-        if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
-            try:
-                Fetch.try_mirrors(ud.mirrortarball)
-            except:
-                pass
-
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
             bb.mkdirhier(ud.clonedir)

index ca9d925..acb01c6 100644
@@ -61,24 +61,14 @@ class Git(Fetch):
         ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
 
         ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+        ud.repochanged = False
 
     def localpath(self, url, ud, d):
         ud.tag = ud.revision
         if not ud.tag or ud.tag == "master":
             ud.tag = self.latest_revision(url, ud, d)
 
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdir = subdir[:-1]
-            subdirpath = os.path.join(ud.path, subdir);
-        else:
-            subdirpath = ud.path;
-
-        if 'fullclone' in ud.parm:
-            ud.localfile = ud.mirrortarball
-        else:
-            ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
+        ud.localfile = ud.mirrortarball
 
         if 'noclone' in ud.parm:
             ud.localfile = None

@@ -91,8 +81,6 @@ class Git(Fetch):
             return True
         if 'noclone' in ud.parm:
             return False
-        if os.path.exists(ud.localpath):
-            return False
         if not self._contains_ref(ud.tag, d):
             return True
         return False

@@ -139,60 +127,18 @@ class Git(Fetch):
             runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+            ud.repochanged = True
 
     def build_mirror_data(self, url, ud, d):
         # Generate a mirror tarball if needed
-        coname = '%s' % (ud.tag)
-        codir = os.path.join(ud.clonedir, coname)
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
         os.chdir(ud.clonedir)
         mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
-        if mirror_tarballs != "0" or 'fullclone' in ud.parm:
+        if mirror_tarballs != "0" or 'fullclone' in ud.parm or ud.repochanged:
             logger.info("Creating tarball of git repository")
             runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
 
-        if 'fullclone' in ud.parm:
-            return
-
-        if os.path.exists(codir):
-            bb.utils.prunedir(codir)
-
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdirbase = os.path.basename(subdir[:-1])
-            else:
-                subdirbase = os.path.basename(subdir)
-        else:
-            subdirbase = ""
-
-        if subdir != "":
-            readpathspec = ":%s" % (subdir)
-            codir = os.path.join(codir, "git")
-            coprefix = os.path.join(codir, subdirbase, "")
-        else:
-            readpathspec = ""
-            coprefix = os.path.join(codir, "git", "")
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
-            os.chdir(coprefix)
-            runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-        else:
-            bb.mkdirhier(codir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-            runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
-
-        os.chdir(codir)
-        logger.info("Creating tarball of git checkout")
-        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
-
-        os.chdir(ud.clonedir)
-        bb.utils.prunedir(codir)
-
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
         subdir = ud.parm.get("subpath", "")

index acb01c6..8f574b7 100644
@@ -113,12 +113,14 @@ class Git(Fetch):
 
         # If the repo still doesn't exist, fallback to cloning it
         if not os.path.exists(ud.clonedir):
+            bb.fetch2.check_network_access(d, "git clone %s%s" % (ud.host, ud.path))
             runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
 
         os.chdir(ud.clonedir)
         # Update the checkout if needed
         if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm:
             # Remove all but the .git directory
+            bb.fetch2.check_network_access(d, "git fetch %s%s" %(ud.host, ud.path))
             runfetchcmd("rm * -Rf", d)
             if 'fullclone' in ud.parm:
                 runfetchcmd("%s fetch --all" % (ud.basecmd), d)

@@ -186,6 +188,7 @@ class Git(Fetch):
         else:
             username = ""
 
+        bb.fetch2.check_network_access(d, "git ls-remote %s%s %s" % (ud.host, ud.path, ud.branch))
         basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
         cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch)
         output = runfetchcmd(cmd, d, True)

index 91cfafb..88cb8c7 100644
@@ -69,6 +69,7 @@ class Wget(Fetch):
             fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
             logger.info("fetch " + uri)
             logger.debug(2, "executing " + fetchcmd)
+            bb.fetch2.check_network_access(d, fetchcmd)
             runfetchcmd(fetchcmd, d)
 
             # Sanity check since wget can pretend it succeed when it didn't

index 547c04f..8d76802 100644
@@ -139,6 +139,7 @@ class Svn(Fetch):
             # update sources there
             os.chdir(ud.moddir)
             logger.debug(1, "Running %s", svnupdatecmd)
+            bb.fetch2.check_network_access(d, svnupdatecmd)
             runfetchcmd(svnupdatecmd, d)
         else:
             svnfetchcmd = self._buildsvncommand(ud, d, "fetch")

@@ -147,6 +148,7 @@ class Svn(Fetch):
             bb.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", svnfetchcmd)
+            bb.fetch2.check_network_access(d, svnfetchcmd)
             runfetchcmd(svnfetchcmd, d)
 
         scmdata = ud.parm.get("scmdata", "")

@@ -180,7 +182,7 @@ class Svn(Fetch):
         """
         Return the latest upstream revision number
         """
-        logger.debug(2, "SVN fetcher hitting network for %s", url)
+        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "info"))
 
         output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True)
 

index 8e72090..4915e74 100644
@@ -131,6 +131,7 @@ class Cvs(Fetch):
         moddir = os.path.join(pkgdir, localdir)
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
             logger.info("Update " + loc)
+            bb.fetch2.check_network_access(d, cvsupdatecmd)
             # update sources there
             os.chdir(moddir)
             myret = os.system(cvsupdatecmd)

@@ -140,6 +141,7 @@ class Cvs(Fetch):
             bb.mkdirhier(pkgdir)
             os.chdir(pkgdir)
             logger.debug(1, "Running %s", cvscmd)
+            bb.fetch2.check_network_access(d, cvscmd)
             myret = os.system(cvscmd)
 
         if myret != 0 or not os.access(moddir, os.R_OK):

index 23b17f2..d1d60f7 100644
@@ -135,7 +135,10 @@ python base_do_fetch() {
 		raise bb.build.FuncFailed("Malformed URL: %s" % value)
 
 	try:
-		bb.fetch.go(localdata)
+		if bb.fetch.__version__ == "1":
+			bb.fetch.go(localdata)
+		else:
+			bb.fetch.download(localdata)
 	except bb.fetch.MissingParameterError:
 		(type, value, traceback) = sys.exc_info()
 		raise bb.build.FuncFailed("Missing parameters: %s" % value)

index b6e6c92..fa184bc 100644
@@ -319,7 +319,10 @@ def pstaging_fetch(sstatepkg, d):
         # we will build the package
         try:
             bb.fetch.init([srcuri], localdata)
-            bb.fetch.go(localdata, [srcuri])
+            if bb.fetch.__version__ == "1":
+                bb.fetch.go(localdata, [srcuri])
+            else:
+                bb.fetch.download(localdata, [srcuri])
             # Need to optimise this, if using file:// urls, the fetcher just changes the local path
             # For now work around by symlinking
             localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)

index 466b62e..ad94231 100644
@@ -474,6 +474,16 @@ def runfetchcmd(cmd, d, quiet = False):
 
     return output
 
+def check_network_access(d, info = ""):
+    """
+    log remote network access, and error if BB_NO_NETWORK is set
+    """
+    if bb.data.getVar("BB_NO_NETWORK", d, True) == "1":
+        bb.error("BB_NO_NETWORK is set, but fetcher access netowrk with [%s] " % info)
+        raise FetchError("BB_NO_NETWORK violation")
+    else:
+        bb.note("Fetcher access netowrk with [%s]" % info)
+
 def try_mirrors(d, uri, mirrors, check = False, force = False):
     """
     Try to use a mirrored version of the sources.

index 635ecbf..4ba28c7 100644
@@ -123,6 +123,7 @@ class Hg(Fetch):
             # update sources there
             os.chdir(ud.moddir)
             logger.debug(1, "Running %s", updatecmd)
+            bb.fetch2.check_network_access(d, updatecmd)
             runfetchcmd(updatecmd, d)
 
         else:

@@ -132,6 +133,7 @@ class Hg(Fetch):
             bb.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", fetchcmd)
+            bb.fetch2.check_network_access(d, fetchcmd)
             runfetchcmd(fetchcmd, d)
 
         # Even when we clone (fetch), we still need to update as hg's clone

@@ -165,6 +167,7 @@ class Hg(Fetch):
         """
         Compute tip revision for the url
         """
+        bb.fetch2.check_network_access(d, self._buildhgcommand(ud, d, "info"))
         output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
         return output.strip()
 

index 608ecc7..6e1970b 100644
@@ -85,11 +85,13 @@ class Bzr(Fetch):
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
             bzrcmd = self._buildbzrcommand(ud, d, "update")
             logger.debug(1, "BZR Update %s", loc)
+            bb.fetch2.check_network_access(d, bzrcmd)
             os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
             runfetchcmd(bzrcmd, d)
         else:
             bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
             bzrcmd = self._buildbzrcommand(ud, d, "fetch")
+            bb.fetch2.check_network_access(d, bzrcmd)
             logger.debug(1, "BZR Checkout %s", loc)
             bb.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)

@@ -130,6 +132,8 @@ class Bzr(Fetch):
         """
         logger.debug(2, "BZR fetcher hitting network for %s", url)
 
+        bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"))
+
         output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True)
 
         return output.strip()

index 619e2f1..25dcb7b 100644
@@ -92,6 +92,7 @@ class Osc(Fetch):
             # update sources there
             os.chdir(ud.moddir)
             logger.debug(1, "Running %s", oscupdatecmd)
+            bb.fetch2.check_network_access(d, oscupdatecmd)
             runfetchcmd(oscupdatecmd, d)
         else:
             oscfetchcmd = self._buildosccommand(ud, d, "fetch")

@@ -100,6 +101,7 @@ class Osc(Fetch):
             bb.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", oscfetchcmd)
+            bb.fetch2.check_network_access(d, oscfetchcmd)
             runfetchcmd(oscfetchcmd, d)
 
         os.chdir(os.path.join(ud.pkgdir + ud.path))

index 78f55a6..711e668 100644
@@ -112,6 +112,8 @@ class SSH(Fetch):
             commands.mkarg(ldir)
         )
 
+        bb.fetch2.check_network_access(d, cmd)
+
         (exitstatus, output) = commands.getstatusoutput(cmd)
         if exitstatus != 0:
             print(output)

index 510ba46..c80fe5d 100644
@@ -74,8 +74,10 @@ class Repo(Fetch):
         bb.mkdirhier(os.path.join(codir, "repo"))
         os.chdir(os.path.join(codir, "repo"))
         if not os.path.exists(os.path.join(codir, "repo", ".repo")):
+            bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path))
             runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
 
+        bb.fetch2.check_network_access(d, "repo sync %s" % ud.url)
         runfetchcmd("repo sync", d)
         os.chdir(codir)