fetch overhaul - API reorg and git optimization

index c621457..e85f1da 100644
@@ -22,6 +22,7 @@ BitBake 'Fetch' git implementation
 
 import os
 import bb
+import bb.persist_data
 from   bb    import data
 from   bb.fetch2 import Fetch
 from   bb.fetch2 import runfetchcmd

@@ -116,6 +117,7 @@ class Git(Fetch):
 
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
+
         coname = '%s' % (ud.tag)
         codir = os.path.join(ud.clonedir, coname)
 

@@ -205,11 +207,19 @@ class Git(Fetch):
         output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
         return output.split()[0] != "0"
 
-    def _revision_key(self, url, ud, d):
+    def _revision_key(self, url, ud, d, branch=False):
         """
         Return a unique key for the url
         """
-        return "git:" + ud.host + ud.path.replace('/', '.') + ud.branch
+        key = 'git:' + ud.host + ud.path.replace('/', '.')
+        if branch:
+            return key + ud.branch
+        else:
+            return key
+
+    def generate_revision_key(self, url, ud, d, branch=False):
+        key = self._revision_key(url, ud, d, branch)
+        return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
 
     def _latest_revision(self, url, ud, d):
         """

@@ -227,6 +237,74 @@ class Git(Fetch):
             raise bb.fetch2.FetchError("Fetch command %s gave empty output\n" % (cmd))
         return output.split()[0]
 
+    def latest_revision(self, url, ud, d):
+        """
+        Look in the cache for the latest revision, if not present ask the SCM.
+        """
+        persisted = bb.persist_data.persist(d)
+        revs = persisted['BB_URI_HEADREVS']
+
+        key = self.generate_revision_key(url, ud, d, branch=True)
+        rev = revs[key]
+        if rev is None:
+            # Compatibility with old key format, no branch included
+            oldkey = self.generate_revision_key(url, ud, d, branch=False)
+            rev = revs[oldkey]
+            if rev is not None:
+                del revs[oldkey]
+            else:
+                rev = self._latest_revision(url, ud, d)
+            revs[key] = rev
+
+        return str(rev)
+
+    def sortable_revision(self, url, ud, d):
+        """
+
+        """
+        pd = bb.persist_data.persist(d)
+        localcounts = pd['BB_URI_LOCALCOUNT']
+        key = self.generate_revision_key(url, ud, d, branch=True)
+        oldkey = self.generate_revision_key(url, ud, d, branch=False)
+
+        latest_rev = self._build_revision(url, ud, d)
+        last_rev = localcounts[key + '_rev']
+        if last_rev is None:
+            last_rev = localcounts[oldkey + '_rev']
+            if last_rev is not None:
+                del localcounts[oldkey + '_rev']
+                localcounts[key + '_rev'] = last_rev
+
+        uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
+        count = None
+        if uselocalcount:
+            count = Fetch.localcount_internal_helper(ud, d)
+        if count is None:
+            count = localcounts[key + '_count']
+        if count is None:
+            count = localcounts[oldkey + '_count']
+            if count is not None:
+                del localcounts[oldkey + '_count']
+                localcounts[key + '_count'] = count
+
+        if last_rev == latest_rev:
+            return str(count + "+" + latest_rev)
+
+        buildindex_provided = hasattr(self, "_sortable_buildindex")
+        if buildindex_provided:
+            count = self._sortable_buildindex(url, ud, d, latest_rev)
+        if count is None:
+            count = "0"
+        elif uselocalcount or buildindex_provided:
+            count = str(count)
+        else:
+            count = str(int(count) + 1)
+
+        localcounts[key + '_rev'] = latest_rev
+        localcounts[key + '_count'] = count
+
+        return str(count + "+" + latest_rev)
+
     def _build_revision(self, url, ud, d):
         return ud.tag
 

index 751e514..4b4ac47 100644
@@ -259,6 +259,13 @@ def verify_checksum(u, ud, d):
                      ud.sha256_expected, sha256data)
         raise FetchError("%s checksum mismatch." % u)
 
+def subprocess_setup():
+    import signal
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    # SIGPIPE errors are known issues with gzip/bash
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
 def go(d, urls = None):
     """
     Fetch all urls

@@ -629,6 +636,83 @@ class Fetch(object):
         """
         raise NoMethodError("Missing implementation for url")
 
+    def unpack(file, data, url = None):
+        import subprocess
+        if not url:
+            url = "file://%s" % file
+        dots = file.split(".")
+        if dots[-1] in ['gz', 'bz2', 'Z']:
+            efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
+        else:
+            efile = file
+        cmd = None
+        if file.endswith('.tar'):
+            cmd = 'tar x --no-same-owner -f %s' % file
+        elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+            cmd = 'tar xz --no-same-owner -f %s' % file
+        elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
+            cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
+        elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
+            cmd = 'gzip -dc %s > %s' % (file, efile)
+        elif file.endswith('.bz2'):
+            cmd = 'bzip2 -dc %s > %s' % (file, efile)
+        elif file.endswith('.tar.xz'):
+            cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
+        elif file.endswith('.xz'):
+            cmd = 'xz -dc %s > %s' % (file, efile)
+        elif file.endswith('.zip') or file.endswith('.jar'):
+            cmd = 'unzip -q -o'
+            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+            if 'dos' in parm:
+                cmd = '%s -a' % cmd
+            cmd = "%s '%s'" % (cmd, file)
+        elif os.path.isdir(file):
+            filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
+            destdir = "."
+            if file[0:len(filesdir)] == filesdir:
+                destdir = file[len(filesdir):file.rfind('/')]
+                destdir = destdir.strip('/')
+                if len(destdir) < 1:
+                    destdir = "."
+                elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
+                    os.makedirs("%s/%s" % (os.getcwd(), destdir))
+            cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+        else:
+            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
+            if not 'patch' in parm:
+                # The "destdir" handling was specifically done for FILESPATH
+                # items.  So, only do so for file:// entries.
+                if type == "file" and path.find("/") != -1:
+                    destdir = path.rsplit("/", 1)[0]
+                else:
+                    destdir = "."
+                bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
+                cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+
+        if not cmd:
+            return True
+
+        dest = os.path.join(os.getcwd(), os.path.basename(file))
+        if os.path.exists(dest):
+            if os.path.samefile(file, dest):
+                return True
+
+        # Change to subdir before executing command
+        save_cwd = os.getcwd();
+        parm = bb.decodeurl(url)[5]
+        if 'subdir' in parm:
+            newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+            bb.mkdirhier(newdir)
+            os.chdir(newdir)
+
+        cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
+        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+
+        os.chdir(save_cwd)
+
+        return ret == 0
+
     def try_premirror(self, url, urldata, d):
         """
         Should premirrors be used?

index 4b4ac47..7e0f420 100644
@@ -636,10 +636,9 @@ class Fetch(object):
         """
         raise NoMethodError("Missing implementation for url")
 
-    def unpack(file, data, url = None):
+    def unpack(self, urldata, rootdir, data):
         import subprocess
-        if not url:
-            url = "file://%s" % file
+        file = urldata.localpath
         dots = file.split(".")
         if dots[-1] in ['gz', 'bz2', 'Z']:
             efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))

@@ -662,8 +661,7 @@ class Fetch(object):
             cmd = 'xz -dc %s > %s' % (file, efile)
         elif file.endswith('.zip') or file.endswith('.jar'):
             cmd = 'unzip -q -o'
-            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-            if 'dos' in parm:
+            if 'dos' in urldata.parm:
                 cmd = '%s -a' % cmd
             cmd = "%s '%s'" % (cmd, file)
         elif os.path.isdir(file):

@@ -674,34 +672,33 @@ class Fetch(object):
                 destdir = destdir.strip('/')
                 if len(destdir) < 1:
                     destdir = "."
-                elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
-                    os.makedirs("%s/%s" % (os.getcwd(), destdir))
-            cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
+                elif not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
+                    os.makedirs("%s/%s" % (rootdir, destdir))
+            cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir)
         else:
-            (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-            if not 'patch' in parm:
+            if not 'patch' in urldata.parm:
                 # The "destdir" handling was specifically done for FILESPATH
                 # items.  So, only do so for file:// entries.
                 if type == "file" and path.find("/") != -1:
                     destdir = path.rsplit("/", 1)[0]
                 else:
                     destdir = "."
-                bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
-                cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
+                bb.mkdirhier("%s/%s" % (rootdir, destdir))
+                cmd = 'cp %s %s/%s/' % (file, rootdir, destdir)
 
         if not cmd:
             return True
 
-        dest = os.path.join(os.getcwd(), os.path.basename(file))
+        dest = os.path.join(rootdir, os.path.basename(file))
         if os.path.exists(dest):
             if os.path.samefile(file, dest):
                 return True
 
         # Change to subdir before executing command
         save_cwd = os.getcwd();
-        parm = bb.decodeurl(url)[5]
-        if 'subdir' in parm:
-            newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
+        os.chdir(rootdir)
+        if 'subdir' in urldata.parm:
+            newdir = ("%s/%s" % (rootdir, urldata.parm['subdir']))
             bb.mkdirhier(newdir)
             os.chdir(newdir)
 

index 07eb77d..49fe502 100644
@@ -32,6 +32,8 @@ import bb
 from   bb import data
 from   bb import persist_data
 
+__version__ = "1"
+
 logger = logging.getLogger("BitBake.Fetch")
 
 class MalformedUrl(Exception):

index 7e0f420..ef1a52a 100644
@@ -32,6 +32,8 @@ import bb
 from   bb import data
 from   bb import persist_data
 
+__version__ = "2"
+
 logger = logging.getLogger("BitBake.Fetch")
 
 class MalformedUrl(Exception):

index f1ffb45..23b17f2 100644
@@ -256,7 +256,13 @@ python base_do_unpack() {
 			continue
 		local = os.path.realpath(local)
 		lf = bb.utils.lockfile(urldata[url].lockfile)
-		ret = oe_unpack_file(local, localdata, url)
+		if bb.fetch.__version__ == "1":
+			ret = oe_unpack_file(local, localdata, url)
+		else:
+			# use bb.fetch2 unpack API
+			ud = urldata[url]
+			rootdir = bb.data.getVar('WORKDIR', localdata, True)
+			ret = ud.method.unpack(ud, rootdir, localdata)
 		bb.utils.unlockfile(lf)
 		if not ret:
 			raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)

index ef1a52a..8d0f907 100644
@@ -268,7 +268,7 @@ def subprocess_setup():
     # SIGPIPE errors are known issues with gzip/bash
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
-def go(d, urls = None):
+def download(d, urls = None):
     """
     Fetch all urls
     init must have previously been called

@@ -298,7 +298,7 @@ def go(d, urls = None):
         if m.forcefetch(u, ud, d) or not localpath:
             # Next try fetching from the original uri, u
             try:
-                m.go(u, ud, d)
+                m.download(u, ud, d)
                 localpath = ud.localpath
             except FetchError:
                 # Remove any incomplete file

@@ -504,7 +504,7 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
                     if found:
                         return found
                 else:
-                    ud.method.go(newuri, ud, ld)
+                    ud.method.download(newuri, ud, ld)
                     return ud.localpath
             except (bb.fetch2.MissingParameterError,
                     bb.fetch2.FetchError,

@@ -631,7 +631,7 @@ class Fetch(object):
         """
         return False
 
-    def go(self, url, urldata, d):
+    def download(self, url, urldata, d):
         """
         Fetch urls
         Assumes localpath was called first

index 97b042b..608ecc7 100644
@@ -79,7 +79,7 @@ class Bzr(Fetch):
 
         return bzrcmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):

index 1570cab..8e72090 100644
@@ -72,7 +72,7 @@ class Cvs(Fetch):
             return True
         return False
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
 
         method = ud.parm.get('method', 'pserver')
         localdir = ud.parm.get('localdir', ud.module)

index e85f1da..c40a19c 100644
@@ -107,7 +107,7 @@ class Git(Fetch):
 
         return True
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         if ud.user:

@@ -320,7 +320,7 @@ class Git(Fetch):
 
         if not os.path.exists(ud.clonedir):
             print("no repo")
-            self.go(None, ud, d)
+            self.download(None, ud, d)
             if not os.path.exists(ud.clonedir):
                 logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
                 return None

@@ -328,7 +328,7 @@ class Git(Fetch):
 
         os.chdir(ud.clonedir)
         if not self._contains_ref(rev, d):
-            self.go(None, ud, d)
+            self.download(None, ud, d)
 
         output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
         os.chdir(cwd)

index 0ba8433..635ecbf 100644
@@ -112,7 +112,7 @@ class Hg(Fetch):
 
         return cmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")

index bcb30df..89fbdf6 100644
@@ -56,7 +56,7 @@ class Local(Fetch):
         # We don't set localfile as for this fetcher the file is already local!
         return newpath
 
-    def go(self, url, urldata, d):
+    def download(self, url, urldata, d):
         """Fetch urls (no-op for Local method)"""
         # no need to fetch local files, we'll deal with them in place.
         return 1

index 06ac5a9..619e2f1 100644
@@ -79,7 +79,7 @@ class Osc(Fetch):
 
         return osccmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """
         Fetch url
         """

index 18b2781..bda0bb8 100644
@@ -121,7 +121,7 @@ class Perforce(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """
         Fetch urls
         """

index 3330957..510ba46 100644
@@ -55,7 +55,7 @@ class Repo(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):

index 8b28322..78f55a6 100644
@@ -74,7 +74,7 @@ class SSH(Fetch):
         lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
         return lpath
 
-    def go(self, url, urldata, d):
+    def download(self, url, urldata, d):
         dldir = data.getVar('DL_DIR', d, 1)
 
         m = __pattern__.match(url)

index 7990ff2..3bb4c38 100644
@@ -57,7 +57,7 @@ class Svk(Fetch):
     def forcefetch(self, url, ud, d):
         return ud.date == "now"
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch urls"""
 
         svkroot = ud.host + ud.path

index 1116795..547c04f 100644
@@ -128,7 +128,7 @@ class Svn(Fetch):
 
         return svncmd
 
-    def go(self, loc, ud, d):
+    def download(self, loc, ud, d):
         """Fetch url"""
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")

index cf36cca..91cfafb 100644
@@ -48,7 +48,7 @@ class Wget(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
 
-    def go(self, uri, ud, d, checkonly = False):
+    def download(self, uri, ud, d, checkonly = False):
         """Fetch urls"""
 
         def fetch_uri(uri, ud, d):

@@ -90,4 +90,4 @@ class Wget(Fetch):
 
 
     def checkstatus(self, uri, ud, d):
-        return self.go(uri, ud, d, True)
+        return self.download(uri, ud, d, True)

index 23b17f2..d1d60f7 100644
@@ -135,7 +135,10 @@ python base_do_fetch() {
 		raise bb.build.FuncFailed("Malformed URL: %s" % value)
 
 	try:
-		bb.fetch.go(localdata)
+		if bb.fetch.__version__ == "1":
+			bb.fetch.go(localdata)
+		else:
+			bb.fetch.download(localdata)
 	except bb.fetch.MissingParameterError:
 		(type, value, traceback) = sys.exc_info()
 		raise bb.build.FuncFailed("Missing parameters: %s" % value)

index b6e6c92..fa184bc 100644
@@ -319,7 +319,10 @@ def pstaging_fetch(sstatepkg, d):
         # we will build the package
         try:
             bb.fetch.init([srcuri], localdata)
-            bb.fetch.go(localdata, [srcuri])
+            if bb.fetch.__version__ == "1":
+                bb.fetch.go(localdata, [srcuri])
+            else:
+                bb.fetch.download(localdata, [srcuri])
             # Need to optimise this, if using file:// urls, the fetcher just changes the local path
             # For now work around by symlinking
             localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)

index 8d0f907..3ff5b7b 100644
@@ -299,6 +299,8 @@ def download(d, urls = None):
             # Next try fetching from the original uri, u
             try:
                 m.download(u, ud, d)
+                if hasattr(m, "build_mirror_data"):
+                    m.build_mirror_data(u, ud, d)
                 localpath = ud.localpath
             except FetchError:
                 # Remove any incomplete file

@@ -505,6 +507,8 @@ def try_mirrors(d, uri, mirrors, check = False, force = False):
                         return found
                 else:
                     ud.method.download(newuri, ud, ld)
+                    if hasattr(ud.method,"build_mirror_data"):
+                        ud.method.build_mirror_data(newuri, ud, ld)
                     return ud.localpath
             except (bb.fetch2.MissingParameterError,
                     bb.fetch2.FetchError,

index c40a19c..690bb08 100644
@@ -118,9 +118,6 @@ class Git(Fetch):
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
 
-        coname = '%s' % (ud.tag)
-        codir = os.path.join(ud.clonedir, coname)
-
         # If we have no existing clone and no mirror tarball, try and obtain one
         if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
             try:

@@ -151,7 +148,12 @@ class Git(Fetch):
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
 
+    def build_mirror_data(self, url, ud, d):
         # Generate a mirror tarball if needed
+        coname = '%s' % (ud.tag)
+        codir = os.path.join(ud.clonedir, coname)
+        repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
+
         os.chdir(ud.clonedir)
         mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
         if mirror_tarballs != "0" or 'fullclone' in ud.parm:

index 690bb08..438756a 100644
@@ -117,14 +117,6 @@ class Git(Fetch):
 
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
-
-        # If we have no existing clone and no mirror tarball, try and obtain one
-        if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
-            try:
-                Fetch.try_mirrors(ud.mirrortarball)
-            except:
-                pass
-
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
             bb.mkdirhier(ud.clonedir)

index 438756a..bb62a87 100644
@@ -193,6 +193,22 @@ class Git(Fetch):
         os.chdir(ud.clonedir)
         bb.utils.prunedir(codir)
 
+    def unpack(self, ud, destdir, d):
+        """ unpack the downloaded src to destdir"""
+        subdir = ud.parm.get("subpath", "")
+        if subdir != "":
+            readpathspec = ":%s" % (subdir)
+        else:
+            readpathspec = ""
+
+        # extract to ${WORKDIR}/git
+        destdir = destdir + "/git/"
+
+        os.chdir(ud.clonedir)
+        runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+        runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
+        return 1
+
     def supports_srcrev(self):
         return True
 

index bb62a87..7bfe178 100644
@@ -61,24 +61,14 @@ class Git(Fetch):
         ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
 
         ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+        ud.repochanged = False
 
     def localpath(self, url, ud, d):
         ud.tag = ud.revision
         if not ud.tag or ud.tag == "master":
             ud.tag = self.latest_revision(url, ud, d)
 
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdir = subdir[:-1]
-            subdirpath = os.path.join(ud.path, subdir);
-        else:
-            subdirpath = ud.path;
-
-        if 'fullclone' in ud.parm:
-            ud.localfile = ud.mirrortarball
-        else:
-            ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
+        ud.localfile = ud.mirrortarball
 
         if 'noclone' in ud.parm:
             ud.localfile = None

@@ -91,8 +81,6 @@ class Git(Fetch):
             return True
         if 'noclone' in ud.parm:
             return False
-        if os.path.exists(ud.localpath):
-            return False
         if not self._contains_ref(ud.tag, d):
             return True
         return False

@@ -139,60 +127,18 @@ class Git(Fetch):
             runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+            ud.repochanged = True
 
     def build_mirror_data(self, url, ud, d):
         # Generate a mirror tarball if needed
-        coname = '%s' % (ud.tag)
-        codir = os.path.join(ud.clonedir, coname)
         repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
 
         os.chdir(ud.clonedir)
         mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
-        if mirror_tarballs != "0" or 'fullclone' in ud.parm:
+        if mirror_tarballs != "0" or 'fullclone' in ud.parm or ud.repochanged:
             logger.info("Creating tarball of git repository")
             runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
 
-        if 'fullclone' in ud.parm:
-            return
-
-        if os.path.exists(codir):
-            bb.utils.prunedir(codir)
-
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdirbase = os.path.basename(subdir[:-1])
-            else:
-                subdirbase = os.path.basename(subdir)
-        else:
-            subdirbase = ""
-
-        if subdir != "":
-            readpathspec = ":%s" % (subdir)
-            codir = os.path.join(codir, "git")
-            coprefix = os.path.join(codir, subdirbase, "")
-        else:
-            readpathspec = ""
-            coprefix = os.path.join(codir, "git", "")
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
-            os.chdir(coprefix)
-            runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-        else:
-            bb.mkdirhier(codir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-            runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
-
-        os.chdir(codir)
-        logger.info("Creating tarball of git checkout")
-        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
-
-        os.chdir(ud.clonedir)
-        bb.utils.prunedir(codir)
-
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
         subdir = ud.parm.get("subpath", "")

index 08daa20..eadbf33 100644
@@ -194,6 +194,31 @@ class Git(Fetch):
         os.chdir(ud.clonedir)
         bb.utils.prunedir(codir)
 
+    def unpack(self, ud, destdir, d):
+        """ unpack the downloaded src to destdir"""
+
+        subdir = ud.parm.get("subpath", "")
+        if subdir != "":
+            readpathspec = ":%s" % (subdir)
+        else:
+            readpathspec = ""
+
+        destdir = os.path.join(destdir, "git/")
+        if os.path.exists(destdir):
+            bb.utils.prunedir(destdir)
+
+        if 'fullclone' in ud.parm:
+            runfetchcmd("git clone -l -n %s %s" % (ud.clonedir, destdir), d)
+            if os.path.exists("%s/.git/refs/remotes/origin" % ud.clonedir):
+                runfetchcmd("cp -af %s/.git/refs/remotes/origin/* %s/.git/refs/remotes/origin/" %(ud.clonedir, destdir), d)
+            if os.path.exists("%s/.git/packed-refs" % ud.clonedir):
+                runfetchcmd("cp -af %s/.git/packed-refs %s/.git/" %(ud.clonedir, destdir), d)
+        else:
+            os.chdir(ud.clonedir)
+            runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
+            runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, destdir), d)
+        return True
+
     def supports_srcrev(self):
         return True
 

index 8479b39..ca1fb15 100644
@@ -89,11 +89,11 @@ do_patch() {
 }
 
 do_kernel_checkout() {
-       if [ -d ${WORKDIR}/.git/refs/remotes/origin ]; then
+       if [ -d ${WORKDIR}/git/.git/refs/remotes/origin ]; then
                echo "Fixing up git directory for ${LINUX_KERNEL_TYPE}/${KMACHINE}"
                rm -rf ${S}
                mkdir ${S}
-               mv ${WORKDIR}/.git ${S}
+               mv ${WORKDIR}/git/.git ${S}
 
                if [ -e ${S}/.git/packed-refs ]; then
                        cd ${S}