Implement git shallow mirror tarball support

index 374d846..e88736d 100644
@@ -66,8 +66,10 @@ Supported SRC_URI options are:
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import itertools
 import os
 import re
+import tempfile
 import bb
 from   bb    import data
 from   bb.fetch2 import FetchMethod

@@ -118,12 +120,24 @@ class Git(FetchMethod):
         branches = ud.parm.get("branch", "master").split(',')
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
+
+        shallow_default = d.getVar("BB_GIT_SHALLOW", True)
+        ud.shallows = {}
         ud.branches = {}
-        for name in ud.names:
-            branch = branches[ud.names.index(name)]
+        for pos, name in enumerate(ud.names):
+            branch = branches[pos]
             ud.branches[name] = branch
             ud.unresolvedrev[name] = branch
 
+            shallow = d.getVar("BB_GIT_SHALLOW_%s" % name, True) or shallow_default
+            if shallow == "0":
+                shallow = None
+            ud.shallows[name] = shallow
+
+        if not shallow_default and not filter(None, ud.shallows.itervalues()):
+            # No shallows
+            ud.shallows = None
+
         ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0"
 
         ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable

@@ -137,32 +151,44 @@ class Git(FetchMethod):
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':','.'), ud.path.replace('/', '.').replace('*', '.'))
+        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
         # for rebaseable git repo, it is necessary to keep mirror tar ball
         # per revision, so that even the revision disappears from the
         # upstream repo in the future, the mirror will remain intact and still
         # contains the revision
-        if ud.rebaseable:
+        if ud.shallows:
+            for name, shallow in ud.shallows.iteritems():
+                gitsrcname = '%s_%s_%s' % (gitsrcname, ud.branches[name].replace('/', '.'), ud.revisions[name])
+                if shallow:
+                    gitsrcname = gitsrcname + "_" + shallow
+        elif ud.rebaseable:
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
-        ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
-        gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
-        ud.clonedir = os.path.join(gitdir, gitsrcname)
 
+        dl_dir = d.getVar("DL_DIR", True)
+        gitdir = d.getVar("GITDIR", True) or (dl_dir + "/git2/")
+        ud.clonedir = os.path.join(gitdir, gitsrcname)
         ud.localfile = ud.clonedir
 
+        ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname
+        ud.fullmirror = os.path.join(dl_dir, ud.mirrortarball)
+
     def localpath(self, ud, d):
-        return ud.clonedir
+        if ud.shallows and os.path.exists(ud.fullmirror):
+            return ud.fullmirror
+        else:
+            return ud.clonedir
 
     def need_update(self, ud, d):
+        if ud.shallows and os.path.exists(ud.fullmirror):
+            return False
         if not os.path.exists(ud.clonedir):
             return True
         os.chdir(ud.clonedir)
         for name in ud.names:
             if not self._contains_ref(ud, d, name):
                 return True
-        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
+        if ud.write_tarballs and not os.path.exists(ud.fullmirror if not ud.shallows else ud.fullmirror):
             return True
         return False
 

@@ -182,9 +208,14 @@ class Git(FetchMethod):
 
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
-            bb.utils.mkdirhier(ud.clonedir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
+            if ud.shallows:
+                ud.localpath = ud.fullmirror
+                ud.ignore_checksums = True
+                return
+            else:
+                bb.utils.mkdirhier(ud.clonedir)
+                os.chdir(ud.clonedir)
+                runfetchcmd("tar -xzf %s" % ud.fullmirror, d)
 
         repourl = self._get_repo_url(ud)
 

@@ -225,22 +256,68 @@ class Git(FetchMethod):
 
     def build_mirror_data(self, ud, d):
         # Generate a mirror tarball if needed
-        if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+        if ud.shallows:
+            should_write = not os.path.exists(ud.fullmirror)
+        else:
+            should_write = ud.repochanged or not os.path.exists(ud.fullmirror)
+
+        if ud.write_ud.fullmirrors and should_write:
             # it's possible that this symlink points to read-only filesystem with PREMIRROR
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
-            os.chdir(ud.clonedir)
-            logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+            if ud.shallows:
+                tempdir = tempfile.mkdtemp()
+                shallowclone = os.path.join(tempdir, 'git')
+                os.makedirs(shallowclone)
+                try:
+                    os.chdir(shallowclone)
+                    runfetchcmd("%s init --bare" % ud.basecmd, d)
+                    with open(os.path.join(shallowclone, 'shallow'), 'w') as f:
+                        for name, shallow in ud.shallows.iteritems():
+                            if not shallow:
+                                continue
+                            revision = ud.revisions[name]
+
+                            for rev in [shallow + "^{}", shallow]:
+                                try:
+                                    revision = runfetchcmd("GIT_DIR=%s %s rev-parse %s" % (ud.clonedir, ud.basecmd, rev), d)
+                                except bb.fetch2.FetchError:
+                                    pass
+                                else:
+                                    break
+                            else:
+                                try:
+                                    shallow = int(shallow)
+                                except ValueError:
+                                    raise bb.fetch2.FetchError("Invalid shallow= parameter value '%s' in %s" % (shallow, ud.url))
+                                else:
+                                    revision = runfetchcmd("GIT_DIR=%s %s rev-parse %s~%d" % (ud.clonedir, ud.basecmd, revision, shallow - 1), d)
+
+                            f.write(revision)
+
+                    runfetchcmd("%s remote add origin %s" % (ud.basecmd, ud.clonedir), d)
+                    for name, branch in ud.branches.iteritems():
+                        runfetchcmd("%s fetch -a origin %s" % (ud.basecmd, branch), d)
+                    repourl = self._get_repo_url(ud)
+                    runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
+                    logger.info("Creating ud.fullmirror of git repository")
+                    runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".")), d)
+                    runfetchcmd("touch %s.done" % ud.fullmirror, d)
+                finally:
+                    bb.utils.remove(tempdir, recurse=True)
+            else:
+                os.chdir(ud.clonedir)
+                logger.info("Creating ud.fullmirror of git repository")
+                runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".")), d)
+                runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
         subdir = ud.parm.get("subpath", "")
         if subdir != "":
-            readpathspec = ":%s" % (subdir)
+            readpathspec = ":%s" % subdir
             def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
         else:
             readpathspec = ""

@@ -251,27 +328,35 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        cloneflags = "-s -n"
-        if ud.bareclone:
-            cloneflags += " --mirror"
-
-        # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused
-        # and you end up with some horrible union of the two when you attempt to clone it
-        # The least invasive workaround seems to be a symlink to the real directory to
-        # fool git into ignoring any .git version that may also be present.
-        #
-        # The issue is fixed in more recent versions of git so we can drop this hack in future
-        # when that version becomes common enough.
-        clonedir = ud.clonedir
-        if not ud.path.endswith(".git"):
-            indirectiondir = destdir[:-1] + ".indirectionsymlink"
-            if os.path.exists(indirectiondir):
-                os.remove(indirectiondir)
-            bb.utils.mkdirhier(os.path.dirname(indirectiondir))
-            os.symlink(ud.clonedir, indirectiondir)
-            clonedir = indirectiondir
-
-        runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
+        if ud.shallows and not os.path.exists(ud.clonedir):
+            gitdir = os.path.join(destdir, '.git')
+            bb.utils.mkdirhier(gitdir)
+            os.chdir(gitdir)
+            runfetchcmd("tar -xzf %s" % ud.fullmirror, d)
+            runfetchcmd("git config core.bare false", d)
+        else:
+            cloneflags = "-s -n"
+            if ud.bareclone:
+                cloneflags += " --mirror"
+
+            # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused
+            # and you end up with some horrible union of the two when you attempt to clone it
+            # The least invasive workaround seems to be a symlink to the real directory to
+            # fool git into ignoring any .git version that may also be present.
+            #
+            # The issue is fixed in more recent versions of git so we can drop this hack in future
+            # when that version becomes common enough.
+            clonedir = ud.clonedir
+            if not ud.path.endswith(".git"):
+                indirectiondir = destdir[:-1] + ".indirectionsymlink"
+                if os.path.exists(indirectiondir):
+                    os.remove(indirectiondir)
+                bb.utils.mkdirhier(os.path.dirname(indirectiondir))
+                os.symlink(ud.clonedir, indirectiondir)
+                clonedir = indirectiondir
+
+            runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
+
         os.chdir(destdir)
         repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)

index e88736d..c19b111 100644
@@ -299,6 +299,11 @@ class Git(FetchMethod):
                     runfetchcmd("%s remote add origin %s" % (ud.basecmd, ud.clonedir), d)
                     for name, branch in ud.branches.iteritems():
                         runfetchcmd("%s fetch -a origin %s" % (ud.basecmd, branch), d)
+                        runfetchcmd("%s update-ref refs/remotes/origin/%s %s" % (ud.basecmd, branch, ud.revisions[name]), d)
+                    runfetchcmd("%s gc --prune" % ud.basecmd, d)
+                    runfetchcmd("%s prune-packed" % ud.basecmd, d)
+                    runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+
                     repourl = self._get_repo_url(ud)
                     runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
                     logger.info("Creating ud.fullmirror of git repository")

index 7b4d130..c5e1758 100644
@@ -933,8 +933,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         # We may be obtaining a mirror tarball which needs further processing by the real fetcher
         # If that tarball is a local file:// we need to provide a symlink to it
         dldir = ld.getVar("DL_DIR", True)
-        if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
-                and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+        if origud.method != ud.method:
             # Create donestamp in old format to avoid triggering a re-download
             bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
             open(ud.donestamp, 'w').close()

@@ -943,7 +942,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
                 os.symlink(ud.localpath, dest)
             if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
                 origud.method.download(origud, ld)
-                if hasattr(origud.method,"build_mirror_data"):
+                if hasattr(origud.method, "build_mirror_data"):
                     origud.method.build_mirror_data(origud, ld)
             return ud.localpath
         # Otherwise the result is a local file:// and we symlink to it

index c5e1758..207c4e4 100644
@@ -429,7 +429,7 @@ def encodeurl(decoded):
 
     return url
 
-def uri_replace(ud, uri_find, uri_replace, replacements, d):
+def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     if not ud.url or not uri_find or not uri_replace:
         logger.error("uri_replace: passed an undefined value, not replacing")
         return None

@@ -468,9 +468,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d):
             if loc == 2:
                 # Handle path manipulations
                 basename = None
-                if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball:
+                if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball:
                     # If the source and destination url types differ, must be a mirrortarball mapping
-                    basename = os.path.basename(ud.mirrortarball)
+                    basename = os.path.basename(mirrortarball)
                     # Kill parameters, they make no sense for mirror tarballs
                     uri_decoded[5] = {}
                 elif ud.localpath and ud.method.supports_checksum(ud):

@@ -858,13 +858,13 @@ def build_mirroruris(origud, mirrors, ld):
     replacements["BASENAME"] = origud.path.split("/")[-1]
     replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
 
-    def adduri(ud, uris, uds):
+    def adduri(ud, uris, uds, tarball):
         for line in mirrors:
             try:
                 (find, replace) = line
             except ValueError:
                 continue
-            newuri = uri_replace(ud, find, replace, replacements, ld)
+            newuri = uri_replace(ud, find, replace, replacements, ld, tarball)
             if not newuri or newuri in uris or newuri == origud.url:
                 continue
 

@@ -881,16 +881,20 @@ def build_mirroruris(origud, mirrors, ld):
                 try:
                     # setup_localpath of file:// urls may fail, we should still see 
                     # if mirrors of the url exist
-                    adduri(newud, uris, uds)
+                    adduri(newud, uris, uds, tarball)
                 except UnboundLocalError:
                     pass
-                continue   
+                continue
             uris.append(newuri)
             uds.append(newud)
 
-            adduri(newud, uris, uds)
+            adduri(newud, uris, uds, tarball)
 
-    adduri(origud, uris, uds)
+    if hasattr(origud, 'mirrortarballs'):
+        for tarball in origud.mirrortarballs:
+            adduri(origud, uris, uds, tarball)
+    else:
+        adduri(origud, uris, uds, getattr(origud, 'mirrortarball', None))
 
     return uris, uds
 

index c19b111..cb2e9a7 100644
@@ -156,12 +156,7 @@ class Git(FetchMethod):
         # per revision, so that even the revision disappears from the
         # upstream repo in the future, the mirror will remain intact and still
         # contains the revision
-        if ud.shallows:
-            for name, shallow in ud.shallows.iteritems():
-                gitsrcname = '%s_%s_%s' % (gitsrcname, ud.branches[name].replace('/', '.'), ud.revisions[name])
-                if shallow:
-                    gitsrcname = gitsrcname + "_" + shallow
-        elif ud.rebaseable:
+        if ud.rebaseable:
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
 

@@ -172,15 +167,24 @@ class Git(FetchMethod):
 
         ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname
         ud.fullmirror = os.path.join(dl_dir, ud.mirrortarball)
+        if ud.shallows:
+            tarballname = gitsrcname
+            for name, shallow in ud.shallows.iteritems():
+                tarballname = '%s_%s_%s' % (tarballname, ud.branches[name].replace('/', '.'), ud.revisions[name])
+                if shallow:
+                    tarballname = tarballname + "_" + shallow
+            ud.shallowtarball = 'git2_%s.tar.gz' % tarballname
+            ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
+            ud.mirrortarballs = [ud.shallowtarball, ud.mirrortarball]
 
     def localpath(self, ud, d):
-        if ud.shallows and os.path.exists(ud.fullmirror):
-            return ud.fullmirror
+        if ud.shallows and os.path.exists(ud.fullshallow):
+            return ud.fullshallow
         else:
             return ud.clonedir
 
     def need_update(self, ud, d):
-        if ud.shallows and os.path.exists(ud.fullmirror):
+        if ud.shallows and os.path.exists(ud.fullshallow):
             return False
         if not os.path.exists(ud.clonedir):
             return True

@@ -188,7 +192,7 @@ class Git(FetchMethod):
         for name in ud.names:
             if not self._contains_ref(ud, d, name):
                 return True
-        if ud.write_tarballs and not os.path.exists(ud.fullmirror if not ud.shallows else ud.fullmirror):
+        if ud.write_tarballs and not os.path.exists(ud.fullmirror if not ud.shallows else ud.fullshallow):
             return True
         return False
 

@@ -207,12 +211,12 @@ class Git(FetchMethod):
         ud.repochanged = not os.path.exists(ud.fullmirror)
 
         # If the checkout doesn't exist and the mirror tarball does, extract it
-        if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
-            if ud.shallows:
-                ud.localpath = ud.fullmirror
+        if not os.path.exists(ud.clonedir):
+            if ud.shallows and os.path.exists(ud.fullshallow):
+                ud.localpath = ud.fullshallow
                 ud.ignore_checksums = True
                 return
-            else:
+            elif os.path.exists(ud.fullmirror):
                 bb.utils.mkdirhier(ud.clonedir)
                 os.chdir(ud.clonedir)
                 runfetchcmd("tar -xzf %s" % ud.fullmirror, d)

@@ -257,14 +261,16 @@ class Git(FetchMethod):
     def build_mirror_data(self, ud, d):
         # Generate a mirror tarball if needed
         if ud.shallows:
-            should_write = not os.path.exists(ud.fullmirror)
+            tarball = ud.fullshallow
+            should_write = not os.path.exists(ud.fullshallow)
         else:
+            tarball = ud.fullmirror
             should_write = ud.repochanged or not os.path.exists(ud.fullmirror)
 
-        if ud.write_ud.fullmirrors and should_write:
+        if ud.write_tarballs and should_write:
             # it's possible that this symlink points to read-only filesystem with PREMIRROR
-            if os.path.islink(ud.fullmirror):
-                os.unlink(ud.fullmirror)
+            if os.path.islink(tarball):
+                os.unlink(tarball)
 
             if ud.shallows:
                 tempdir = tempfile.mkdtemp()

@@ -306,16 +312,16 @@ class Git(FetchMethod):
 
                     repourl = self._get_repo_url(ud)
                     runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
-                    logger.info("Creating ud.fullmirror of git repository")
-                    runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".")), d)
-                    runfetchcmd("touch %s.done" % ud.fullmirror, d)
+                    logger.info("Creating tarball of git repository")
+                    runfetchcmd("tar -czf %s %s" % (tarball, os.path.join(".")), d)
+                    runfetchcmd("touch %s.done" % tarball, d)
                 finally:
                     bb.utils.remove(tempdir, recurse=True)
             else:
                 os.chdir(ud.clonedir)
-                logger.info("Creating ud.fullmirror of git repository")
-                runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".")), d)
-                runfetchcmd("touch %s.done" % ud.fullmirror, d)
+                logger.info("Creating tarball of git repository")
+                runfetchcmd("tar -czf %s %s" % (tarball, os.path.join(".")), d)
+                runfetchcmd("touch %s.done" % tarball, d)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""

@@ -337,7 +343,7 @@ class Git(FetchMethod):
             gitdir = os.path.join(destdir, '.git')
             bb.utils.mkdirhier(gitdir)
             os.chdir(gitdir)
-            runfetchcmd("tar -xzf %s" % ud.fullmirror, d)
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d)
             runfetchcmd("git config core.bare false", d)
         else:
             cloneflags = "-s -n"