Skip to contentSkip to conversation

mirror pushing

PATCH In review Junio C Hamano 3 participants 5 messages Jun 15, 2016

lore
Share & tools
lore
#0 lore raw

Existing "git push --all" is almost perfect for backing up to another repository, except that "--all" only means "all branches" in modern git, and it does not delete old branches and tags that exist at the back-up repository that you have removed from your local repository.

This teaches "git-send-pack" a new "--mirror" option. The difference from the "--all" option are that (1) it sends all refs, not just branches, and (2) it deletes old refs you no longer have on the local side from the remote side.

---

* This even applies to "maint", but it probably should be done on top of Daniel's remote.c changes. Teaching this to "git push" wrapper is left as an exercise to the reader.

 remote.c    |   15 ++++++++++-----
 send-pack.c |   35 ++++++++++++++++++++++++-----------
 2 files changed, 34 insertions(+), 16 deletions(-)
remote.c +10 -5
index bb774d0..a3aa5ad 100644
@@ -574,10 +574,12 @@ static const struct refspec *check_pattern_match(const struct refspec *rs,
  * without thinking.
  */
 int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,
-	       int nr_refspec, char **refspec, int all)
+	       int nr_refspec, char **refspec, int flags)
 {
 	struct refspec *rs =
 		parse_ref_spec(nr_refspec, (const char **) refspec);
+	int send_all = flags & 01;
+	int send_mirror = flags & 02;
 
 	if (match_explicit_refs(src, dst, dst_tail, rs, nr_refspec))
 		return -1;
@@ -594,7 +596,7 @@ int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,
 			if (!pat)
 				continue;
 		}
-		else if (prefixcmp(src->name, "refs/heads/"))
+		else if (!send_mirror && prefixcmp(src->name, "refs/heads/"))
 			/*
 			 * "matching refs"; traditionally we pushed everything
 			 * including refs outside refs/heads/ hierarchy, but
@@ -615,10 +617,13 @@ int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,
 		if (dst_peer && dst_peer->peer_ref)
 			/* We're already sending something to this ref. */
 			goto free_name;
-		if (!dst_peer && !nr_refspec && !all)
-			/* Remote doesn't have it, and we have no
+
+		if (!dst_peer && !nr_refspec && !(send_all || send_mirror))
+			/*
+			 * Remote doesn't have it, and we have no
 			 * explicit pattern, and we don't have
-			 * --all. */
+			 * --all nor --mirror.
+			 */
 			goto free_name;
 		if (!dst_peer) {
 			/* Create a new one and link it */
send-pack.c +24 -11
index 9fc8a81..39b4b17 100644
@@ -7,11 +7,12 @@
 #include "remote.h"
 
 static const char send_pack_usage[] =
-"git-send-pack [--all] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n"
+"git-send-pack [--all | --mirror] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n"
 "  --all and explicit <ref> specification are mutually exclusive.";
 static const char *receivepack = "git-receive-pack";
 static int verbose;
 static int send_all;
+static int send_mirror;
 static int force_update;
 static int use_thin_pack;
 
@@ -200,7 +201,7 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 	if (!remote_tail)
 		remote_tail = &remote_refs;
 	if (match_refs(local_refs, remote_refs, &remote_tail,
-		       nr_refspec, refspec, send_all))
+		       nr_refspec, refspec, (send_all | (send_mirror << 1))))
 		return -1;
 
 	if (!remote_refs) {
@@ -215,19 +216,24 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 	for (ref = remote_refs; ref; ref = ref->next) {
 		char old_hex[60], *new_hex;
 		int will_delete_ref;
+		const unsigned char *new_sha1;
 
-		if (!ref->peer_ref)
-			continue;
-
+		if (!ref->peer_ref) {
+			if (!send_mirror)
+				continue;
+			new_sha1 = null_sha1;
+		}
+		else
+			new_sha1 = ref->peer_ref->new_sha1;
 
-		will_delete_ref = is_null_sha1(ref->peer_ref->new_sha1);
+		will_delete_ref = is_null_sha1(new_sha1);
 		if (will_delete_ref && !allow_deleting_refs) {
 			error("remote does not support deleting refs");
 			ret = -2;
 			continue;
 		}
 		if (!will_delete_ref &&
-		    !hashcmp(ref->old_sha1, ref->peer_ref->new_sha1)) {
+		    !hashcmp(ref->old_sha1, new_sha1)) {
 			if (verbose)
 				fprintf(stderr, "'%s': up-to-date\n", ref->name);
 			continue;
@@ -257,8 +263,7 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 		    !is_null_sha1(ref->old_sha1) &&
 		    !ref->force) {
 			if (!has_sha1_file(ref->old_sha1) ||
-			    !ref_newer(ref->peer_ref->new_sha1,
-				       ref->old_sha1)) {
+			    !ref_newer(new_sha1, ref->old_sha1)) {
 				/* We do not have the remote ref, or
 				 * we know that the remote ref is not
 				 * an ancestor of what we are trying to
@@ -276,7 +281,7 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 				continue;
 			}
 		}
-		hashcpy(ref->new_sha1, ref->peer_ref->new_sha1);
+		hashcpy(ref->new_sha1, new_sha1);
 		if (!will_delete_ref)
 			new_refs++;
 		strcpy(old_hex, sha1_to_hex(ref->old_sha1));
@@ -396,6 +401,10 @@ int main(int argc, char **argv)
 				send_all = 1;
 				continue;
 			}
+			if (!strcmp(arg, "--mirror")) {
+				send_mirror = 1;
+				continue;
+			}
 			if (!strcmp(arg, "--force")) {
 				force_update = 1;
 				continue;
@@ -420,7 +429,11 @@ int main(int argc, char **argv)
 	}
 	if (!dest)
 		usage(send_pack_usage);
-	if (heads && send_all)
+	/*
+	 * --all and --mirror are incompatible; neither makes sense
+	 * with any refspecs.
+	 */
+	if ((heads && (send_all || send_mirror)) || (send_all && send_mirror))
 		usage(send_pack_usage);
 	verify_remote_names(nr_heads, heads);
 
#1 lore raw
Quoted 155 lines · Junio C Hamano

Existing "git push --all" is almost perfect for backing up to another repository, except that "--all" only means "all branches" in modern git, and it does not delete old branches and tags that exist at the back-up repository that you have removed from your local repository.

This teaches "git-send-pack" a new "--mirror" option. The difference from the "--all" option are that (1) it sends all refs, not just branches, and (2) it deletes old refs you no longer have on the local side from the remote side.

---

* This even applies to "maint", but it probably should be done on top of Daniel's remote.c changes. Teaching this to "git push" wrapper is left as an exercise to the reader.

 remote.c    |   15 ++++++++++-----
 send-pack.c |   35 ++++++++++++++++++++++++-----------
 2 files changed, 34 insertions(+), 16 deletions(-)
remote.c +10 -5
index bb774d0..a3aa5ad 100644
@@ -574,10 +574,12 @@ static const struct refspec *check_pattern_match(const struct refspec *rs,
  * without thinking.
  */
 int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,
-	       int nr_refspec, char **refspec, int all)
+	       int nr_refspec, char **refspec, int flags)
 {
 	struct refspec *rs =
 		parse_ref_spec(nr_refspec, (const char **) refspec);
+	int send_all = flags & 01;
+	int send_mirror = flags & 02;
 
 	if (match_explicit_refs(src, dst, dst_tail, rs, nr_refspec))
 		return -1;
@@ -594,7 +596,7 @@ int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,
 			if (!pat)
 				continue;
 		}
-		else if (prefixcmp(src->name, "refs/heads/"))
+		else if (!send_mirror && prefixcmp(src->name, "refs/heads/"))
 			/*
 			 * "matching refs"; traditionally we pushed everything
 			 * including refs outside refs/heads/ hierarchy, but
@@ -615,10 +617,13 @@ int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,
 		if (dst_peer && dst_peer->peer_ref)
 			/* We're already sending something to this ref. */
 			goto free_name;
-		if (!dst_peer && !nr_refspec && !all)
-			/* Remote doesn't have it, and we have no
+
+		if (!dst_peer && !nr_refspec && !(send_all || send_mirror))
+			/*
+			 * Remote doesn't have it, and we have no
 			 * explicit pattern, and we don't have
-			 * --all. */
+			 * --all nor --mirror.
+			 */
 			goto free_name;
 		if (!dst_peer) {
 			/* Create a new one and link it */
send-pack.c +19 -10
index 9fc8a81..39b4b17 100644
@@ -7,11 +7,12 @@
 #include "remote.h"
 
 static const char send_pack_usage[] =
-"git-send-pack [--all] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n"
+"git-send-pack [--all | --mirror] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n"
 "  --all and explicit <ref> specification are mutually exclusive.";
 static const char *receivepack = "git-receive-pack";
 static int verbose;
 static int send_all;
+static int send_mirror;
 static int force_update;
 static int use_thin_pack;
 
@@ -200,7 +201,7 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 	if (!remote_tail)
 		remote_tail = &remote_refs;
 	if (match_refs(local_refs, remote_refs, &remote_tail,
-		       nr_refspec, refspec, send_all))
+		       nr_refspec, refspec, (send_all | (send_mirror << 1))))
 		return -1;
 
 	if (!remote_refs) {
@@ -215,19 +216,24 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 	for (ref = remote_refs; ref; ref = ref->next) {
 		char old_hex[60], *new_hex;
 		int will_delete_ref;
+		const unsigned char *new_sha1;
 
-		if (!ref->peer_ref)
-			continue;
-
+		if (!ref->peer_ref) {
+			if (!send_mirror)
+				continue;
+			new_sha1 = null_sha1;
+		}
+		else
+			new_sha1 = ref->peer_ref->new_sha1;
 
-		will_delete_ref = is_null_sha1(ref->peer_ref->new_sha1);
+		will_delete_ref = is_null_sha1(new_sha1);
 		if (will_delete_ref && !allow_deleting_refs) {
 			error("remote does not support deleting refs");
 			ret = -2;
 			continue;
 		}
 		if (!will_delete_ref &&
-		    !hashcmp(ref->old_sha1, ref->peer_ref->new_sha1)) {
+		    !hashcmp(ref->old_sha1, new_sha1)) {
 			if (verbose)
 				fprintf(stderr, "'%s': up-to-date\n", ref->name);
 			continue;
@@ -257,8 +263,7 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 		    !is_null_sha1(ref->old_sha1) &&
 		    !ref->force) {
 			if (!has_sha1_file(ref->old_sha1) ||
-			    !ref_newer(ref->peer_ref->new_sha1,
-				       ref->old_sha1)) {
+			    !ref_newer(new_sha1, ref->old_sha1)) {
 				/* We do not have the remote ref, or
 				 * we know that the remote ref is not
 				 * an ancestor of what we are trying to
@@ -276,7 +281,7 @@ static int send_pack(int in, int out, struct remote *remote, int nr_refspec, cha
 				continue;
 			}
 		}
-		hashcpy(ref->new_sha1, ref->peer_ref->new_sha1);
+		hashcpy(ref->new_sha1, new_sha1);
 		if (!will_delete_ref)
 			new_refs++;
 		strcpy(old_hex, sha1_to_hex(ref->old_sha1));
@@ -396,6 +401,10 @@ int main(int argc, char **argv)
 				send_all = 1;
 				continue;
 			}
+			if (!strcmp(arg, "--mirror")) {
+				send_mirror = 1;
+				continue;
+			}
 			if (!strcmp(arg, "--force")) {
 				force_update = 1;
 				continue;
@@ -420,7 +429,11 @@ int main(int argc, char **argv)
 	}

send-pack.c int main(int argc, char **argv) · line 438

Quoted 10 lines
send-pack.c +5 -1
 	if (!dest)
 		usage(send_pack_usage);
-	if (heads && send_all)
+	/*
+	 * --all and --mirror are incompatible; neither makes sense
+	 * with any refspecs.
+	 */
+	if ((heads && (send_all || send_mirror)) || (send_all && send_mirror))
 		usage(send_pack_usage);
 	verify_remote_names(nr_heads, heads);

Ok, I put together a patch to tie this in (will post that following this email). However this mirror mode seems to do something odd. With matching repo's it reports all the tags as missing on the remote end, and attempts to resend them:

$ git push --mirror ssh://.../~apw/git/git 
updating 'junio-gpg-pub'
  from 0000000000000000000000000000000000000000
  to   a0e7d36193b96f552073558acf5fcc1f10528917
updating 'v0.99'
  from 0000000000000000000000000000000000000000
  to   d6602ec5194c87b0fc87103ca4d67251c76f233a
[...]
Generating pack...
Done counting 0 objects.
Writing 0 objects...
Total 0 (delta 0), reused 0 (delta 0)
error: Ref junio-gpg-pub is at a0e7d36193b96f552073558acf5fcc1f10528917 but expected 0000000000000000000000000000000000000000
error: failed to lock junio-gpg-pub
error: Ref v0.99 is at d6602ec5194c87b0fc87103ca4d67251c76f233a but expected 0000000000000000000000000000000000000000
error: failed to lock v0.99
[...]

It appears that they are ending up in the wrong place:

apw@kernel:~/git/git$ ls .git
branches       v0.99.5   v0.99.9a  v1.0.0b  v1.0rc5  v1.3.0      v1.4.2.1
config         v0.99.6   v0.99.9b  v1.0.10  v1.0rc6  v1.3.0-rc1  v1.4.2.2
description    v0.99.7   v0.99.9c  v1.0.11  v1.1.0   v1.3.0-rc2  v1.4.2.3
HEAD           v0.99.7a  v0.99.9d  v1.0.12  v1.1.1   v1.3.0-rc3  v1.4.2.4
hooks          v0.99.7b  v0.99.9e  v1.0.13  v1.1.2   v1.3.0-rc4  v1.4.2-rc1
index          v0.99.7c  v0.99.9f  v1.0.3   v1.1.3   v1.3.1      v1.4.2-rc2
info           v0.99.7d  v0.99.9g  v1.0.4   v1.1.4   v1.3.2      v1.4.2-rc3
junio-gpg-pub  v0.99.8   v0.99.9h  v1.0.5   v1.1.5   v1.3.3      v1.4.2-rc4
logs           v0.99.8a  v0.99.9i  v1.0.6   v1.1.6   v1.4.0      v1.4.3
objects        v0.99.8b  v0.99.9j  v1.0.7   v1.2.0   v1.4.0-rc1  v1.4.3.1
refs           v0.99.8c  v0.99.9k  v1.0.8   v1.2.1   v1.4.0-rc2  v1.4.3.2
v0.99          v0.99.8d  v0.99.9l  v1.0.9   v1.2.2   v1.4.1      v1.4.3-rc1
v0.99.1        v0.99.8e  v0.99.9m  v1.0rc1  v1.2.3   v1.4.1.1    v1.4.3-rc2
v0.99.2        v0.99.8f  v0.99.9n  v1.0rc2  v1.2.4   v1.4.1-rc1  v1.4.3-rc3
v0.99.3        v0.99.8g  v1.0.0    v1.0rc3  v1.2.5   v1.4.1-rc2
v0.99.4        v0.99.9   v1.0.0a   v1.0rc4  v1.2.6   v1.4.2

-apw

#2 lore raw

PATCH 1/1 git-push: plumb in --mirror mode

In reply to Junio C Hamano

Plumb in the --mirror mode for git-push.

---
 builtin-push.c |   14 +++++++++++++-
 transport.c    |    8 ++++++++
 transport.h    |    1 +
 3 files changed, 22 insertions(+), 1 deletions(-)
builtin-push.c +13 -1
index 4ee36c2..e421e96 100644
@@ -8,7 +8,7 @@
 #include "remote.h"
 #include "transport.h"
 
-static const char push_usage[] = "git-push [--all] [--tags] [--receive-pack=<git-receive-pack>] [--repo=all] [-f | --force] [-v] [<repository> <refspec>...]";
+static const char push_usage[] = "git-push [--all | --mirror] [--tags] [--receive-pack=<git-receive-pack>] [--repo=all] [-f | --force] [-v] [<repository> <refspec>...]";
 
 static int all, thin, verbose;
 static const char *receivepack;
@@ -85,6 +85,7 @@ int cmd_push(int argc, const char **argv, const char *prefix)
 {
 	int i;
 	int flags = 0;
+	int modes_specified = 0;
 	const char *repo = NULL;	/* default repository */
 
 	for (i = 1; i < argc; i++) {
@@ -105,6 +106,12 @@ int cmd_push(int argc, const char **argv, const char *prefix)
 		}
 		if (!strcmp(arg, "--all")) {
 			flags |= TRANSPORT_PUSH_ALL;
+			modes_specified++;
+			continue;
+		}
+		if (!strcmp(arg, "--mirror")) {
+			flags |= TRANSPORT_PUSH_MIRROR;
+			modes_specified++;
 			continue;
 		}
 		if (!strcmp(arg, "--tags")) {
@@ -137,5 +144,10 @@ int cmd_push(int argc, const char **argv, const char *prefix)
 	if (all && refspec)
 		usage(push_usage);
 
+	if (modes_specified > 1) {
+		error("--all and --mirror are incompatible");
+		usage(push_usage);
+	}
+
 	return do_push(repo, flags);
 }
transport.c +8 -0
index 7266fd3..e45f3c0 100644
@@ -281,6 +281,9 @@ static int rsync_transport_push(struct transport *transport,
 	struct child_process rsync;
 	const char *args[8];
 
+	if (flags & TRANSPORT_PUSH_MIRROR)
+		return error("rsync transport does not support mirror mode");
+
 	/* first push the objects */
 
 	strbuf_addstr(&buf, transport->url);
@@ -373,6 +376,9 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons
 	int argc;
 	int err;
 
+	if (flags & TRANSPORT_PUSH_MIRROR)
+		return error("http transport does not support mirror mode");
+
 	argv = xmalloc((refspec_nr + 11) * sizeof(char *));
 	argv[0] = "http-push";
 	argc = 1;
@@ -667,6 +673,8 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const
 		argv[argc++] = "--all";
 	if (flags & TRANSPORT_PUSH_FORCE)
 		argv[argc++] = "--force";
+	if (flags & TRANSPORT_PUSH_MIRROR)
+		argv[argc++] = "--mirror";
 	if (data->receivepack) {
 		char *rp = xmalloc(strlen(data->receivepack) + 16);
 		sprintf(rp, "--receive-pack=%s", data->receivepack);
transport.h +1 -0
index 6e318e4..8383774 100644
@@ -29,6 +29,7 @@ struct transport {
 
 #define TRANSPORT_PUSH_ALL 1
 #define TRANSPORT_PUSH_FORCE 2
+#define TRANSPORT_PUSH_MIRROR 4
 
 /* Returns a transport suitable for the url */
 struct transport *transport_get(struct remote *, const char *);
#3 lore raw

Re: [PATCH] git-push: plumb in --mirror mode

Hi,

builtin-push.c int cmd_push(int argc, const char **argv, const char *prefix) · line 151

Quoted 9 lines · Andy Whitcroft
builtin-push.c +5 -0
@@ -137,5 +144,10 @@ int cmd_push(int argc, const char **argv, const char *prefix)
 	if (all && refspec)
 		usage(push_usage);
 
+	if (modes_specified > 1) {
+		error("--all and --mirror are incompatible");
+		usage(push_usage);
+	}
+

Why not

	if ((flags & TRANSPORT_PUSH_ALL) && (flags & TRANSPORT_PUSH_MIRROR))?

It's more explicit.

transport.c static int git_transport_push(struct transport *transport, int refspec_nr, const · line 680

Quoted 9 lines
transport.c +2 -0
@@ -667,6 +673,8 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const
 		argv[argc++] = "--all";
 	if (flags & TRANSPORT_PUSH_FORCE)
 		argv[argc++] = "--force";
+	if (flags & TRANSPORT_PUSH_MIRROR)
+		argv[argc++] = "--mirror";
 	if (data->receivepack) {
 		char *rp = xmalloc(strlen(data->receivepack) + 16);
 		sprintf(rp, "--receive-pack=%s", data->receivepack);

Shouldn't you then increment the "11" a few lines before that, to ensure enough space for the new argument?

Ciao,
Dscho

#4 lore raw

Re: [PATCH] git-push: plumb in --mirror mode

Quoted 16 lines · Johannes Schindelin

Hi,

builtin-push.c int cmd_push(int argc, const char **argv, const char *prefix) · line 151

Quoted 9 lines · Andy Whitcroft
builtin-push.c +5 -0
@@ -137,5 +144,10 @@ int cmd_push(int argc, const char **argv, const char *prefix)
 	if (all && refspec)
 		usage(push_usage);
 
+	if (modes_specified > 1) {
+		error("--all and --mirror are incompatible");
+		usage(push_usage);
+	}
+

Why not

Quoted 3 lines
	if ((flags & TRANSPORT_PUSH_ALL) && (flags & TRANSPORT_PUSH_MIRROR))?

It's more explicit.

Yep, that does seem cleaner.

Quoted 4 lines
Quoted 4 lines
transport.c +0 -0
@@ -667,6 +673,8 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const
 		argv[argc++] = "--all";
 	if (flags & TRANSPORT_PUSH_FORCE)
 		argv[argc++] = "--force";
Quoted 8 lines

transport.c static int git_transport_push(struct transport *transport, int refspec_nr, const · line 680

Quoted 5 lines
transport.c +2 -0
+	if (flags & TRANSPORT_PUSH_MIRROR)
+		argv[argc++] = "--mirror";
 	if (data->receivepack) {
 		char *rp = xmalloc(strlen(data->receivepack) + 16);
 		sprintf(rp, "--receive-pack=%s", data->receivepack);

Shouldn't you then increment the "11" a few lines before that, to ensure enough space for the new argument?

I should have mentioned I'd not even reviewed it as the basic underlying functionality seemed to be broken. I'll look over it if I get a chance to try and debug the underlying failure.

-apw