git push mirror mode V4 (replacement stack)
Following this mail is a complete replacement git push mirror mode stack (V4). It folds down all the various patches into a logical sequence (thanks Dscho). This stack passes the entire test suite, and I have been using the same code for real work here.
Please replace what you have in pu with this stack.
-apw
PATCH 1/4 Teach send-pack a mirror mode
Existing "git push --all" is almost perfect for backing up to another repository, except that "--all" only means "all branches" in modern git, and it does not delete old branches and tags that exist at the back-up repository that you have removed from your local repository.
This teaches "git-send-pack" a new "--mirror" option. The difference from the "--all" option are that (1) it sends all refs, not just branches, and (2) it deletes old refs you no longer have on the local side from the remote side.
Original patch by Junio C Hamano.
Signed-off-by Andy Whitcroft <[email protected]>
--- builtin-send-pack.c | 46 ++++++++++++++++++++++++++++++++++------------ http-push.c | 4 ++-- remote.c | 15 ++++++++++----- remote.h | 7 +++++++ send-pack.h | 1 + 5 files changed, 54 insertions(+), 19 deletions(-)
builtin-send-pack.c +34 -12
@@ -8,7 +8,7 @@#include "send-pack.h" static const char send_pack_usage[] = -"git-send-pack [--all] [--dry-run] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n" +"git-send-pack [--all | --mirror] [--dry-run] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n" " --all and explicit <ref> specification are mutually exclusive." ;static struct send_pack_args args = {
@@ -227,6 +227,12 @@ static int do_send_pack(int in, int out, struct remote *remote, const char *destint allow_deleting_refs =0 ;int expect_status_report =0 ;int shown_dest =0 ; +int flags =MATCH_REFS_NONE ; + +if (args.send_all) + flags |=MATCH_REFS_ALL ; +if (args.send_mirror) + flags |=MATCH_REFS_MIRROR ;/* No funny business with the matcher */ remote_tail =get_remote_heads (in, &remote_refs,0 ,NULL ,REF_NORMAL );
@@ -242,7 +248,7 @@ static int do_send_pack(int in, int out, struct remote *remote, const char *destif (!remote_tail) remote_tail = &remote_refs;if (match_refs (local_refs, remote_refs, &remote_tail, - nr_refspec, refspec, args.send_all)) + nr_refspec, refspec, flags))return -1 ;if (!remote_refs) {
@@ -259,20 +265,28 @@ static int do_send_pack(int in, int out, struct remote *remote, const char *destchar old_hex[60 ], *new_hex;int will_delete_ref;const char *pretty_ref; -const char *pretty_peer; +const char *pretty_peer =NULL ;/* only used when not deleting */ +const unsigned char *new_sha1; -if (!ref->peer_ref) -continue ; +if (!ref->peer_ref) { +if (!args.send_mirror) +continue ; + new_sha1 = null_sha1; + } +else + new_sha1 = ref->peer_ref->new_sha1;if (!shown_dest) {fprintf (stderr,"To %s\n" , dest); shown_dest =1 ; } + will_delete_ref =is_null_sha1 (new_sha1); + pretty_ref =prettify_ref (ref->name); - pretty_peer =prettify_ref (ref->peer_ref->name); +if (!will_delete_ref) + pretty_peer =prettify_ref (ref->peer_ref->name); - will_delete_ref =is_null_sha1 (ref->peer_ref->new_sha1);if (will_delete_ref && !allow_deleting_refs) {fprintf (stderr," ! %-*s %s (remote does not support deleting refs)\n" ,SUMMARY_WIDTH ,"[rejected]" , pretty_ref);
@@ -280,7 +294,7 @@ static int do_send_pack(int in, int out, struct remote *remote, const char *destcontinue ; }if (!will_delete_ref && - !hashcmp (ref->old_sha1, ref->peer_ref->new_sha1)) { + !hashcmp (ref->old_sha1, new_sha1)) {if (args.verbose) fprintf(stderr, " = %-*s %s -> %s\n", SUMMARY_WIDTH, "[up to date]",
@@ -312,8 +326,7 @@ static int do_send_pack(int in, int out, struct remote *remote, const char *dest !is_null_sha1(ref->old_sha1) && !ref->force) {if (!has_sha1_file (ref->old_sha1) || - !ref_newer (ref->peer_ref->new_sha1, - ref->old_sha1)) { + !ref_newer (new_sha1, ref->old_sha1)) {/* We do not have the remote ref, or * we know that the remote ref is not * an ancestor of what we are trying to
@@ -328,7 +341,7 @@ static int do_send_pack(int in, int out, struct remote *remote, const char *destcontinue ; } } -hashcpy (ref->new_sha1, ref->peer_ref->new_sha1); +hashcpy (ref->new_sha1, new_sha1);if (!will_delete_ref) new_refs++;strcpy (old_hex,sha1_to_hex (ref->old_sha1));
@@ -459,6 +472,10 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) args.dry_run =1 ;continue ; } +if (!strcmp (arg,"--mirror" )) { + args.send_mirror =1 ; +continue ; + }if (!strcmp (arg,"--force" )) { args.force_update =1 ;continue ;
@@ -483,7 +500,12 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) }if (!dest)usage (send_pack_usage); -if (heads && args.send_all) +/* +* --all and --mirror are incompatible; neither makes sense +* with any refspecs. +*/ +if ((heads && (args.send_all || args.send_mirror)) || + (args.send_all && args.send_mirror))usage (send_pack_usage);if (remote_name) {
http-push.c +2 -2
@@ -78,7 +78,7 @@ static struct curl_slist *no_pragma_header;static struct curl_slist *default_headers;static int push_verbosely; -static int push_all; +static int push_all =MATCH_REFS_NONE ;static int force_all;static int dry_run;
@@ -2300,7 +2300,7 @@ int main(int argc, char **argv)if (*arg =='-' ) {if (!strcmp (arg,"--all" )) { - push_all =1 ; + push_all =MATCH_REFS_ALL ;continue ; }if (!strcmp (arg,"--force" )) {
remote.c +10 -5
@@ -722,10 +722,12 @@ static const struct refspec *check_pattern_match(const struct refspec *rs,* without thinking. */ int match_refs (struct ref *src,struct ref *dst,struct ref ***dst_tail, -int nr_refspec,const char **refspec,int all) +int nr_refspec,const char **refspec,int flags) {struct refspec *rs =parse_ref_spec (nr_refspec, (const char **) refspec); +int send_all = flags &MATCH_REFS_ALL ; +int send_mirror = flags &MATCH_REFS_MIRROR ;if (match_explicit_refs (src, dst, dst_tail, rs, nr_refspec))return -1 ;
@@ -742,7 +744,7 @@ int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,if (!pat)continue ; } -else if (prefixcmp (src->name,"refs/heads/" )) +else if (!send_mirror &&prefixcmp (src->name,"refs/heads/" ))/* * "matching refs"; traditionally we pushed everything * including refs outside refs/heads/ hierarchy, but
@@ -763,10 +765,13 @@ int match_refs(struct ref *src, struct ref *dst, struct ref ***dst_tail,if (dst_peer && dst_peer->peer_ref)/* We're already sending something to this ref. */ goto free_name; -if (!dst_peer && !nr_refspec && !all) -/* Remote doesn't have it, and we have no + +if (!dst_peer && !nr_refspec && !(send_all || send_mirror)) +/* +* Remote doesn't have it, and we have no * explicit pattern, and we don't have -* --all. */ +* --all nor --mirror. +*/ goto free_name;if (!dst_peer) {/* Create a new one and link it */
remote.h +7 -0
@@ -102,4 +102,11 @@ struct branch *branch_get(const char *name);int branch_has_merge_config (struct branch *branch);int branch_merge_matches (struct branch *,int n,const char *); +/* Flags to match_refs. */ +enum match_refs_flags { +MATCH_REFS_NONE =0 , +MATCH_REFS_ALL = (1 <<0 ), +MATCH_REFS_MIRROR = (1 <<1 ), +}; +#endif
send-pack.h +1 -0
@@ -5,6 +5,7 @@ struct send_pack_args {const char *receivepack; unsigned verbose:1, send_all:1, + send_mirror:1, force_update:1, use_thin_pack:1, dry_run:1;
PATCH 2/4 git-push: plumb in --mirror mode
In reply to Andy WhitcroftPlumb in the --mirror mode for git-push.
Signed-off-by Andy Whitcroft <[email protected]>
--- builtin-push.c | 14 ++++++++++++-- transport.c | 7 +++++++ transport.h | 1 + 3 files changed, 20 insertions(+), 2 deletions(-)
builtin-push.c +12 -2
@@ -10,7 +10,7 @@#include "parse-options.h" static const char *const push_usage[] = { -"git-push [--all] [--dry-run] [--tags] [--receive-pack=<git-receive-pack>] [--repo=all] [-f | --force] [-v] [<repository> <refspec>...]" , +"git-push [--all | --mirror] [--dry-run] [--tags] [--receive-pack=<git-receive-pack>] [--repo=all] [-f | --force] [-v] [<repository> <refspec>...]" ,NULL , };
@@ -91,6 +91,7 @@ int cmd_push(int argc, const char **argv, const char *prefix) {int flags =0 ;int all =0 ; +int mirror =0 ;int dry_run =0 ;int force =0 ;int tags =0 ;
@@ -100,6 +101,7 @@ int cmd_push(int argc, const char **argv, const char *prefix)OPT__VERBOSE (&verbose),OPT_STRING (0 ,"repo" , &repo,"repository" ,"repository" ),OPT_BOOLEAN (0 ,"all" , &all,"push all refs" ), +OPT_BOOLEAN (0 ,"mirror" , &mirror,"mirror all refs" ),OPT_BOOLEAN (0 ,"tags" , &tags,"push tags" ),OPT_BOOLEAN (0 ,"dry-run" , &dry_run,"dry run" ), OPT_BOOLEAN('f', "force", &force, "force updates"),
@@ -119,13 +121,21 @@ int cmd_push(int argc, const char **argv, const char *prefix)add_refspec ("refs/tags/*" );if (all) flags |=TRANSPORT_PUSH_ALL ; +if (mirror) + flags |= (TRANSPORT_PUSH_MIRROR |TRANSPORT_PUSH_FORCE );if (argc >0 ) { repo = argv[0 ];set_refspecs (argv +1 , argc -1 ); } -if ((flags &TRANSPORT_PUSH_ALL ) && refspec) +if ((flags & (TRANSPORT_PUSH_ALL |TRANSPORT_PUSH_MIRROR )) && refspec)usage_with_options (push_usage, options); +if ((flags & (TRANSPORT_PUSH_ALL |TRANSPORT_PUSH_MIRROR )) == + (TRANSPORT_PUSH_ALL |TRANSPORT_PUSH_MIRROR )) { +error ("--all and --mirror are incompatible" ); +usage_with_options (push_usage, options); + } +return do_push (repo, flags); }
transport.c +7 -0
@@ -284,6 +284,9 @@ static int rsync_transport_push(struct transport *transport,struct child_process rsync;const char *args[10 ]; +if (flags &TRANSPORT_PUSH_MIRROR ) +return error ("rsync transport does not support mirror mode" ); +/* first push the objects */ strbuf_addstr (&buf, transport->url);
@@ -387,6 +390,9 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, consint argc;int err; +if (flags &TRANSPORT_PUSH_MIRROR ) +return error ("http transport does not support mirror mode" ); + argv =xmalloc ((refspec_nr +11 ) *sizeof (char *)); argv[0 ] ="http-push" ; argc =1 ;
@@ -655,6 +661,7 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const args.receivepack = data->receivepack; args.send_all = !!(flags &TRANSPORT_PUSH_ALL ); + args.send_mirror = !!(flags &TRANSPORT_PUSH_MIRROR ); args.force_update = !!(flags &TRANSPORT_PUSH_FORCE ); args.use_thin_pack = data->thin; args.verbose = transport->verbose;
transport.h +1 -0
@@ -30,6 +30,7 @@ struct transport {#define TRANSPORT_PUSH_ALL 1#define TRANSPORT_PUSH_FORCE 2#define TRANSPORT_PUSH_DRY_RUN 4 +#define TRANSPORT_PUSH_MIRROR 8/* Returns a transport suitable for the url */ struct transport *transport_get (struct remote *,const char *);
PATCH 3/4 Add tests for git push'es mirror mode
In reply to Andy WhitcroftAdd some tests for git push --mirror mode.
Signed-off-by Andy Whitcroft <[email protected]>
--- t/t5517-push-mirror.sh | 228 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 228 insertions(+), 0 deletions(-)
t/t5517-push-mirror.sh new file +228 -0
@@ -0,0 +1,228 @@ +#!/bin/sh + +test_description='pushing to a mirror repository' + +. ./test-lib.sh + +D=`pwd` + +invert () { + if "$@"; then + return 1 + else + return 0 + fi +} + +mk_repo_pair () { + rm -rf master mirror && + mkdir mirror && + ( + cd mirror && + git init + ) && + mkdir master && + ( + cd master && + git init && + git config remote.up.url ../mirror + ) +} + + +# BRANCH tests +test_expect_success 'push mirror does not create new branches' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/heads/master) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/heads/master) && + test "$master_master" = "$mirror_master" + +' + +test_expect_success 'push mirror does not update existing branches' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git push --mirror up && + echo two >foo && git add foo && git commit -m two && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/heads/master) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/heads/master) && + test "$master_master" = "$mirror_master" + +' + +test_expect_success 'push mirror does not force update existing branches' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git push --mirror up && + echo two >foo && git add foo && git commit -m two && + git push --mirror up && + git reset --hard HEAD^ + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/heads/master) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/heads/master) && + test "$master_master" = "$mirror_master" + +' + +test_expect_success 'push mirror does not remove branches' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git branch remove master && + git push --mirror up && + git branch -D remove + git push --mirror up + ) && + ( + cd mirror && + invert git show-ref -s --verify refs/heads/remove + ) + +' + +test_expect_success 'push mirror does not add, update and remove branches together' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git branch remove master && + git push --mirror up && + git branch -D remove && + git branch add master && + echo two >foo && git add foo && git commit -m two && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/heads/master) && + master_add=$(cd master && git show-ref -s --verify refs/heads/add) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/heads/master) && + mirror_add=$(cd mirror && git show-ref -s --verify refs/heads/add) && + test "$master_master" = "$mirror_master" && + test "$master_add" = "$mirror_add" && + ( + cd mirror && + invert git show-ref -s --verify refs/heads/remove + ) + +' + + +# TAG tests +test_expect_success 'push mirror does not create new tags' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git tag -f tmaster master && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/tags/tmaster) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/tags/tmaster) && + test "$master_master" = "$mirror_master" + +' + +test_expect_success 'push mirror does not update existing tags' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git tag -f tmaster master && + git push --mirror up && + echo two >foo && git add foo && git commit -m two && + git tag -f tmaster master && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/tags/tmaster) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/tags/tmaster) && + test "$master_master" = "$mirror_master" + +' + +test_expect_success 'push mirror does not force update existing tags' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git tag -f tmaster master && + git push --mirror up && + echo two >foo && git add foo && git commit -m two && + git tag -f tmaster master && + git push --mirror up && + git reset --hard HEAD^ + git tag -f tmaster master && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/tags/tmaster) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/tags/tmaster) && + test "$master_master" = "$mirror_master" + +' + +test_expect_success 'push mirror does not remove tags' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git tag -f tremove master && + git push --mirror up && + git tag -d tremove + git push --mirror up + ) && + ( + cd mirror && + invert git show-ref -s --verify refs/tags/tremove + ) + +' + +test_expect_success 'push mirror does not add, update and remove tags together' ' + + mk_repo_pair && + ( + cd master && + echo one >foo && git add foo && git commit -m one && + git tag -f tmaster master && + git tag -f tremove master && + git push --mirror up && + git tag -d tremove && + git tag tadd master && + echo two >foo && git add foo && git commit -m two && + git tag -f tmaster master && + git push --mirror up + ) && + master_master=$(cd master && git show-ref -s --verify refs/tags/tmaster) && + master_add=$(cd master && git show-ref -s --verify refs/tags/tadd) && + mirror_master=$(cd mirror && git show-ref -s --verify refs/tags/tmaster) && + mirror_add=$(cd mirror && git show-ref -s --verify refs/tags/tadd) && + test "$master_master" = "$mirror_master" && + test "$master_add" = "$mirror_add" && + ( + cd mirror && + invert git show-ref -s --verify refs/tags/tremove + ) + +' + +test_done
PATCH 4/4 git-push: add documentation for the newly added --mirror mode
In reply to Andy WhitcroftAdd some basic documentation on the --mirror mode for git-push.
Signed-off-by Andy Whitcroft <[email protected]>
--- Documentation/git-push.txt | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-)
Documentation/git-push.txt +8 -0
@@ -63,6 +63,14 @@ the remote repository. Instead of naming each ref to push, specifies that all refs under `$GIT_DIR/refs/heads/` be pushed. +\--mirror:: + Instead of naming each ref to push, specifies that all + refs under `$GIT_DIR/refs/heads/` and `$GIT_DIR/refs/tags/` + be mirrored to the remote repository. Newly created local + refs will be pushed to the remote end, locally updated refs + will be force updated on the remote end, and deleted refs + will be removed from the remote end. + \--dry-run:: Do everything except actually send the updates.
Quoted 4 lines · Andy Whitcroft
Following this mail is a complete replacement git push mirror mode stack (V4). It folds down all the various patches into a logical sequence (thanks Dscho). This stack passes the entire test suite, and I have been using the same code for real work here.
Ok, I have spotted one oddity with this feature. The symbolic refs are getting converted to real refs in the mirror. Generally speaking this is the <remote>/HEAD refs but I guess it may be possible to have others.
I have had a looked about and I am actually confused as to whether we maintain remote symbolic refs at all? Cirtainly git-clone.sh seems to do some hoop jumping, comparing the sha1's of all of the fetched branches and replacing the HEAD reference with a symbolic reference should it find a match.
I am unsure if this a huge problem or not. Its not preventing me using it as an effective mirror, but if we assume one is making the mirror as a backup, then there would be slightly more than an rsync to convert the the repo back into your original, though I guess there already is as you would want to insert your config into the remote also.
Perhaps someone with a better understanding could point me to where we we maintain these refs, if we indeed do?
-apw
Quoted 9 lines · Andy Whitcroft
Quoted 4 lines · Andy Whitcroft
Following this mail is a complete replacement git push mirror mode stack (V4). It folds down all the various patches into a logical sequence (thanks Dscho). This stack passes the entire test suite, and I have been using the same code for real work here.
Ok, I have spotted one oddity with this feature. The symbolic refs are getting converted to real refs in the mirror. Generally speaking this is the <remote>/HEAD refs but I guess it may be possible to have others.
Currently there is no way to remotely create a symref.
It should be easy to make DAV based http push create a symref file.
The native "send-pack" protocol does not support such feature, so this needs a protocol extension, which hopefully would be not too involved.
Also, to be symmetric, we would need a protocol extension on the "fetch-pack" protocol side to allow fetchers to find out which one is a symref pointing out what other ref. The lack of the information is why git-clone has to guess.