clone: send ref-prefixes when using protocol v2

PATCH Superseded Brandon Williams 3 participants 4 messages Jul 20, 2018

lore

Share & tools

lore

Brandon Williams

author

2018-07-20 19:27 UTC #0 lore raw

PATCH 1/1 clone: send ref-prefixes when using protocol v2

Signed-off-by Brandon Williams <[email protected]>

---

Noticed we miss out on server side filtering of refs when cloning using protocol v2, this will enable that.

 builtin/clone.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

builtin/clone.c +17 -5

index 99e73dae85..55cc10e93a 100644
@@ -895,7 +895,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	int err = 0, complete_refs_before_fetch = 1;
 	int submodule_progress;
 
-	struct refspec_item refspec;
+	struct refspec rs = REFSPEC_INIT_FETCH;
+	struct argv_array ref_prefixes = ARGV_ARRAY_INIT;
 
 	fetch_if_missing = 0;

@@ -1077,7 +1078,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (option_required_reference.nr || option_optional_reference.nr)
 		setup_reference();
 
-	refspec_item_init(&refspec, value.buf, REFSPEC_FETCH);
+	refspec_append(&rs, value.buf);
 
 	strbuf_reset(&value);

@@ -1134,10 +1135,20 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (transport->smart_options && !deepen && !filter_options.choice)
 		transport->smart_options->check_self_contained_and_connected = 1;
 
-	refs = transport_get_remote_refs(transport, NULL);
+
+	argv_array_push(&ref_prefixes, "HEAD");
+	refspec_ref_prefixes(&rs, &ref_prefixes);
+	if (option_branch) {
+		expand_ref_prefix(&ref_prefixes, option_branch);
+	}
+	if (!option_no_tags) {
+		argv_array_push(&ref_prefixes, "refs/tags/");
+	}
+
+	refs = transport_get_remote_refs(transport, &ref_prefixes);
 
 	if (refs) {
-		mapped_refs = wanted_peer_refs(refs, &refspec);
+		mapped_refs = wanted_peer_refs(refs, &rs.items[0]);
 		/*
 		 * transport_get_remote_refs() may return refs with null sha-1
 		 * in mapped_refs (see struct transport->get_refs_list

@@ -1231,6 +1242,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	strbuf_release(&value);
 	junk_mode = JUNK_LEAVE_ALL;
 
-	refspec_item_clear(&refspec);
+	refspec_clear(&rs);
+	argv_array_clear(&ref_prefixes);
 	return err;
 }

Signature · 1 line

2.18.0.233.g985f88cf7e-goog

Junio C Hamano

2018-07-20 19:53 UTC #1 lore raw

Brandon Williams <[email protected]> writes:

Is there an end-user visible effect, caused by the lack of "prefix" being fixed with this patch, that is worth describing here? "The server ended up showing refs that are irrelevant to the normal clone request which is only for heads and tags, wasting time and bandwidth", for example?

Quoted 66 lines

Signed-off-by Brandon Williams <[email protected]>

---

Noticed we miss out on server side filtering of refs when cloning using protocol v2, this will enable that.

 builtin/clone.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

builtin/clone.c +17 -5

index 99e73dae85..55cc10e93a 100644
@@ -895,7 +895,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	int err = 0, complete_refs_before_fetch = 1;
 	int submodule_progress;
 
-	struct refspec_item refspec;
+	struct refspec rs = REFSPEC_INIT_FETCH;
+	struct argv_array ref_prefixes = ARGV_ARRAY_INIT;
 
 	fetch_if_missing = 0;

@@ -1077,7 +1078,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (option_required_reference.nr || option_optional_reference.nr)
 		setup_reference();
 
-	refspec_item_init(&refspec, value.buf, REFSPEC_FETCH);
+	refspec_append(&rs, value.buf);
 
 	strbuf_reset(&value);

@@ -1134,10 +1135,20 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (transport->smart_options && !deepen && !filter_options.choice)
 		transport->smart_options->check_self_contained_and_connected = 1;
 
-	refs = transport_get_remote_refs(transport, NULL);
+
+	argv_array_push(&ref_prefixes, "HEAD");
+	refspec_ref_prefixes(&rs, &ref_prefixes);
+	if (option_branch) {
+		expand_ref_prefix(&ref_prefixes, option_branch);
+	}
+	if (!option_no_tags) {
+		argv_array_push(&ref_prefixes, "refs/tags/");
+	}
+
+	refs = transport_get_remote_refs(transport, &ref_prefixes);
 
 	if (refs) {
-		mapped_refs = wanted_peer_refs(refs, &refspec);
+		mapped_refs = wanted_peer_refs(refs, &rs.items[0]);
 		/*
 		 * transport_get_remote_refs() may return refs with null sha-1
 		 * in mapped_refs (see struct transport->get_refs_list

@@ -1231,6 +1242,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	strbuf_release(&value);
 	junk_mode = JUNK_LEAVE_ALL;
 
-	refspec_item_clear(&refspec);
+	refspec_clear(&rs);
+	argv_array_clear(&ref_prefixes);
 	return err;
 }

Jonathan Nieder

2018-07-20 19:54 UTC #2 lore raw

In reply to Brandon Williams

Hi,

Quoted 7 lines · Brandon Williams

Signed-off-by Brandon Williams <[email protected]>
---
Noticed we miss out on server side filtering of refs when cloning using protocol v2, this will enable that.
 builtin/clone.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

Nice! The implementation looks good.

Can you add a test to ensure this filtering doesn't regress later?

[...]

Quoted 1 line

+++ b/builtin/clone.c

[...]

Quoted 4 lines

builtin/clone.c +0 -0

@@ -1134,10 +1135,20 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (transport->smart_options && !deepen && !filter_options.choice)
 		transport->smart_options->check_self_contained_and_connected = 1;

builtin/clone.c int cmd_clone(int argc, const char **argv, const char *prefix) · line 1146

Quoted 10 lines

builtin/clone.c +9 -1

-	refs = transport_get_remote_refs(transport, NULL);
+
+	argv_array_push(&ref_prefixes, "HEAD");
+	refspec_ref_prefixes(&rs, &ref_prefixes);
+	if (option_branch) {
+		expand_ref_prefix(&ref_prefixes, option_branch);
+	}
+	if (!option_no_tags) {
+		argv_array_push(&ref_prefixes, "refs/tags/");
+	}

nit: no need for braces around one-line "if" body

Thanks,
Jonathan

Brandon Williams

author

2018-07-20 22:08 UTC #3 lore raw

PATCH v2 1/1 clone: send ref-prefixes when using protocol v2

In reply to Brandon Williams

Teach clone to send a list of ref-prefixes, when using protocol v2, to allow the server to filter out irrelevant references from the ref-advertisement. This reduces wasted time and bandwidth when cloning repositories with a larger number of references.

Signed-off-by Brandon Williams <[email protected]>

---
 builtin/clone.c        | 20 +++++++++++++++-----
 t/t5702-protocol-v2.sh |  7 ++++++-
 2 files changed, 21 insertions(+), 6 deletions(-)

builtin/clone.c +15 -5

index 99e73dae85..5c0adbd6d0 100644
@@ -895,7 +895,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	int err = 0, complete_refs_before_fetch = 1;
 	int submodule_progress;
 
-	struct refspec_item refspec;
+	struct refspec rs = REFSPEC_INIT_FETCH;
+	struct argv_array ref_prefixes = ARGV_ARRAY_INIT;
 
 	fetch_if_missing = 0;

@@ -1077,7 +1078,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (option_required_reference.nr || option_optional_reference.nr)
 		setup_reference();
 
-	refspec_item_init(&refspec, value.buf, REFSPEC_FETCH);
+	refspec_append(&rs, value.buf);
 
 	strbuf_reset(&value);

@@ -1134,10 +1135,18 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	if (transport->smart_options && !deepen && !filter_options.choice)
 		transport->smart_options->check_self_contained_and_connected = 1;
 
-	refs = transport_get_remote_refs(transport, NULL);
+
+	argv_array_push(&ref_prefixes, "HEAD");
+	refspec_ref_prefixes(&rs, &ref_prefixes);
+	if (option_branch)
+		expand_ref_prefix(&ref_prefixes, option_branch);
+	if (!option_no_tags)
+		argv_array_push(&ref_prefixes, "refs/tags/");
+
+	refs = transport_get_remote_refs(transport, &ref_prefixes);
 
 	if (refs) {
-		mapped_refs = wanted_peer_refs(refs, &refspec);
+		mapped_refs = wanted_peer_refs(refs, &rs.items[0]);
 		/*
 		 * transport_get_remote_refs() may return refs with null sha-1
 		 * in mapped_refs (see struct transport->get_refs_list

@@ -1231,6 +1240,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 	strbuf_release(&value);
 	junk_mode = JUNK_LEAVE_ALL;
 
-	refspec_item_clear(&refspec);
+	refspec_clear(&rs);
+	argv_array_clear(&ref_prefixes);
 	return err;
 }

t/t5702-protocol-v2.sh +6 -1

index a4fe6508bd..9c6ea04a69 100755
@@ -181,7 +181,12 @@ test_expect_success 'clone with file:// using protocol v2' '
 	test_cmp expect actual &&
 
 	# Server responded using protocol v2
-	grep "clone< version 2" log
+	grep "clone< version 2" log &&
+	
+	# Client sent ref-prefixes to filter the ref-advertisement 
+	grep "ref-prefix HEAD" log &&
+	grep "ref-prefix refs/heads/" log &&
+	grep "ref-prefix refs/tags/" log
 '
 
 test_expect_success 'fetch with file:// using protocol v2' '

Signature · 1 line

2.18.0.233.g985f88cf7e-goog