Skip to contentSkip to conversation

Protocol v2 fix: http and auth

PATCH Superseded Jonathan Tan 5 participants 24 messages Feb 14, 2019 – Mar 4, 2019

lore
Share & tools
lore
#0 lore raw

cover Protocol v2 fix: http and auth

Peff noticed an issue with my http auth in protocol v2 patch earlier [1], and in the ensuing discussion, I thought that it would be best to make v2 use post_rpc() as well (to be the same as v0/v1) instead of using its own functions, to fix this issue and try to avoid a similar issue in the future. Besides that, there is also a net reduction in lines of code.

So here are the patches. First 4 are refactoring - the last one has the actual change. This is on the master branch.

[1] https://public-inbox.org/git/[email protected]/

Jonathan Tan (5):
remote-curl: reduce scope of rpc_state.argv
remote-curl: reduce scope of rpc_state.stdin_preamble
remote-curl: reduce scope of rpc_state.result
remote-curl: refactor reading into rpc_state's buf
remote-curl: use post_rpc() for protocol v2 also

 pkt-line.c             |   2 +-
 pkt-line.h             |   1 +
 remote-curl.c          | 351 ++++++++++++++++++-----------------------
 t/t5702-protocol-v2.sh |  26 ++-
 4 files changed, 181 insertions(+), 199 deletions(-)
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#1 lore raw

PATCH 1/5 remote-curl: reduce scope of rpc_state.argv

The argv field in struct rpc_state is only used in rpc_service(), and not in any functions it directly or indirectly calls. Refactor it to become an argument of rpc_service() instead.

---
 remote-curl.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
remote-curl.c +5 -7
index 2e04d53ac8..3bc5055da6 100644
@@ -491,7 +491,6 @@ static void output_refs(struct ref *refs)
 
 struct rpc_state {
 	const char *service_name;
-	const char **argv;
 	struct strbuf *stdin_preamble;
 	char *service_url;
 	char *hdr_content_type;
@@ -815,7 +814,8 @@ static int post_rpc(struct rpc_state *rpc)
 	return err;
 }
 
-static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
+static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
+		       const char **client_argv)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
@@ -826,7 +826,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
 	client.in = -1;
 	client.out = -1;
 	client.git_cmd = 1;
-	client.argv = rpc->argv;
+	client.argv = client_argv;
 	if (start_command(&client))
 		exit(1);
 	if (preamble)
@@ -964,11 +964,10 @@ static int fetch_git(struct discovery *heads,
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-upload-pack",
-	rpc.argv = args.argv;
 	rpc.stdin_preamble = &preamble;
 	rpc.gzip_request = 1;
 
-	err = rpc_service(&rpc, heads);
+	err = rpc_service(&rpc, heads, args.argv);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
@@ -1098,10 +1097,9 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-receive-pack",
-	rpc.argv = args.argv;
 	rpc.stdin_preamble = &preamble;
 
-	err = rpc_service(&rpc, heads);
+	err = rpc_service(&rpc, heads, args.argv);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#2 lore raw

PATCH 2/5 remote-curl: reduce scope of rpc_state.stdin_preamble

In reply to Jonathan Tan

The stdin_preamble field in struct rpc_state is only used in rpc_service(), and not in any functions it directly or indirectly calls. Refactor it to become an argument of rpc_service() instead.

An observation of all callers of rpc_service() shows that the preamble is always set, so we no longer need the "if (preamble)" check.

---
 remote-curl.c | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)
remote-curl.c +4 -9
index 3bc5055da6..bff0bb9af6 100644
@@ -491,7 +491,6 @@ static void output_refs(struct ref *refs)
 
 struct rpc_state {
 	const char *service_name;
-	struct strbuf *stdin_preamble;
 	char *service_url;
 	char *hdr_content_type;
 	char *hdr_accept;
@@ -815,11 +814,10 @@ static int post_rpc(struct rpc_state *rpc)
 }
 
 static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
-		       const char **client_argv)
+		       const char **client_argv, const struct strbuf *preamble)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
-	struct strbuf *preamble = rpc->stdin_preamble;
 	struct child_process client = CHILD_PROCESS_INIT;
 	int err = 0;
 
@@ -829,8 +827,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 	client.argv = client_argv;
 	if (start_command(&client))
 		exit(1);
-	if (preamble)
-		write_or_die(client.in, preamble->buf, preamble->len);
+	write_or_die(client.in, preamble->buf, preamble->len);
 	if (heads)
 		write_or_die(client.in, heads->buf, heads->len);
 
@@ -964,10 +961,9 @@ static int fetch_git(struct discovery *heads,
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-upload-pack",
-	rpc.stdin_preamble = &preamble;
 	rpc.gzip_request = 1;
 
-	err = rpc_service(&rpc, heads, args.argv);
+	err = rpc_service(&rpc, heads, args.argv, &preamble);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
@@ -1097,9 +1093,8 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-receive-pack",
-	rpc.stdin_preamble = &preamble;
 
-	err = rpc_service(&rpc, heads, args.argv);
+	err = rpc_service(&rpc, heads, args.argv, &preamble);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#3 lore raw

PATCH 3/5 remote-curl: reduce scope of rpc_state.result

In reply to Jonathan Tan

The result field in struct rpc_state is only used in rpc_service(), and not in any functions it directly or indirectly calls. Refactor it to become an argument of rpc_service() instead.

---
 remote-curl.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)
remote-curl.c +13 -12
index bff0bb9af6..2eb39774d0 100644
@@ -502,7 +502,6 @@ struct rpc_state {
 	int in;
 	int out;
 	int any_written;
-	struct strbuf result;
 	unsigned gzip_request : 1;
 	unsigned initial_buffer : 1;
 };
@@ -814,7 +813,8 @@ static int post_rpc(struct rpc_state *rpc)
 }
 
 static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
-		       const char **client_argv, const struct strbuf *preamble)
+		       const char **client_argv, const struct strbuf *preamble,
+		       struct strbuf *rpc_result)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
@@ -835,7 +835,6 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 	rpc->buf = xmalloc(rpc->alloc);
 	rpc->in = client.in;
 	rpc->out = client.out;
-	strbuf_init(&rpc->result, 0);
 
 	strbuf_addf(&buf, "%s%s", url.buf, svc);
 	rpc->service_url = strbuf_detach(&buf, NULL);
@@ -863,7 +862,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 	close(client.in);
 	client.in = -1;
 	if (!err) {
-		strbuf_read(&rpc->result, client.out, 0);
+		strbuf_read(rpc_result, client.out, 0);
 	} else {
 		char buf[4096];
 		for (;;)
@@ -916,6 +915,7 @@ static int fetch_git(struct discovery *heads,
 	struct strbuf preamble = STRBUF_INIT;
 	int i, err;
 	struct argv_array args = ARGV_ARRAY_INIT;
+	struct strbuf rpc_result = STRBUF_INIT;
 
 	argv_array_pushl(&args, "fetch-pack", "--stateless-rpc",
 			 "--stdin", "--lock-pack", NULL);
@@ -963,10 +963,10 @@ static int fetch_git(struct discovery *heads,
 	rpc.service_name = "git-upload-pack",
 	rpc.gzip_request = 1;
 
-	err = rpc_service(&rpc, heads, args.argv, &preamble);
-	if (rpc.result.len)
-		write_or_die(1, rpc.result.buf, rpc.result.len);
-	strbuf_release(&rpc.result);
+	err = rpc_service(&rpc, heads, args.argv, &preamble, &rpc_result);
+	if (rpc_result.len)
+		write_or_die(1, rpc_result.buf, rpc_result.len);
+	strbuf_release(&rpc_result);
 	strbuf_release(&preamble);
 	argv_array_clear(&args);
 	return err;
@@ -1061,6 +1061,7 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 	struct argv_array args;
 	struct string_list_item *cas_option;
 	struct strbuf preamble = STRBUF_INIT;
+	struct strbuf rpc_result = STRBUF_INIT;
 
 	argv_array_init(&args);
 	argv_array_pushl(&args, "send-pack", "--stateless-rpc", "--helper-status",
@@ -1094,10 +1095,10 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-receive-pack",
 
-	err = rpc_service(&rpc, heads, args.argv, &preamble);
-	if (rpc.result.len)
-		write_or_die(1, rpc.result.buf, rpc.result.len);
-	strbuf_release(&rpc.result);
+	err = rpc_service(&rpc, heads, args.argv, &preamble, &rpc_result);
+	if (rpc_result.len)
+		write_or_die(1, rpc_result.buf, rpc_result.len);
+	strbuf_release(&rpc_result);
 	strbuf_release(&preamble);
 	argv_array_clear(&args);
 	return err;
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#4 lore raw

PATCH 4/5 remote-curl: refactor reading into rpc_state's buf

In reply to Jonathan Tan

Currently, whenever remote-curl reads pkt-lines from its response file descriptor, only the payload is written to its buf, not the 4 characters denoting the length. A future patch will require the ability to also write those 4 characters, so in preparation for that, refactor this read into its own function.

---
 remote-curl.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)
remote-curl.c +24 -9
index 2eb39774d0..32c133f636 100644
@@ -506,6 +506,25 @@ struct rpc_state {
 	unsigned initial_buffer : 1;
 };
 
+/*
+ * Appends the result of reading from rpc->out to the string represented by
+ * rpc->buf and rpc->len if there is enough space. Returns 1 if there was
+ * enough space, 0 otherwise.
+ *
+ * Writes the number of bytes appended into appended.
+ */
+static int rpc_read_from_out(struct rpc_state *rpc, size_t *appended) {
+	size_t left = rpc->alloc - rpc->len;
+	char *buf = rpc->buf + rpc->len;
+
+	if (left < LARGE_PACKET_MAX)
+		return 0;
+
+	*appended = packet_read(rpc->out, NULL, NULL, buf, left, 0);
+	rpc->len += *appended;
+	return 1;
+}
+
 static size_t rpc_out(void *ptr, size_t eltsize,
 		size_t nmemb, void *buffer_)
 {
@@ -515,11 +534,12 @@ static size_t rpc_out(void *ptr, size_t eltsize,
 
 	if (!avail) {
 		rpc->initial_buffer = 0;
-		avail = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0);
+		rpc->len = 0;
+		if (!rpc_read_from_out(rpc, &avail))
+			BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");
 		if (!avail)
 			return 0;
 		rpc->pos = 0;
-		rpc->len = avail;
 	}
 
 	if (max < avail)
@@ -663,20 +683,15 @@ static int post_rpc(struct rpc_state *rpc)
 	 * chunked encoding mess.
 	 */
 	while (1) {
-		size_t left = rpc->alloc - rpc->len;
-		char *buf = rpc->buf + rpc->len;
-		int n;
+		size_t n;
 
-		if (left < LARGE_PACKET_MAX) {
+		if (!rpc_read_from_out(rpc, &n)) {
 			large_request = 1;
 			use_gzip = 0;
 			break;
 		}
-
-		n = packet_read(rpc->out, NULL, NULL, buf, left, 0);
 		if (!n)
 			break;
-		rpc->len += n;
 	}
 
 	if (large_request) {
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#5 lore raw

PATCH 5/5 remote-curl: use post_rpc() for protocol v2 also

In reply to Jonathan Tan

When transmitting and receiving POSTs for protocol v0 and v1, remote-curl uses post_rpc() (and associated functions), but when doing the same for protocol v2, it uses a separate set of functions (proxy_rpc() and others). Besides duplication of code, this has caused at least one bug: the auth retry mechanism that was implemented in v0/v1 was not implemented in v2.

To fix this issue and avoid it in the future, make remote-curl also use post_rpc() when handling protocol v2. Because line lengths are written to the HTTP request in protocol v2 (unlike in protocol v0/v1), this necessitates changes in post_rpc() and some of the functions it uses; perform these changes too.

A test has been included to ensure that the code for both the unchunked and chunked variants of the HTTP request is exercised.

Note: stateless_connect() has been updated to use the lower-level packet reading functions instead of struct packet_reader. The low-level control is necessary here because we cannot change the destination buffer of struct packet_reader while it is being used; struct packet_buffer has a peeking mechanism which relies on the destination buffer being present in between a peek and a read.

---
 pkt-line.c             |   2 +-
 pkt-line.h             |   1 +
 remote-curl.c          | 298 +++++++++++++++++------------------------
 t/t5702-protocol-v2.sh |  26 +++-
 4 files changed, 150 insertions(+), 177 deletions(-)
pkt-line.c +1 -1
index d4b71d3e82..60329b301b 100644
@@ -117,7 +117,7 @@ void packet_buf_delim(struct strbuf *buf)
 	strbuf_add(buf, "0001", 4);
 }
 
-static void set_packet_header(char *buf, const int size)
+void set_packet_header(char *buf, const int size)
 {
 	static char hexchar[] = "0123456789abcdef";
 
pkt-line.h +1 -0
index ad9a4a2cd7..c36cb788ed 100644
@@ -25,6 +25,7 @@ void packet_delim(int fd);
 void packet_write_fmt(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
 void packet_buf_flush(struct strbuf *buf);
 void packet_buf_delim(struct strbuf *buf);
+void set_packet_header(char *buf, int size);
 void packet_write(int fd_out, const char *buf, size_t size);
 void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
 void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len);
remote-curl.c +123 -175

362 lines not on this page · raw message

t/t5702-protocol-v2.sh +25 -1
index db4ae09f2f..61acf99d80 100755
@@ -542,7 +542,31 @@ test_expect_success 'clone with http:// using protocol v2' '
 	# Client requested to use protocol v2
 	grep "Git-Protocol: version=2" log &&
 	# Server responded using protocol v2
-	grep "git< version 2" log
+	grep "git< version 2" log &&
+	# Verify that the chunked encoding sending codepath is NOT exercised
+	! grep "Send header: Transfer-Encoding: chunked" log
+'
+
+test_expect_success 'clone big repository with http:// using protocol v2' '
+	test_when_finished "rm -f log" &&
+
+	git init "$HTTPD_DOCUMENT_ROOT_PATH/big" &&
+	# Ensure that the list of wants is greater than http.postbuffer below
+	for i in $(seq 1 1500)
+	do
+		test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/big" "commit$i"
+	done &&
+
+	GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git \
+		-c protocol.version=2 -c http.postbuffer=65536 \
+		clone "$HTTPD_URL/smart/big" big_child &&
+
+	# Client requested to use protocol v2
+	grep "Git-Protocol: version=2" log &&
+	# Server responded using protocol v2
+	grep "git< version 2" log &&
+	# Verify that the chunked encoding sending codepath is exercised
+	grep "Send header: Transfer-Encoding: chunked" log
 '
 
 test_expect_success 'fetch with http:// using protocol v2' '
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#6 lore raw

Re: [PATCH 5/5] remote-curl: use post_rpc() for protocol v2 also

t/t5702-protocol-v2.sh test_expect_success 'clone with http:// using protocol v2' ' · line 555

Quoted 2 lines · Jonathan Tan
t/t5702-protocol-v2.sh +2 -0
+	# Ensure that the list of wants is greater than http.postbuffer below
+	for i in $(seq 1 1500)

test_seq.

Quoted 17 lines
t/t5702-protocol-v2.sh +14 -0
+	do
+		test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/big" "commit$i"
+	done &&
+
+	GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git \
+		-c protocol.version=2 -c http.postbuffer=65536 \
+		clone "$HTTPD_URL/smart/big" big_child &&
+
+	# Client requested to use protocol v2
+	grep "Git-Protocol: version=2" log &&
+	# Server responded using protocol v2
+	grep "git< version 2" log &&
+	# Verify that the chunked encoding sending codepath is exercised
+	grep "Send header: Transfer-Encoding: chunked" log
 '
 
 test_expect_success 'fetch with http:// using protocol v2' '
#7 lore raw

Re: [PATCH 5/5] remote-curl: use post_rpc() for protocol v2 also

In reply to Jonathan Tan
Quoted 5 lines · Jonathan Tan
remote-curl.c +0 -0
index 32c133f636..13836e4c28 100644

remote-curl.c struct rpc_state { · line 512

Quoted 9 lines
remote-curl.c +6 -0
@@ -504,6 +504,18 @@ struct rpc_state {
 	int any_written;
 	unsigned gzip_request : 1;
 	unsigned initial_buffer : 1;
+
+	/*
+	 * Whenever a pkt-line is read into buf, append the 4 characters
+	 * denoting its length before appending the payload.
+	 */
+	unsigned write_line_lengths : 1;

Hmm, so we read a packet, and then we "append its length" before appending the contents. But that would always be the length we just read, right? I wonder if it would be simpler to just call this option something like "proxy_packets" or "full_packets", teach the packet code to give us the full packets, and then just treat that whole buffer as a unit. I dunno. There might be some gotchas in practice, and it's not like it's that much simpler. Just a thought.

remote-curl.c struct rpc_state { · line 518

Quoted 5 lines
remote-curl.c +5 -0
+	/*
+	 * rpc_out uses this to keep track of whether it should continue
+	 * reading to populate the current request. Initialize to 0.
+	 */
+	unsigned stop_reading : 1;

OK, so we need this because the v2 proxying will require us to stop reading but keep the channel open? Kind of awkward, but I don't see a way around it.

Quoted 4 lines
remote-curl.c +4 -0
+static int rpc_read_from_out(struct rpc_state *rpc, int options,
+			     size_t *appended,
+			     enum packet_read_status *status) {
+	size_t left;

remote-curl.c struct rpc_state { · line 544

Quoted 10 lines
remote-curl.c +10 -0
+	char *buf;
+	int pktlen_raw;
+
+	if (rpc->write_line_lengths) {
+		left = rpc->alloc - rpc->len - 4;
+		buf = rpc->buf + rpc->len + 4;
+	} else {
+		left = rpc->alloc - rpc->len;
+		buf = rpc->buf + rpc->len;
+	}

OK, so we push the packets 4 bytes further into the buffer in that case, leaving room for the header. Makes sense.

Quoted 19 lines
remote-curl.c +14 -2
 	if (left < LARGE_PACKET_MAX)
 		return 0;
 
-	*appended = packet_read(rpc->out, NULL, NULL, buf, left, 0);
-	rpc->len += *appended;
+	*status = packet_read_with_status(rpc->out, NULL, NULL, buf,
+			left, &pktlen_raw, options);
+	if (*status != PACKET_READ_EOF) {
+		*appended = pktlen_raw + (rpc->write_line_lengths ? 4 : 0);
+		rpc->len += *appended;
+	}
+
+	if (rpc->write_line_lengths) {
+		switch (*status) {
+		case PACKET_READ_EOF:
+			if (!(options & PACKET_READ_GENTLE_ON_EOF))
+				die("shouldn't have EOF when not gentle on EOF");
+			break;
+		case PACKET_READ_NORMAL:

remote-curl.c struct rpc_state { · line 572

Quoted 10 lines
remote-curl.c +10 -0
+			set_packet_header(buf - 4, *appended);
+			break;
+		case PACKET_READ_DELIM:
+			memcpy(buf - 4, "0001", 4);
+			break;
+		case PACKET_READ_FLUSH:
+			memcpy(buf - 4, "0000", 4);
+			break;
+		}
+	}

And here we fill it in. Make sense. It's a little awkward that we have to re-translate READ_DELIM, etc, back into their headers.

Quoted 7 lines
remote-curl.c +1 -0
@@ -531,15 +580,32 @@ static size_t rpc_out(void *ptr, size_t eltsize,
 	size_t max = eltsize * nmemb;
 	struct rpc_state *rpc = buffer_;
 	size_t avail = rpc->len - rpc->pos;
+	enum packet_read_status status;
 
 	if (!avail) {

remote-curl.c static size_t rpc_out(void *ptr, size_t eltsize, · line 591

Quoted 10 lines
remote-curl.c +3 -4
 		rpc->initial_buffer = 0;
 		rpc->len = 0;
-		if (!rpc_read_from_out(rpc, &avail))
-			BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");
-		if (!avail)
-			return 0;
 		rpc->pos = 0;
+		if (!rpc->stop_reading) {
+			if (!rpc_read_from_out(rpc, 0, &avail, &status))
+				BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");

Do we actually need it to be LARGE_PACKET_MAX+4 here? I guess not, because LARGE_PACKET_DATA_MAX is the "-4" version. So I think this BUG() was perhaps already wrong?

Quoted 7 lines
remote-curl.c +7 -0
+			if (status == PACKET_READ_FLUSH)
+				/*
+				 * We are done reading for this request, but we
+				 * still need to send this line out (if
+				 * rpc->write_line_lengths is true) so do not
+				 * return yet.
+				 */

remote-curl.c static size_t rpc_out(void *ptr, size_t eltsize, · line 608

Quoted 10 lines
remote-curl.c +10 -0
+				rpc->stop_reading = 1;
+		}
+	}
+	if (!avail && rpc->stop_reading) {
+		/*
+		 * "return 0" will notify Curl that this RPC request is done,
+		 * so reset stop_reading back to 0 for the next request.
+		 */
+		rpc->stop_reading = 0;
+		return 0;

OK, and here's where we handle the stop_reading thing. It is indeed awkward, but I think your comments make it clear what's going on.

If we get stop_reading, do we care about "avail"? I.e., shouldn't we be able to return non-zero to say "we got the whole input, this is not a too-large request"?

remote-curl.c static size_t rpc_out(void *ptr, size_t eltsize,

Quoted 9 lines
remote-curl.c +9 -0
+test_expect_success 'clone big repository with http:// using protocol v2' '
+	test_when_finished "rm -f log" &&
+
+	git init "$HTTPD_DOCUMENT_ROOT_PATH/big" &&
+	# Ensure that the list of wants is greater than http.postbuffer below
+	for i in $(seq 1 1500)
+	do
+		test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/big" "commit$i"
+	done &&

As Junio noted, this should be test_seq. But I think it would be nice to avoid looping on test_commit here at all. It kicks off at least 3 processes; multiplying that by 1500 is going to be slow.

Making a big input is often much faster by generating a fast-import stream (which can often be done entirely in-shell). There's some prior art in t3302, t5551, t5608, and others.

-Peff

#9 lore raw
In reply to Jonathan Tan
Quoted 9 lines · Jonathan Tan

Peff noticed an issue with my http auth in protocol v2 patch earlier [1], and in the ensuing discussion, I thought that it would be best to make v2 use post_rpc() as well (to be the same as v0/v1) instead of using its own functions, to fix this issue and try to avoid a similar issue in the future. Besides that, there is also a net reduction in lines of code.

So here are the patches. First 4 are refactoring - the last one has the actual change. This is on the master branch.

I think this is the right approach. Patches 1-4 look pretty nice. I didn't see anything wrong with the code in the 5th one, but I did have some thoughts about how we might be able to do it a bit more cleanly (though I will not at all be surprised if they turn out to be bad suggestions).

-Peff

#8 lore raw

Re: [PATCH 5/5] remote-curl: use post_rpc() for protocol v2 also

In reply to Jeff King
Quoted 15 lines
Quoted 5 lines
remote-curl.c +0 -0
index 32c133f636..13836e4c28 100644

remote-curl.c struct rpc_state { · line 512

Quoted 9 lines
remote-curl.c +6 -0
@@ -504,6 +504,18 @@ struct rpc_state {
 	int any_written;
 	unsigned gzip_request : 1;
 	unsigned initial_buffer : 1;
+
+	/*
+	 * Whenever a pkt-line is read into buf, append the 4 characters
+	 * denoting its length before appending the payload.
+	 */
+	unsigned write_line_lengths : 1;
Quoted 7 lines

Hmm, so we read a packet, and then we "append its length" before appending the contents. But that would always be the length we just read, right? I wonder if it would be simpler to just call this option something like "proxy_packets" or "full_packets", teach the packet code to give us the full packets, and then just treat that whole buffer as a unit. I dunno. There might be some gotchas in practice, and it's not like it's that much simpler. Just a thought.

Yes, the length is the length we just read. And it might not even be simpler - this shifts the complexity to the code that does not use the 4 characters (unless we always return 2 pointers, which seems redundant, and which now changes the problem of ensuring that the correct pointer is used). I think that this is a good default - proxying still seems rarer than just consuming payloads.

I'm OK with changing the name, although I think that both "proxy" and "full" are less clear than "write_line_lengths" - aren't you still proxying even if you're changing the format a little, and isn't a packet "full" even without the line lengths?

Quoted 9 lines

remote-curl.c struct rpc_state { · line 518

Quoted 5 lines
remote-curl.c +5 -0
+	/*
+	 * rpc_out uses this to keep track of whether it should continue
+	 * reading to populate the current request. Initialize to 0.
+	 */
+	unsigned stop_reading : 1;

OK, so we need this because the v2 proxying will require us to stop reading but keep the channel open? Kind of awkward, but I don't see a way around it.

I'll improve the comments here and elsewhere in the next version. This basically means that "we read a flush but we haven't sent the 0000 to libcurl yet, so don't read anything more until we have sent the 0000", and "flush_read_but_not_sent" is probably a better name.

Quoted 6 lines
Quoted 6 lines
remote-curl.c +1 -0
@@ -531,15 +580,32 @@ static size_t rpc_out(void *ptr, size_t eltsize,
 	size_t max = eltsize * nmemb;
 	struct rpc_state *rpc = buffer_;
 	size_t avail = rpc->len - rpc->pos;
+	enum packet_read_status status;
 
Quoted 15 lines

remote-curl.c static size_t rpc_out(void *ptr, size_t eltsize, · line 591

Quoted 11 lines
remote-curl.c +3 -4
 	if (!avail) {
 		rpc->initial_buffer = 0;
 		rpc->len = 0;
-		if (!rpc_read_from_out(rpc, &avail))
-			BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");
-		if (!avail)
-			return 0;
 		rpc->pos = 0;
+		if (!rpc->stop_reading) {
+			if (!rpc_read_from_out(rpc, 0, &avail, &status))
+				BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");

Do we actually need it to be LARGE_PACKET_MAX+4 here? I guess not, because LARGE_PACKET_DATA_MAX is the "-4" version. So I think this BUG() was perhaps already wrong?

In this patch, yes (if not, the non-chunked code is useless). Previously, the BUG should have been LARGE_PACKET_DATA_MAX, yes. The BUG() was introduced in patch 4 of this set - I'll update that one to LARGE_PACKET_DATA_MAX and this one to LARGE_PACKET_MAX.

Quoted 21 lines
Quoted 7 lines
remote-curl.c +7 -0
+			if (status == PACKET_READ_FLUSH)
+				/*
+				 * We are done reading for this request, but we
+				 * still need to send this line out (if
+				 * rpc->write_line_lengths is true) so do not
+				 * return yet.
+				 */

remote-curl.c static size_t rpc_out(void *ptr, size_t eltsize, · line 608

Quoted 10 lines
remote-curl.c +10 -0
+				rpc->stop_reading = 1;
+		}
+	}
+	if (!avail && rpc->stop_reading) {
+		/*
+		 * "return 0" will notify Curl that this RPC request is done,
+		 * so reset stop_reading back to 0 for the next request.
+		 */
+		rpc->stop_reading = 0;
+		return 0;

OK, and here's where we handle the stop_reading thing. It is indeed awkward, but I think your comments make it clear what's going on.

Quoted 3 lines

If we get stop_reading, do we care about "avail"? I.e., shouldn't we be able to return non-zero to say "we got the whole input, this is not a too-large request"?

This code is in rpc_out(), which is a callback passed as CURLOPT_READFUNCTION. So returning non-zero means "send these bytes", and returning zero means EOF.

We set stop_reading when we receive a flush, as you can see from the quoted code snippet. But this does not mean that the buffer is empty - the buffer may contain "0000" (if rpc->write_line_lengths is true, as the comment states). We have to let libcurl repeatedly call this function until all of the "0000" is sent (and return non-zero each time). But once all of the "0000" is sent - we know this by avail being zero - and libcurl calls this function once more, we have to remember to do nothing except to reset stop_reading and return 0 to indicate EOF.

Quoted 14 lines

remote-curl.c static size_t rpc_out(void *ptr, size_t eltsize,

Quoted 9 lines
remote-curl.c +9 -0
+test_expect_success 'clone big repository with http:// using protocol v2' '
+	test_when_finished "rm -f log" &&
+
+	git init "$HTTPD_DOCUMENT_ROOT_PATH/big" &&
+	# Ensure that the list of wants is greater than http.postbuffer below
+	for i in $(seq 1 1500)
+	do
+		test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/big" "commit$i"
+	done &&

As Junio noted, this should be test_seq. But I think it would be nice to avoid looping on test_commit here at all. It kicks off at least 3 processes; multiplying that by 1500 is going to be slow.

Quoted 3 lines

Making a big input is often much faster by generating a fast-import stream (which can often be done entirely in-shell). There's some prior art in t3302, t5551, t5608, and others.

OK - I'll look at generating a fast-import stream. Thanks for all your comments.

#10 lore raw

v2 cover Protocol v2 fix: http and auth

In reply to Jonathan Tan

Thanks to Junio and Peff for their reviews.

The main changes are the name and comments around the stop_reading part. I have renamed it to flush_read_but_not_sent and have added a lot of comments - I tried to err on the side of overcommenting here, since it is rather complicated.

Other changes:
- used test_seq instead of seq
- used fast_import instead of test_commit 1500 times
- in patch 4, the BUG statement now reads "...larger than LARGE_PACKET_DATA_MAX; the corresponding BUG in patch 5 is LARGE_PACKET_MAX (this means that the total diff does not change, and is thus not visible in the interdiff below)

I discovered a slight issue in which the full http.postBuffer is not used, because Git immediately switches to chunked mode if the buffer cannot contain a maximally sized pkt-line (without first reading the pkt-line to see if it fits). This means that I could replace "test-seq 1 1500" with "test-seq 1 2" and the test would still pass, but I'm still using 1500 in this patch set so that the test will pass if we ever decide to use the http.postBuffer slightly more efficiently.

Jonathan Tan (5):
remote-curl: reduce scope of rpc_state.argv
remote-curl: reduce scope of rpc_state.stdin_preamble
remote-curl: reduce scope of rpc_state.result
remote-curl: refactor reading into rpc_state's buf
remote-curl: use post_rpc() for protocol v2 also

 pkt-line.c             |   2 +-
 pkt-line.h             |   1 +
 remote-curl.c          | 362 +++++++++++++++++++----------------------
 t/t5702-protocol-v2.sh |  33 +++-
 4 files changed, 199 insertions(+), 199 deletions(-)

Interdiff against v1:

remote-curl.c +28 -17
index 2394a00650..8c03c78fc6 100644
@@ -526,10 +526,13 @@ struct rpc_state {
 	unsigned write_line_lengths : 1;
 
 	/*
-	 * rpc_out uses this to keep track of whether it should continue
-	 * reading to populate the current request. Initialize to 0.
+	 * Used by rpc_out; initialize to 0. This is true if a flush has been
+	 * read, but the corresponding line length (if write_line_lengths is
+	 * true) and EOF have not been sent to libcurl. Since each flush marks
+	 * the end of a request, each flush must be completely sent before any
+	 * further reading occurs.
 	 */
-	unsigned stop_reading : 1;
+	unsigned flush_read_but_not_sent : 1;
 };
 
 /*
@@ -600,26 +603,34 @@ static size_t rpc_out(void *ptr, size_t eltsize,
 		rpc->initial_buffer = 0;
 		rpc->len = 0;
 		rpc->pos = 0;
-		if (!rpc->stop_reading) {
+		if (!rpc->flush_read_but_not_sent) {
 			if (!rpc_read_from_out(rpc, 0, &avail, &status))
 				BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");
 			if (status == PACKET_READ_FLUSH)
-				/*
-				 * We are done reading for this request, but we
-				 * still need to send this line out (if
-				 * rpc->write_line_lengths is true) so do not
-				 * return yet.
-				 */
-				rpc->stop_reading = 1;
+				rpc->flush_read_but_not_sent = 1;
 		}
+		/*
+		 * If flush_read_but_not_sent is true, we have already read one
+		 * full request but have not fully sent it + EOF, which is why
+		 * we need to refrain from reading.
+		 */
 	}
-	if (!avail && rpc->stop_reading) {
+	if (rpc->flush_read_but_not_sent) {
+		if (!avail) {
+			/*
+			 * The line length either does not need to be sent at
+			 * all or has already been completely sent. Now we can
+			 * return 0, indicating EOF, meaning that the flush has
+			 * been fully sent.
+			 */
+			rpc->flush_read_but_not_sent = 0;
+			return 0;
+		}
 		/*
-		 * "return 0" will notify Curl that this RPC request is done,
-		 * so reset stop_reading back to 0 for the next request.
+		 * If avail is non-zerp, the line length for the flush still
+		 * hasn't been fully sent. Proceed with sending the line
+		 * length.
 		 */
-		rpc->stop_reading = 0;
-		return 0;
 	}
 
 	if (max < avail)
@@ -1290,7 +1301,7 @@ static int stateless_connect(const char *service_name)
 	rpc.gzip_request = 1;
 	rpc.initial_buffer = 0;
 	rpc.write_line_lengths = 1;
-	rpc.stop_reading = 0;
+	rpc.flush_read_but_not_sent = 0;
 
 	/*
 	 * Dump the capability listing that we got from the server earlier
t/t5702-protocol-v2.sh +10 -3
index 61acf99d80..e112b6086c 100755
@@ -552,10 +552,17 @@ test_expect_success 'clone big repository with http:// using protocol v2' '
 
 	git init "$HTTPD_DOCUMENT_ROOT_PATH/big" &&
 	# Ensure that the list of wants is greater than http.postbuffer below
-	for i in $(seq 1 1500)
+	for i in $(test_seq 1 1500)
 	do
-		test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/big" "commit$i"
-	done &&
+		# do not use here-doc, because it requires a process
+		# per loop iteration
+		echo "commit refs/heads/too-many-refs-$i" &&
+		echo "committer git <[email protected]> $i +0000" &&
+		echo "data 0" &&
+		echo "M 644 inline bla.txt" &&
+		echo "data 4" &&
+		echo "bla"
+	done | git -C "$HTTPD_DOCUMENT_ROOT_PATH/big" fast-import &&
 
 	GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git \
 		-c protocol.version=2 -c http.postbuffer=65536 \
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#11 lore raw

PATCH v2 1/5 remote-curl: reduce scope of rpc_state.argv

The argv field in struct rpc_state is only used in rpc_service(), and not in any functions it directly or indirectly calls. Refactor it to become an argument of rpc_service() instead.

---
 remote-curl.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
remote-curl.c +5 -7
index bb7421023b..8b7baf6298 100644
@@ -505,7 +505,6 @@ static void output_refs(struct ref *refs)
 
 struct rpc_state {
 	const char *service_name;
-	const char **argv;
 	struct strbuf *stdin_preamble;
 	char *service_url;
 	char *hdr_content_type;
@@ -829,7 +828,8 @@ static int post_rpc(struct rpc_state *rpc)
 	return err;
 }
 
-static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
+static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
+		       const char **client_argv)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
@@ -840,7 +840,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
 	client.in = -1;
 	client.out = -1;
 	client.git_cmd = 1;
-	client.argv = rpc->argv;
+	client.argv = client_argv;
 	if (start_command(&client))
 		exit(1);
 	if (preamble)
@@ -978,11 +978,10 @@ static int fetch_git(struct discovery *heads,
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-upload-pack",
-	rpc.argv = args.argv;
 	rpc.stdin_preamble = &preamble;
 	rpc.gzip_request = 1;
 
-	err = rpc_service(&rpc, heads);
+	err = rpc_service(&rpc, heads, args.argv);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
@@ -1112,10 +1111,9 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-receive-pack",
-	rpc.argv = args.argv;
 	rpc.stdin_preamble = &preamble;
 
-	err = rpc_service(&rpc, heads);
+	err = rpc_service(&rpc, heads, args.argv);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#12 lore raw

PATCH v2 2/5 remote-curl: reduce scope of rpc_state.stdin_preamble

In reply to Jonathan Tan

The stdin_preamble field in struct rpc_state is only used in rpc_service(), and not in any functions it directly or indirectly calls. Refactor it to become an argument of rpc_service() instead.

An observation of all callers of rpc_service() shows that the preamble is always set, so we no longer need the "if (preamble)" check.

---
 remote-curl.c | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)
remote-curl.c +4 -9
index 8b7baf6298..d33d5bbfa8 100644
@@ -505,7 +505,6 @@ static void output_refs(struct ref *refs)
 
 struct rpc_state {
 	const char *service_name;
-	struct strbuf *stdin_preamble;
 	char *service_url;
 	char *hdr_content_type;
 	char *hdr_accept;
@@ -829,11 +828,10 @@ static int post_rpc(struct rpc_state *rpc)
 }
 
 static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
-		       const char **client_argv)
+		       const char **client_argv, const struct strbuf *preamble)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
-	struct strbuf *preamble = rpc->stdin_preamble;
 	struct child_process client = CHILD_PROCESS_INIT;
 	int err = 0;
 
@@ -843,8 +841,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 	client.argv = client_argv;
 	if (start_command(&client))
 		exit(1);
-	if (preamble)
-		write_or_die(client.in, preamble->buf, preamble->len);
+	write_or_die(client.in, preamble->buf, preamble->len);
 	if (heads)
 		write_or_die(client.in, heads->buf, heads->len);
 
@@ -978,10 +975,9 @@ static int fetch_git(struct discovery *heads,
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-upload-pack",
-	rpc.stdin_preamble = &preamble;
 	rpc.gzip_request = 1;
 
-	err = rpc_service(&rpc, heads, args.argv);
+	err = rpc_service(&rpc, heads, args.argv, &preamble);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
@@ -1111,9 +1107,8 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-receive-pack",
-	rpc.stdin_preamble = &preamble;
 
-	err = rpc_service(&rpc, heads, args.argv);
+	err = rpc_service(&rpc, heads, args.argv, &preamble);
 	if (rpc.result.len)
 		write_or_die(1, rpc.result.buf, rpc.result.len);
 	strbuf_release(&rpc.result);
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#13 lore raw

PATCH v2 3/5 remote-curl: reduce scope of rpc_state.result

In reply to Jonathan Tan

The result field in struct rpc_state is only used in rpc_service(), and not in any functions it directly or indirectly calls. Refactor it to become an argument of rpc_service() instead.

---
 remote-curl.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)
remote-curl.c +13 -12
index d33d5bbfa8..8e0e37ed3d 100644
@@ -516,7 +516,6 @@ struct rpc_state {
 	int in;
 	int out;
 	int any_written;
-	struct strbuf result;
 	unsigned gzip_request : 1;
 	unsigned initial_buffer : 1;
 };
@@ -828,7 +827,8 @@ static int post_rpc(struct rpc_state *rpc)
 }
 
 static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
-		       const char **client_argv, const struct strbuf *preamble)
+		       const char **client_argv, const struct strbuf *preamble,
+		       struct strbuf *rpc_result)
 {
 	const char *svc = rpc->service_name;
 	struct strbuf buf = STRBUF_INIT;
@@ -849,7 +849,6 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 	rpc->buf = xmalloc(rpc->alloc);
 	rpc->in = client.in;
 	rpc->out = client.out;
-	strbuf_init(&rpc->result, 0);
 
 	strbuf_addf(&buf, "%s%s", url.buf, svc);
 	rpc->service_url = strbuf_detach(&buf, NULL);
@@ -877,7 +876,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 	close(client.in);
 	client.in = -1;
 	if (!err) {
-		strbuf_read(&rpc->result, client.out, 0);
+		strbuf_read(rpc_result, client.out, 0);
 	} else {
 		char buf[4096];
 		for (;;)
@@ -930,6 +929,7 @@ static int fetch_git(struct discovery *heads,
 	struct strbuf preamble = STRBUF_INIT;
 	int i, err;
 	struct argv_array args = ARGV_ARRAY_INIT;
+	struct strbuf rpc_result = STRBUF_INIT;
 
 	argv_array_pushl(&args, "fetch-pack", "--stateless-rpc",
 			 "--stdin", "--lock-pack", NULL);
@@ -977,10 +977,10 @@ static int fetch_git(struct discovery *heads,
 	rpc.service_name = "git-upload-pack",
 	rpc.gzip_request = 1;
 
-	err = rpc_service(&rpc, heads, args.argv, &preamble);
-	if (rpc.result.len)
-		write_or_die(1, rpc.result.buf, rpc.result.len);
-	strbuf_release(&rpc.result);
+	err = rpc_service(&rpc, heads, args.argv, &preamble, &rpc_result);
+	if (rpc_result.len)
+		write_or_die(1, rpc_result.buf, rpc_result.len);
+	strbuf_release(&rpc_result);
 	strbuf_release(&preamble);
 	argv_array_clear(&args);
 	return err;
@@ -1075,6 +1075,7 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 	struct argv_array args;
 	struct string_list_item *cas_option;
 	struct strbuf preamble = STRBUF_INIT;
+	struct strbuf rpc_result = STRBUF_INIT;
 
 	argv_array_init(&args);
 	argv_array_pushl(&args, "send-pack", "--stateless-rpc", "--helper-status",
@@ -1108,10 +1109,10 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
 	memset(&rpc, 0, sizeof(rpc));
 	rpc.service_name = "git-receive-pack",
 
-	err = rpc_service(&rpc, heads, args.argv, &preamble);
-	if (rpc.result.len)
-		write_or_die(1, rpc.result.buf, rpc.result.len);
-	strbuf_release(&rpc.result);
+	err = rpc_service(&rpc, heads, args.argv, &preamble, &rpc_result);
+	if (rpc_result.len)
+		write_or_die(1, rpc_result.buf, rpc_result.len);
+	strbuf_release(&rpc_result);
 	strbuf_release(&preamble);
 	argv_array_clear(&args);
 	return err;
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#14 lore raw

PATCH v2 4/5 remote-curl: refactor reading into rpc_state's buf

In reply to Jonathan Tan

Currently, whenever remote-curl reads pkt-lines from its response file descriptor, only the payload is written to its buf, not the 4 characters denoting the length. A future patch will require the ability to also write those 4 characters, so in preparation for that, refactor this read into its own function.

---
 remote-curl.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)
remote-curl.c +24 -9
index 8e0e37ed3d..1f0161475d 100644
@@ -520,6 +520,25 @@ struct rpc_state {
 	unsigned initial_buffer : 1;
 };
 
+/*
+ * Appends the result of reading from rpc->out to the string represented by
+ * rpc->buf and rpc->len if there is enough space. Returns 1 if there was
+ * enough space, 0 otherwise.
+ *
+ * Writes the number of bytes appended into appended.
+ */
+static int rpc_read_from_out(struct rpc_state *rpc, size_t *appended) {
+	size_t left = rpc->alloc - rpc->len;
+	char *buf = rpc->buf + rpc->len;
+
+	if (left < LARGE_PACKET_MAX)
+		return 0;
+
+	*appended = packet_read(rpc->out, NULL, NULL, buf, left, 0);
+	rpc->len += *appended;
+	return 1;
+}
+
 static size_t rpc_out(void *ptr, size_t eltsize,
 		size_t nmemb, void *buffer_)
 {
@@ -529,11 +548,12 @@ static size_t rpc_out(void *ptr, size_t eltsize,
 
 	if (!avail) {
 		rpc->initial_buffer = 0;
-		avail = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0);
+		rpc->len = 0;
+		if (!rpc_read_from_out(rpc, &avail))
+			BUG("The entire rpc->buf should be larger than LARGE_PACKET_DATA_MAX");
 		if (!avail)
 			return 0;
 		rpc->pos = 0;
-		rpc->len = avail;
 	}
 
 	if (max < avail)
@@ -677,20 +697,15 @@ static int post_rpc(struct rpc_state *rpc)
 	 * chunked encoding mess.
 	 */
 	while (1) {
-		size_t left = rpc->alloc - rpc->len;
-		char *buf = rpc->buf + rpc->len;
-		int n;
+		size_t n;
 
-		if (left < LARGE_PACKET_MAX) {
+		if (!rpc_read_from_out(rpc, &n)) {
 			large_request = 1;
 			use_gzip = 0;
 			break;
 		}
-
-		n = packet_read(rpc->out, NULL, NULL, buf, left, 0);
 		if (!n)
 			break;
-		rpc->len += n;
 	}
 
 	if (large_request) {
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#15 lore raw

PATCH v2 5/5 remote-curl: use post_rpc() for protocol v2 also

In reply to Jonathan Tan

When transmitting and receiving POSTs for protocol v0 and v1, remote-curl uses post_rpc() (and associated functions), but when doing the same for protocol v2, it uses a separate set of functions (proxy_rpc() and others). Besides duplication of code, this has caused at least one bug: the auth retry mechanism that was implemented in v0/v1 was not implemented in v2.

To fix this issue and avoid it in the future, make remote-curl also use post_rpc() when handling protocol v2. Because line lengths are written to the HTTP request in protocol v2 (unlike in protocol v0/v1), this necessitates changes in post_rpc() and some of the functions it uses; perform these changes too.

A test has been included to ensure that the code for both the unchunked and chunked variants of the HTTP request is exercised.

Note: stateless_connect() has been updated to use the lower-level packet reading functions instead of struct packet_reader. The low-level control is necessary here because we cannot change the destination buffer of struct packet_reader while it is being used; struct packet_buffer has a peeking mechanism which relies on the destination buffer being present in between a peek and a read.

---
 pkt-line.c             |   2 +-
 pkt-line.h             |   1 +
 remote-curl.c          | 309 ++++++++++++++++++-----------------------
 t/t5702-protocol-v2.sh |  33 ++++-
 4 files changed, 168 insertions(+), 177 deletions(-)
pkt-line.c +1 -1
index d4b71d3e82..60329b301b 100644
@@ -117,7 +117,7 @@ void packet_buf_delim(struct strbuf *buf)
 	strbuf_add(buf, "0001", 4);
 }
 
-static void set_packet_header(char *buf, const int size)
+void set_packet_header(char *buf, const int size)
 {
 	static char hexchar[] = "0123456789abcdef";
 
pkt-line.h +1 -0
index ad9a4a2cd7..c36cb788ed 100644
@@ -25,6 +25,7 @@ void packet_delim(int fd);
 void packet_write_fmt(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
 void packet_buf_flush(struct strbuf *buf);
 void packet_buf_delim(struct strbuf *buf);
+void set_packet_header(char *buf, int size);
 void packet_write(int fd_out, const char *buf, size_t size);
 void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
 void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len);
remote-curl.c +134 -175

373 lines not on this page · raw message

t/t5702-protocol-v2.sh +32 -1
index db4ae09f2f..e112b6086c 100755
@@ -542,7 +542,38 @@ test_expect_success 'clone with http:// using protocol v2' '
 	# Client requested to use protocol v2
 	grep "Git-Protocol: version=2" log &&
 	# Server responded using protocol v2
-	grep "git< version 2" log
+	grep "git< version 2" log &&
+	# Verify that the chunked encoding sending codepath is NOT exercised
+	! grep "Send header: Transfer-Encoding: chunked" log
+'
+
+test_expect_success 'clone big repository with http:// using protocol v2' '
+	test_when_finished "rm -f log" &&
+
+	git init "$HTTPD_DOCUMENT_ROOT_PATH/big" &&
+	# Ensure that the list of wants is greater than http.postbuffer below
+	for i in $(test_seq 1 1500)
+	do
+		# do not use here-doc, because it requires a process
+		# per loop iteration
+		echo "commit refs/heads/too-many-refs-$i" &&
+		echo "committer git <[email protected]> $i +0000" &&
+		echo "data 0" &&
+		echo "M 644 inline bla.txt" &&
+		echo "data 4" &&
+		echo "bla"
+	done | git -C "$HTTPD_DOCUMENT_ROOT_PATH/big" fast-import &&
+
+	GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git \
+		-c protocol.version=2 -c http.postbuffer=65536 \
+		clone "$HTTPD_URL/smart/big" big_child &&
+
+	# Client requested to use protocol v2
+	grep "Git-Protocol: version=2" log &&
+	# Server responded using protocol v2
+	grep "git< version 2" log &&
+	# Verify that the chunked encoding sending codepath is exercised
+	grep "Send header: Transfer-Encoding: chunked" log
 '
 
 test_expect_success 'fetch with http:// using protocol v2' '
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
#16 lore raw

Re: [PATCH v2 5/5] remote-curl: use post_rpc() for protocol v2 also

Quoted 8 lines · Jonathan Tan
t/t5702-protocol-v2.sh +6 -0
@@ -542,7 +542,38 @@ test_expect_success 'clone with http:// using protocol v2' '
+test_expect_success 'clone big repository with http:// using protocol v2' '
+       test_when_finished "rm -f log" &&
+
+       git init "$HTTPD_DOCUMENT_ROOT_PATH/big" &&
+       # Ensure that the list of wants is greater than http.postbuffer below
+       for i in $(test_seq 1 1500)

t/t5702-protocol-v2.sh test_expect_success 'clone with http:// using protocol v2' ' · line 557

Quoted 10 lines
t/t5702-protocol-v2.sh +10 -0
+       do
+               # do not use here-doc, because it requires a process
+               # per loop iteration
+               echo "commit refs/heads/too-many-refs-$i" &&
+               echo "committer git <[email protected]> $i +0000" &&
+               echo "data 0" &&
+               echo "M 644 inline bla.txt" &&
+               echo "data 4" &&
+               echo "bla"
+       done | git -C "$HTTPD_DOCUMENT_ROOT_PATH/big" fast-import &&

Although not outright wrong, maintaining an &&-chain in the loop body is a bit misleading here for a couple reasons. First, without "|| exit 1" at the end of the body, neither the loop nor the test itself will abort early if some command within the body fails. Second, the exit status of the loop is lost anyhow due to being upstream of a pipe.

By the way, I see that this same script already contains a couple tests with loops neglecting the trailing "|| exit 1", hence they too fail to abort early if something goes wrong inside the body (which, in both cases, contains a test_commit() invocation). Fixing those might be fodder as a follow-up patch for someone.

#17 lore raw

Re: [PATCH v2 5/5] remote-curl: use post_rpc() for protocol v2 also

Quoted 13 lines · Eric Sunshine

t/t5702-protocol-v2.sh test_expect_success 'clone with http:// using protocol v2' ' · line 565

Quoted 11 lines · Jonathan Tan
t/t5702-protocol-v2.sh +11 -0
+       for i in $(test_seq 1 1500)
+       do
+               # do not use here-doc, because it requires a process
+               # per loop iteration
+               echo "commit refs/heads/too-many-refs-$i" &&
+               echo "committer git <[email protected]> $i +0000" &&
+               echo "data 0" &&
+               echo "M 644 inline bla.txt" &&
+               echo "data 4" &&
+               echo "bla"
+       done | git -C "$HTTPD_DOCUMENT_ROOT_PATH/big" fast-import &&
Quoted 4 lines

Although not outright wrong, maintaining an &&-chain in the loop body is a bit misleading here for a couple reasons. First, without "|| exit 1" at the end of the body, neither the loop nor the test itself will abort early if some command within the body fails. [...]

I meant "|| return 1", not "|| exit 1".

Quoted 3 lines

By the way, I see that this same script already contains a couple tests with loops neglecting the trailing "|| exit 1", hence they too fail to abort early if something goes wrong inside the body [...]

Ditto.

#18 lore raw

Re: [PATCH v2 5/5] remote-curl: use post_rpc() for protocol v2 also

In reply to Jonathan Tan
Quoted 7 lines · Jonathan Tan

When transmitting and receiving POSTs for protocol v0 and v1, remote-curl uses post_rpc() (and associated functions), but when doing the same for protocol v2, it uses a separate set of functions (proxy_rpc() and others). Besides duplication of code, this has caused at least one bug: the auth retry mechanism that was implemented in v0/v1 was not implemented in v2.

Quoted 5 lines

To fix this issue and avoid it in the future, make remote-curl also use post_rpc() when handling protocol v2. Because line lengths are written to the HTTP request in protocol v2 (unlike in protocol v0/v1), this necessitates changes in post_rpc() and some of the functions it uses; perform these changes too.

This patch seems to cause noop fetches from GitHub to report "the remote end hung up unexpectedly" at the end of the fetch (I have protocol.version=2 set in the examples below):

[parent is good]

  $ git checkout 0cdb2a12ad0300b5d0cb5bb6e8999034ae4b9bef^
  $ make
  $ bin-wrappers/git fetch https://github.com/git/git
  From https://github.com/git/git
   * branch                  HEAD       -> FETCH_HEAD

[this patch is bad]

  $ git checkout 0cdb2a12ad0300b5d0cb5bb6e8999034ae4b9bef
  $ make
  $ bin-wrappers/git fetch https://github.com/git/git
  From https://github.com/git/git
   * branch                  HEAD       -> FETCH_HEAD
  fatal: the remote end hung up unexpectedly

I haven't dug, so it's possible that GitHub's server-side is doing something slightly odd or wrong. But given the nature of the patch, I'd suspect we just aren't noticing EOF correctly.

-Peff

#19 lore raw

[FIXUP] Fixup on tip of jt/http-auth-proto-v2-fix

Thanks, Peff, for noticing this. It's because the client sometimes sends "0000" as a single request (that is, it flushes, and then before it sends any data, it flushes again). And post_rpc() assumes that it can always read something - which is usually correct, but not in this case; we read in stateless_connect() first, and if we read "0000", we need to tell post_rpc() to not read at all.

This is a fixup on the tip of jt/http-auth-proto-v2-fix that fixes that.

As for why the client sends a lone "0000", I'm not sure, but that's outside the scope of this patch set, I think.

---
 remote-curl.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)
remote-curl.c +19 -13
index 2394a00650..6c0207f4f9 100644
@@ -747,7 +747,11 @@ static curl_off_t xcurl_off_t(size_t len)
 	return (curl_off_t)size;
 }
 
-static int post_rpc(struct rpc_state *rpc)
+/*
+ * If flush_received is true, do not attempt to read any more; just use what's
+ * in rpc->buf.
+ */
+static int post_rpc(struct rpc_state *rpc, int flush_received)
 {
 	struct active_request_slot *slot;
 	struct curl_slist *headers = http_copy_default_headers();
@@ -762,17 +766,19 @@ static int post_rpc(struct rpc_state *rpc)
 	 * allocated buffer space we can use HTTP/1.0 and avoid the
 	 * chunked encoding mess.
 	 */
-	while (1) {
-		size_t n;
-		enum packet_read_status status;
-
-		if (!rpc_read_from_out(rpc, 0, &n, &status)) {
-			large_request = 1;
-			use_gzip = 0;
-			break;
+	if (!flush_received) {
+		while (1) {
+			size_t n;
+			enum packet_read_status status;
+
+			if (!rpc_read_from_out(rpc, 0, &n, &status)) {
+				large_request = 1;
+				use_gzip = 0;
+				break;
+			}
+			if (status == PACKET_READ_FLUSH)
+				break;
 		}
-		if (status == PACKET_READ_FLUSH)
-			break;
 	}
 
 	if (large_request) {
@@ -952,7 +958,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 			break;
 		rpc->pos = 0;
 		rpc->len = n;
-		err |= post_rpc(rpc);
+		err |= post_rpc(rpc, 0);
 	}
 
 	close(client.in);
@@ -1308,7 +1314,7 @@ static int stateless_connect(const char *service_name)
 			BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");
 		if (status == PACKET_READ_EOF)
 			break;
-		if (post_rpc(&rpc))
+		if (post_rpc(&rpc, status == PACKET_READ_FLUSH))
 			/* We would have an err here */
 			break;
 		/* Reset the buffer for next request */
Signature · 1 line
2.19.0.271.gfe8321ec05.dirty
Tested-by
#20 lore raw

Re: [FIXUP] Fixup on tip of jt/http-auth-proto-v2-fix

Hi,

Quoted 9 lines · Jonathan Tan

Thanks, Peff, for noticing this. It's because the client sometimes sends "0000" as a single request (that is, it flushes, and then before it sends any data, it flushes again). And post_rpc() assumes that it can always read something - which is usually correct, but not in this case; we read in stateless_connect() first, and if we read "0000", we need to tell post_rpc() to not read at all.

This is a fixup on the tip of jt/http-auth-proto-v2-fix that fixes that.

Quoted 7 lines

As for why the client sends a lone "0000", I'm not sure, but that's outside the scope of this patch set, I think.

---
 remote-curl.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

Thanks for fixing it.

Is there a particular patch this should be squashed into, or does it stand alone? It the latter, mind writing a commit message for it?

Thanks,
Jonathan

#21 lore raw

Re: [FIXUP] Fixup on tip of jt/http-auth-proto-v2-fix

Quoted 4 lines

Thanks for fixing it.

Is there a particular patch this should be squashed into, or does it stand alone? It the latter, mind writing a commit message for it?

Not sure if I'm using "fixup" correctly in the subject, but this is meant to be squashed onto the tip of jt/http-auth-proto-v2-fix - specifically, deb7d2094a ("remote-curl: use post_rpc() for protocol v2 also", 2019-02-14).

I don't think it should be a standalone commit, as the tip is buggy - which might break bisect. But if we really want a standalone commit, this commit message should work:

remote-curl: handle consecutive flushes correctly

When the client, using protocol v2, sends two consecutive flushes and then an EOF, remote-curl dies. This is because, at the start of a new request, stateless_connect() reads, and if no EOF is found, then stateless_connect() calls post_rpc() which reads until a flush is encountered. This is a problem when stateless_connect() reads the second consecutive flush (hence, no EOF), and then post_rpc() reads, not expecting an EOF at all.

Teach stateless_connect() to inform post_rpc() to read only if what it read isn't a flush.

#23 lore raw

Re: [FIXUP] Fixup on tip of jt/http-auth-proto-v2-fix

In reply to Jonathan Tan
Quoted 8 lines · Jonathan Tan

Thanks, Peff, for noticing this. It's because the client sometimes sends "0000" as a single request (that is, it flushes, and then before it sends any data, it flushes again). And post_rpc() assumes that it can always read something - which is usually correct, but not in this case; we read in stateless_connect() first, and if we read "0000", we need to tell post_rpc() to not read at all.

This is a fixup on the tip of jt/http-auth-proto-v2-fix that fixes that.

Thanks, I can confirm that this makes the problem go away (and your explanation makes perfect sense to me).

Quoted 2 lines

As for why the client sends a lone "0000", I'm not sure, but that's outside the scope of this patch set, I think.

Yeah, that does seem odd. I noticed it on noop fetches. So after we've done "ls-refs", would "fetch" need to send a flush to say "I don't want anything?" I guess not, since we're stateless, and it is literally making a new HTTP request just to say nothing.

It does seem unique to protocol v2.

-Peff

#22 lore raw

Re: [FIXUP] Fixup on tip of jt/http-auth-proto-v2-fix

In reply to Jonathan Tan
Quoted 9 lines · Jonathan Tan
Quoted 4 lines

Thanks for fixing it.

Is there a particular patch this should be squashed into, or does it stand alone? It the latter, mind writing a commit message for it?

Not sure if I'm using "fixup" correctly in the subject, but this is meant to be squashed onto the tip of jt/http-auth-proto-v2-fix - specifically, deb7d2094a ("remote-curl: use post_rpc() for protocol v2 also", 2019-02-14).

Saying "Fixup on tip of" to mean that you are fixing the 5th one in a 5-patch series is sufficient to convey what you want between humans, but that does not help the machine to help humans reduce mistakes. It would have been even more helpful if you created this commit with

	git commit --fixup 0cdb2a12ad

which would have produced something like the attached patch, and that would have allowed me to do

git checkout jt/http-auth-proto-v2-fix
git am ./+jt-http-auth-proto-v2-fix-fixup
git rebase -i --autosquash HEAD~5

or somesuch.

Thanks, will queue.

-- >8 --
Subject: [PATCH] fixup! remote-curl: use post_rpc() for protocol v2 also

---
 remote-curl.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)
remote-curl.c +19 -13
index 8c03c78fc6..3c3872cca6 100644
@@ -758,7 +758,11 @@ static curl_off_t xcurl_off_t(size_t len)
 	return (curl_off_t)size;
 }
 
-static int post_rpc(struct rpc_state *rpc)
+/*
+ * If flush_received is true, do not attempt to read any more; just use what's
+ * in rpc->buf.
+ */
+static int post_rpc(struct rpc_state *rpc, int flush_received)
 {
 	struct active_request_slot *slot;
 	struct curl_slist *headers = http_copy_default_headers();
@@ -773,17 +777,19 @@ static int post_rpc(struct rpc_state *rpc)
 	 * allocated buffer space we can use HTTP/1.0 and avoid the
 	 * chunked encoding mess.
 	 */
-	while (1) {
-		size_t n;
-		enum packet_read_status status;
-
-		if (!rpc_read_from_out(rpc, 0, &n, &status)) {
-			large_request = 1;
-			use_gzip = 0;
-			break;
+	if (!flush_received) {
+		while (1) {
+			size_t n;
+			enum packet_read_status status;
+
+			if (!rpc_read_from_out(rpc, 0, &n, &status)) {
+				large_request = 1;
+				use_gzip = 0;
+				break;
+			}
+			if (status == PACKET_READ_FLUSH)
+				break;
 		}
-		if (status == PACKET_READ_FLUSH)
-			break;
 	}
 
 	if (large_request) {
@@ -963,7 +969,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads,
 			break;
 		rpc->pos = 0;
 		rpc->len = n;
-		err |= post_rpc(rpc);
+		err |= post_rpc(rpc, 0);
 	}
 
 	close(client.in);
@@ -1319,7 +1325,7 @@ static int stateless_connect(const char *service_name)
 			BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX");
 		if (status == PACKET_READ_EOF)
 			break;
-		if (post_rpc(&rpc))
+		if (post_rpc(&rpc, status == PACKET_READ_FLUSH))
 			/* We would have an err here */
 			break;
 		/* Reset the buffer for next request */
Signature · 1 line
2.21.0