Implementation of fetch-blobs and fetch-refs

228 lines not on this page · raw message

index d07d85ce3..74771a283 100644
@@ -314,29 +314,14 @@ static int next_flush(struct fetch_pack_args *args, int count)
 	return count;
 }
 
-static int find_common(struct fetch_pack_args *args,
-		       int fd[2], unsigned char *result_sha1,
-		       struct ref *refs)
+/*
+ * Generate the "want" lines to be sent. If there are no "want" lines to be
+ * sent, return NULL.
+ */
+static char *get_wants(const struct fetch_pack_args *args, struct ref *refs)
 {
-	int fetching;
-	int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
-	const unsigned char *sha1;
-	unsigned in_vain = 0;
-	int got_continue = 0;
-	int got_ready = 0;
 	struct strbuf req_buf = STRBUF_INIT;
-	size_t state_len = 0;
 
-	if (args->stateless_rpc && multi_ack == 1)
-		die(_("--stateless-rpc requires multi_ack_detailed"));
-	if (marked)
-		for_each_ref(clear_marks, NULL);
-	marked = 1;
-
-	for_each_ref(rev_list_insert_ref_oid, NULL);
-	for_each_cached_alternate(insert_one_alternate_object);
-
-	fetching = 0;
 	for ( ; refs ; refs = refs->next) {
 		unsigned char *remote = refs->old_oid.hash;
 		const char *remote_hex;

@@ -358,7 +343,7 @@ static int find_common(struct fetch_pack_args *args,
 		}
 
 		remote_hex = sha1_to_hex(remote);
-		if (!fetching) {
+		if (!req_buf.len) {
 			struct strbuf c = STRBUF_INIT;
 			if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 			if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");

@@ -378,14 +363,37 @@ static int find_common(struct fetch_pack_args *args,
 			strbuf_release(&c);
 		} else
 			packet_buf_write(&req_buf, "want %s\n", remote_hex);
-		fetching++;
 	}
 
-	if (!fetching) {
-		strbuf_release(&req_buf);
+	return req_buf.len ? strbuf_detach(&req_buf, NULL) : NULL;
+}
+
+static int find_common(struct fetch_pack_args *args,
+		       int fd[2], unsigned char *result_sha1,
+		       char *wants)
+{
+	int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
+	const unsigned char *sha1;
+	unsigned in_vain = 0;
+	int got_continue = 0;
+	int got_ready = 0;
+	struct strbuf req_buf = STRBUF_INIT;
+	size_t state_len = 0;
+
+	if (args->stateless_rpc && multi_ack == 1)
+		die(_("--stateless-rpc requires multi_ack_detailed"));
+	if (marked)
+		for_each_ref(clear_marks, NULL);
+	marked = 1;
+
+	for_each_ref(rev_list_insert_ref_oid, NULL);
+	for_each_cached_alternate(insert_one_alternate_object);
+
+	if (!wants) {
 		packet_flush(fd[1]);
 		return 1;
 	}
+	strbuf_attach(&req_buf, wants, strlen(wants), strlen(wants));
 
 	if (is_repository_shallow())
 		write_shallow_commits(&req_buf, 1, NULL);

@@ -943,7 +951,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 		packet_flush(fd[1]);
 		goto all_done;
 	}
-	if (find_common(args, fd, sha1, ref) < 0)
+	if (find_common(args, fd, sha1, get_wants(args, ref)) < 0)
 		if (!args->keep_pack)
 			/* When cloning, it is not unusual to have
 			 * no common commit.

index 2a1c1c213..4bd83c4ff 100644
@@ -143,6 +143,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 			args.update_shallow = 1;
 			continue;
 		}
+		if (!strcmp("--new-way", arg)) {
+			args.new_way = 1;
+			continue;
+		}
 		usage(fetch_pack_usage);
 	}
 	if (deepen_not.nr)

@@ -193,7 +197,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 		if (!conn)
 			return args.diag_url ? 0 : 1;
 	}
-	get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, &shallow);
+	if (!args.new_way)
+		get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, &shallow);
 
 	ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought,
 			 &shallow, pack_lockfile_ptr);

@@ -219,7 +224,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 	 * remote no-such-ref' would silently succeed without issuing
 	 * an error.
 	 */
-	ret |= report_unmatched_refs(sought, nr_sought);
+	if (!args.new_way)
+		ret |= report_unmatched_refs(sought, nr_sought);
 
 	while (ref) {
 		printf("%s %s\n",

141 lines not on this page · raw message

index a2d46e6e7..ab7a80696 100644
@@ -29,6 +29,7 @@ struct fetch_pack_args {
 	unsigned cloning:1;
 	unsigned update_shallow:1;
 	unsigned deepen:1;
+	unsigned new_way:1;
 };
 
 /*

new file mode 100644
index 000000000..6dcf87980
@@ -0,0 +1,40 @@
+/*
+ * This program takes payloads in pkt-line format (one or more pkt-lines
+ * followed by a flush pkt) and runs the given command once per payload.
+ */
+#include "cache.h"
+#include "pkt-line.h"
+#include "run-command.h"
+
+int cmd_main(int argc, const char **argv)
+{
+	struct child_process *cmd = NULL;
+	char buffer[LARGE_PACKET_MAX];
+	int size;
+
+	while ((size = packet_read(0, NULL, NULL, buffer, sizeof(buffer),
+				   PACKET_READ_GENTLE_ON_EOF)) >= 0) {
+		if (size > 0) {
+			if (!cmd) {
+				cmd = xmalloc(sizeof(*cmd));
+				child_process_init(cmd);
+				cmd->argv = argv + 1;
+				cmd->git_cmd = 1;
+				cmd->in = -1;
+				cmd->out = 0;
+				if (start_command(cmd))
+					die("could not start command");
+			}
+			write_in_full(cmd->in, buffer, size);
+		} else if (cmd) {
+			close(cmd->in);
+			if (finish_command(cmd))
+				die("could not finish command");
+			free(cmd);
+			cmd = NULL;
+			if (size < 0)
+				break;
+		}
+	}
+	return 0;
+}

index a9c0c7c94..870b853a6 100644
@@ -192,6 +192,123 @@ static int fetch_ref(int stateless_rpc)
 	return -1;
 }
 
+/*
+ * Returns 1 if all blobs are reachable. If not, returns 0 and stores the hash
+ * of one of the unreachable blobs in unreachable.
+ */
+static int are_all_reachable(const struct object_array *blobs, struct object_id *unreachable)
+{
+	struct child_process cmd = CHILD_PROCESS_INIT;
+	static const char *argv[] = {
+		"rev-list", "--objects", "--use-bitmap-index", "--stdin", "--not", "--all", "--not", NULL,
+	};
+	int i;
+	char buf[41] = {0};
+
+	cmd.argv = argv;
+	cmd.git_cmd = 1;
+	cmd.in = -1;
+	cmd.out = -1;
+
+	if (start_command(&cmd))
+		goto error;
+	
+	for (i = 0; i < blobs->nr; i++) {
+		write_in_full(cmd.in, sha1_to_hex(blobs->objects[i].item->oid.hash), 40);
+		write_in_full(cmd.in, "\n", 1);
+	}
+	close(cmd.in);
+	cmd.in = -1;
+
+	i = read_in_full(cmd.out, buf, 40);
+	close(cmd.out);
+	cmd.out = -1;
+
+	if (finish_command(&cmd))
+		goto error;
+
+	if (i) {
+		if (get_oid_hex(buf, unreachable))
+			goto error;
+		return 0;
+	}
+
+	return 1;
+
+error:
+	if (cmd.out >= 0)
+		close(cmd.out);
+	die("problem with running rev-list");
+}
+
+static void send_blobs(const struct object_array *blobs)
+{
+	struct child_process cmd = CHILD_PROCESS_INIT;
+	static const char *argv[] = {
+		"pack-objects", "--stdout", NULL
+	};
+	int i;
+
+	cmd.argv = argv;
+	cmd.git_cmd = 1;
+	cmd.in = -1;
+	cmd.out = 0;
+
+	if (start_command(&cmd))
+		goto error;
+	
+	for (i = 0; i < blobs->nr; i++) {
+		write_in_full(cmd.in, sha1_to_hex(blobs->objects[i].item->oid.hash), 40);
+		write_in_full(cmd.in, "\n", 1);
+	}
+	close(cmd.in);
+	cmd.in = -1;
+
+	if (finish_command(&cmd))
+		goto error;
+
+	return;
+
+error:
+	die("problem with running pack-objects");
+}
+
+static int fetch_blob(void)
+{
+	char *line;
+
+	struct object_array wanted_blobs = OBJECT_ARRAY_INIT;
+	struct object_id unreachable;
+
+	while ((line = packet_read_line(0, NULL))) {
+		const char *arg;
+		if (skip_prefix(line, "want ", &arg)) {
+			struct object_id oid;
+			struct object *obj;
+			if (get_oid_hex(arg, &oid)) {
+				packet_write_fmt(1, "ERR invalid object ID <%s>", arg);
+				return 0;
+			}
+			obj = parse_object(oid.hash);
+			if (!obj || obj->type != OBJ_BLOB) {
+				packet_write_fmt(1, "ERR not our blob <%s>", arg);
+				return 0;
+			}
+			add_object_array(obj, NULL, &wanted_blobs);
+		}
+	}
+
+	if (!are_all_reachable(&wanted_blobs, &unreachable)) {
+		packet_write_fmt(1, "ERR not our blob <%s>", oid_to_hex(&unreachable));
+		return 0;
+	}
+
+	packet_write_fmt(1, "ACK\n");
+	send_blobs(&wanted_blobs);
+
+	return 0;
+}
+
 static int server_endpoint_config(const char *var, const char *value, void *unused)
 {
 	return parse_hide_refs_config(var, value, "uploadpack");

@@ -224,5 +341,7 @@ int cmd_main(int argc, const char **argv)
 	line = packet_read_line(0, NULL);
 	if (!strcmp(line, "fetch-refs"))
 		return fetch_ref(stateless_rpc);
-	die("only fetch-refs is supported");
+	if (!strcmp(line, "fetch-blobs"))
+		return fetch_blob();
+	die("only fetch-refs and fetch-blobs are supported");
 }