Skip to contentSkip to conversation

unpack-objects: unpack large object in stream

PATCH Superseded Han Xin 10 participants 211 messages Oct 9, 2021 – Jul 1, 2022 Documentation Page 1 of 2

lore
Share & tools
lore

Han Xin

author
#0 lore raw

PATCH 1/1 unpack-objects: unpack large object in stream

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.

---
 builtin/unpack-objects.c          |  41 +++++++-
 object-file.c                     | 149 +++++++++++++++++++++++++++---
 object-store.h                    |   9 ++
 t/t5590-receive-unpack-objects.sh |  92 ++++++++++++++++++
 4 files changed, 279 insertions(+), 12 deletions(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +40 -1
index 4a9466295b..8ac77e60a8 100644
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
+static void fill_stream(struct git_zstream *stream)
+{
+	stream->next_in = fill(1);
+	stream->avail_in = len;
+}
+
+static void use_stream(struct git_zstream *stream)
+{
+	use(len - stream->avail_in);
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
+	struct git_zstream_reader reader;
+	struct object_id *oid = &obj_list[nr].oid;
+
+	reader.fill = &fill_stream;
+	reader.use = &use_stream;
+
+	if (write_stream_object_file(&reader, size, type_name(OBJ_BLOB),
+				     oid, dry_run))
+		die("failed to write object in stream");
+	if (strict && !dry_run) {
+		struct blob *blob = lookup_blob(the_repository, oid);
+		if (blob)
+			blob->object.flags |= FLAG_WRITTEN;
+		else
+			die("invalid blob object from stream");
+	}
+	obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 				   unsigned nr)
 {
-	void *buf = get_data(size);
+	void *buf;
+
+	/* Write large blob in stream without allocating full buffer. */
+	if (type == OBJ_BLOB && size > big_file_threshold) {
+		write_stream_blob(nr, size);
+		return;
+	}
 
+	buf = get_data(size);
 	if (!dry_run && buf)
 		write_object(nr, type, buf, size);
 	else
object-file.c +138 -11

167 lines not on this page · raw message

object-store.h +9 -0
index d24915ced1..12b113ef93 100644
@@ -33,6 +33,11 @@ struct object_directory {
 	char *path;
 };
 
+struct git_zstream_reader {
+	void (*fill)(struct git_zstream *);
+	void (*use)(struct git_zstream *);
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 int write_object_file(const void *buf, unsigned long len,
 		      const char *type, struct object_id *oid);
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+			     unsigned long len, const char *type,
+			     struct object_id *oid, int dry_run);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
 			       const char *type, struct object_id *oid,
 			       unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -0

92 lines not on this page · raw message

Signature · 1 line
2.33.0.1.g09a6bb964f.dirty
author
#1 lore raw

Any suggestions?

Quoted 384 lines · Han Xin <[email protected]> 于2021年10月9日周六 下午4:21写道:

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.

---
 builtin/unpack-objects.c          |  41 +++++++-
 object-file.c                     | 149 +++++++++++++++++++++++++++---
 object-store.h                    |   9 ++
 t/t5590-receive-unpack-objects.sh |  92 ++++++++++++++++++
 4 files changed, 279 insertions(+), 12 deletions(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +40 -1
index 4a9466295b..8ac77e60a8 100644
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type,
        }
 }
 
+static void fill_stream(struct git_zstream *stream)
+{
+       stream->next_in = fill(1);
+       stream->avail_in = len;
+}
+
+static void use_stream(struct git_zstream *stream)
+{
+       use(len - stream->avail_in);
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
+       struct git_zstream_reader reader;
+       struct object_id *oid = &obj_list[nr].oid;
+
+       reader.fill = &fill_stream;
+       reader.use = &use_stream;
+
+       if (write_stream_object_file(&reader, size, type_name(OBJ_BLOB),
+                                    oid, dry_run))
+               die("failed to write object in stream");
+       if (strict && !dry_run) {
+               struct blob *blob = lookup_blob(the_repository, oid);
+               if (blob)
+                       blob->object.flags |= FLAG_WRITTEN;
+               else
+                       die("invalid blob object from stream");
+       }
+       obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
                                   unsigned nr)
 {
-       void *buf = get_data(size);
+       void *buf;
+
+       /* Write large blob in stream without allocating full buffer. */
+       if (type == OBJ_BLOB && size > big_file_threshold) {
+               write_stream_blob(nr, size);
+               return;
+       }
 
+       buf = get_data(size);
        if (!dry_run && buf)
                write_object(nr, type, buf, size);
        else
object-file.c +138 -11
index a8be899481..06c1693675 100644
@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
        return fd;
 }
 
+static int write_object_buffer(struct git_zstream *stream, git_hash_ctx *c,
+                              int fd, unsigned char *compressed,
+                              int compressed_len, const void *buf,
+                              size_t len, int flush)
+{
+       int ret;
+
+       stream->next_in = (void *)buf;
+       stream->avail_in = len;
+       do {
+               unsigned char *in0 = stream->next_in;
+               ret = git_deflate(stream, flush);
+               the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+               if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+                       die(_("unable to write loose object file"));
+               stream->next_out = compressed;
+               stream->avail_out = compressed_len;
+       } while (ret == Z_OK);
+
+       return ret;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
                              int hdrlen, const void *buf, unsigned long len,
                              time_t mtime)
@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        the_hash_algo->update_fn(&c, hdr, hdrlen);
 
        /* Then the data itself.. */
-       stream.next_in = (void *)buf;
-       stream.avail_in = len;
-       do {
-               unsigned char *in0 = stream.next_in;
-               ret = git_deflate(&stream, Z_FINISH);
-               the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-               if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-                       die(_("unable to write loose object file"));
-               stream.next_out = compressed;
-               stream.avail_out = sizeof(compressed);
-       } while (ret == Z_OK);
+       ret = write_object_buffer(&stream, &c, fd, compressed,
+                                 sizeof(compressed), buf, len,
+                                 Z_FINISH);
 
        if (ret != Z_STREAM_END)
                die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
        return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                            unsigned long len, const char *type,
+                            struct object_id *oid,
+                            int dry_run)
+{
+       git_zstream istream, ostream;
+       unsigned char buf[8192], compressed[4096];
+       char hdr[MAX_HEADER_LEN];
+       int istatus, ostatus, fd = 0, hdrlen, dirlen, flush = 0;
+       int ret = 0;
+       git_hash_ctx c;
+       struct strbuf tmp_file = STRBUF_INIT;
+       struct strbuf filename = STRBUF_INIT;
+
+       /* Write tmpfile in objects dir, because oid is unknown */
+       if (!dry_run) {
+               strbuf_addstr(&filename, the_repository->objects->odb->path);
+               strbuf_addch(&filename, '/');
+               fd = create_tmpfile(&tmp_file, filename.buf);
+               if (fd < 0) {
+                       if (errno == EACCES)
+                               ret = error(_("insufficient permission for adding an object to repository database %s"),
+                                       get_object_directory());
+                       else
+                               ret = error_errno(_("unable to create temporary file"));
+                       goto cleanup;
+               }
+       }
+
+       memset(&istream, 0, sizeof(istream));
+       istream.next_out = buf;
+       istream.avail_out = sizeof(buf);
+       git_inflate_init(&istream);
+
+       if (!dry_run) {
+               /* Set it up */
+               git_deflate_init(&ostream, zlib_compression_level);
+               ostream.next_out = compressed;
+               ostream.avail_out = sizeof(compressed);
+               the_hash_algo->init_fn(&c);
+
+               /* First header */
+               hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %" PRIuMAX, type,
+                               (uintmax_t)len) + 1;
+               ostream.next_in = (unsigned char *)hdr;
+               ostream.avail_in = hdrlen;
+               while (git_deflate(&ostream, 0) == Z_OK)
+                       ; /* nothing */
+               the_hash_algo->update_fn(&c, hdr, hdrlen);
+       }
+
+       /* Then the data itself */
+       do {
+               unsigned char *last_out = istream.next_out;
+               reader->fill(&istream);
+               istatus = git_inflate(&istream, 0);
+               if (istatus == Z_STREAM_END)
+                       flush = Z_FINISH;
+               reader->use(&istream);
+               if (!dry_run)
+                       ostatus = write_object_buffer(&ostream, &c, fd, compressed,
+                                                     sizeof(compressed), last_out,
+                                                     istream.next_out - last_out,
+                                                     flush);
+               istream.next_out = buf;
+               istream.avail_out = sizeof(buf);
+       } while (istatus == Z_OK);
+
+       if (istream.total_out != len || istatus != Z_STREAM_END)
+               die( _("inflate returned %d"), istatus);
+       git_inflate_end(&istream);
+
+       if (dry_run)
+               goto cleanup;
+
+       if (ostatus != Z_STREAM_END)
+               die(_("unable to deflate new object (%d)"), ostatus);
+       ostatus = git_deflate_end_gently(&ostream);
+       if (ostatus != Z_OK)
+               die(_("deflateEnd on object failed (%d)"), ostatus);
+       the_hash_algo->final_fn(oid->hash, &c);
+       close_loose_object(fd);
+
+       /* We get the oid now */
+       loose_object_path(the_repository, &filename, oid);
+
+       dirlen = directory_size(filename.buf);
+       if (dirlen) {
+               struct strbuf dir = STRBUF_INIT;
+               /*
+                * Make sure the directory exists; note that the contents
+                * of the buffer are undefined after mkstemp returns an
+                * error, so we have to rewrite the whole buffer from
+                * scratch.
+                */
+               strbuf_add(&dir, filename.buf, dirlen - 1);
+               if (mkdir(dir.buf, 0777) && errno != EEXIST) {
+                       unlink_or_warn(tmp_file.buf);
+                       strbuf_release(&dir);
+                       ret = -1;
+                       goto cleanup;
+               }
+               strbuf_release(&dir);
+       }
+
+       ret = finalize_object_file(tmp_file.buf, filename.buf);
+
+cleanup:
+       strbuf_release(&tmp_file);
+       strbuf_release(&filename);
+       return ret;
+}
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                               const char *type, struct object_id *oid,
                               unsigned flags)
object-store.h +9 -0
index d24915ced1..12b113ef93 100644
@@ -33,6 +33,11 @@ struct object_directory {
        char *path;
 };
 
+struct git_zstream_reader {
+       void (*fill)(struct git_zstream *);
+       void (*use)(struct git_zstream *);
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
        struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 int write_object_file(const void *buf, unsigned long len,
                      const char *type, struct object_id *oid);
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                            unsigned long len, const char *type,
+                            struct object_id *oid, int dry_run);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                               const char *type, struct object_id *oid,
                               unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -1
new file mode 100755
index 0000000000..7e63dfc0db
@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Han Xin
+#
+
+test_description='Test unpack-objects when receive pack'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+test_expect_success "create commit with big blobs (1.5 MB)" '
+       test-tool genrandom foo 1500000 >big-blob &&
+       test_commit --append foo big-blob &&
+       test-tool genrandom bar 1500000 >big-blob &&
+       test_commit --append bar big-blob &&
+       (
+               cd .git &&
+               find objects/?? -type f | sort
+       ) >expect &&
+       git repack -ad
+'
+
+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
+       GIT_ALLOC_LIMIT=1m &&
+       export GIT_ALLOC_LIMIT
+'
+
+test_expect_success 'prepare dest repository' '
+       git init --bare dest.git &&
+       git -C dest.git config core.bigFileThreshold 2m &&
+       git -C dest.git config receive.unpacklimit 100
+'
+
+test_expect_success 'fail to push: cannot allocate' '
+       test_must_fail git push dest.git HEAD 2>err &&
+       test_i18ngrep "remote: fatal: attempting to allocate" err &&
+       (
+               cd dest.git &&
+               find objects/?? -type f | sort
+       ) >actual &&
+       ! test_cmp expect actual
+'
+
+test_expect_success 'set a lower bigfile threshold' '
+       git -C dest.git config core.bigFileThreshold 1m
+'
+
+test_expect_success 'unpack big object in stream' '
+       git push dest.git HEAD &&
+       git -C dest.git fsck &&
+       (
+               cd dest.git &&
+               find objects/?? -type f | sort
+       ) >actual &&
+       test_cmp expect actual
+'
+
+test_expect_success 'setup for unpack-objects dry-run test' '
+       PACK=$(echo main | git pack-objects --progress --revs test) &&
+       unset GIT_ALLOC_LIMIT &&
+       git init --bare unpack-test.git
+'
+
+test_expect_success 'unpack-objects dry-run with large threshold' '
+       (
+               cd unpack-test.git &&
+               git config core.bigFileThreshold 2m &&
+               git unpack-objects -n <../test-$PACK.pack
+       ) &&
+       (
+               cd unpack-test.git &&
+               find objects/ -type f
+       ) >actual &&
+       test_must_be_empty actual
+'
+
+test_expect_success 'unpack-objects dry-run with small threshold' '
+       (
+               cd unpack-test.git &&
+               git config core.bigFileThreshold 1m &&
+               git unpack-objects -n <../test-$PACK.pack
+       ) &&
+       (
+               cd unpack-test.git &&
+               find objects/ -type f
+       ) >actual &&
+       test_must_be_empty actual
+'
+
+test_done
--

2.33.0.1.g09a6bb964f.dirty

#2 lore raw
In reply to Han Xin
Quoted 35 lines · Han Xin

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.

---
 builtin/unpack-objects.c          |  41 +++++++-
 object-file.c                     | 149 +++++++++++++++++++++++++++---
 object-store.h                    |   9 ++
 t/t5590-receive-unpack-objects.sh |  92 ++++++++++++++++++
 4 files changed, 279 insertions(+), 12 deletions(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +2 -0
index 4a9466295b..8ac77e60a8 100644
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
+static void fill_stream(struct git_zstream *stream)
+{

builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 334

Quoted 10 lines
builtin/unpack-objects.c +10 -0
+	stream->next_in = fill(1);
+	stream->avail_in = len;
+}
+
+static void use_stream(struct git_zstream *stream)
+{
+	use(len - stream->avail_in);
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)

Can we use size_t for the size, and possibly nr, to improve compatibility with Windows systems where unsigned long is only 32 bits?

There has been some work in the past on providing large file support on Windows, which requires numerous long -> size_t changes.

Philip

Quoted 335 lines
builtin/unpack-objects.c +28 -1
+{
+	struct git_zstream_reader reader;
+	struct object_id *oid = &obj_list[nr].oid;
+
+	reader.fill = &fill_stream;
+	reader.use = &use_stream;
+
+	if (write_stream_object_file(&reader, size, type_name(OBJ_BLOB),
+				     oid, dry_run))
+		die("failed to write object in stream");
+	if (strict && !dry_run) {
+		struct blob *blob = lookup_blob(the_repository, oid);
+		if (blob)
+			blob->object.flags |= FLAG_WRITTEN;
+		else
+			die("invalid blob object from stream");
+	}
+	obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 				   unsigned nr)
 {
-	void *buf = get_data(size);
+	void *buf;
+
+	/* Write large blob in stream without allocating full buffer. */
+	if (type == OBJ_BLOB && size > big_file_threshold) {
+		write_stream_blob(nr, size);
+		return;
+	}
 
+	buf = get_data(size);
 	if (!dry_run && buf)
 		write_object(nr, type, buf, size);
 	else
object-file.c +138 -11
index a8be899481..06c1693675 100644
@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+static int write_object_buffer(struct git_zstream *stream, git_hash_ctx *c,
+			       int fd, unsigned char *compressed,
+			       int compressed_len, const void *buf,
+			       size_t len, int flush)
+{
+	int ret;
+
+	stream->next_in = (void *)buf;
+	stream->avail_in = len;
+	do {
+		unsigned char *in0 = stream->next_in;
+		ret = git_deflate(stream, flush);
+		the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+		if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+			die(_("unable to write loose object file"));
+		stream->next_out = compressed;
+		stream->avail_out = compressed_len;
+	} while (ret == Z_OK);
+
+	return ret;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime)
@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	the_hash_algo->update_fn(&c, hdr, hdrlen);
 
 	/* Then the data itself.. */
-	stream.next_in = (void *)buf;
-	stream.avail_in = len;
-	do {
-		unsigned char *in0 = stream.next_in;
-		ret = git_deflate(&stream, Z_FINISH);
-		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-			die(_("unable to write loose object file"));
-		stream.next_out = compressed;
-		stream.avail_out = sizeof(compressed);
-	} while (ret == Z_OK);
+	ret = write_object_buffer(&stream, &c, fd, compressed,
+				  sizeof(compressed), buf, len,
+				  Z_FINISH);
 
 	if (ret != Z_STREAM_END)
 		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
 	return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+			     unsigned long len, const char *type,
+			     struct object_id *oid,
+			     int dry_run)
+{
+	git_zstream istream, ostream;
+	unsigned char buf[8192], compressed[4096];
+	char hdr[MAX_HEADER_LEN];
+	int istatus, ostatus, fd = 0, hdrlen, dirlen, flush = 0;
+	int ret = 0;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+
+	/* Write tmpfile in objects dir, because oid is unknown */
+	if (!dry_run) {
+		strbuf_addstr(&filename, the_repository->objects->odb->path);
+		strbuf_addch(&filename, '/');
+		fd = create_tmpfile(&tmp_file, filename.buf);
+		if (fd < 0) {
+			if (errno == EACCES)
+				ret = error(_("insufficient permission for adding an object to repository database %s"),
+					get_object_directory());
+			else
+				ret = error_errno(_("unable to create temporary file"));
+			goto cleanup;
+		}
+	}
+
+	memset(&istream, 0, sizeof(istream));
+	istream.next_out = buf;
+	istream.avail_out = sizeof(buf);
+	git_inflate_init(&istream);
+
+	if (!dry_run) {
+		/* Set it up */
+		git_deflate_init(&ostream, zlib_compression_level);
+		ostream.next_out = compressed;
+		ostream.avail_out = sizeof(compressed);
+		the_hash_algo->init_fn(&c);
+
+		/* First header */
+		hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %" PRIuMAX, type,
+				(uintmax_t)len) + 1;
+		ostream.next_in = (unsigned char *)hdr;
+		ostream.avail_in = hdrlen;
+		while (git_deflate(&ostream, 0) == Z_OK)
+			; /* nothing */
+		the_hash_algo->update_fn(&c, hdr, hdrlen);
+	}
+
+	/* Then the data itself */
+	do {
+		unsigned char *last_out = istream.next_out;
+		reader->fill(&istream);
+		istatus = git_inflate(&istream, 0);
+		if (istatus == Z_STREAM_END)
+			flush = Z_FINISH;
+		reader->use(&istream);
+		if (!dry_run)
+			ostatus = write_object_buffer(&ostream, &c, fd, compressed,
+						      sizeof(compressed), last_out,
+						      istream.next_out - last_out,
+						      flush);
+		istream.next_out = buf;
+		istream.avail_out = sizeof(buf);
+	} while (istatus == Z_OK);
+
+	if (istream.total_out != len || istatus != Z_STREAM_END)
+		die( _("inflate returned %d"), istatus);
+	git_inflate_end(&istream);
+
+	if (dry_run)
+		goto cleanup;
+
+	if (ostatus != Z_STREAM_END)
+		die(_("unable to deflate new object (%d)"), ostatus);
+	ostatus = git_deflate_end_gently(&ostream);
+	if (ostatus != Z_OK)
+		die(_("deflateEnd on object failed (%d)"), ostatus);
+	the_hash_algo->final_fn(oid->hash, &c);
+	close_loose_object(fd);
+
+	/* We get the oid now */
+	loose_object_path(the_repository, &filename, oid);
+
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		/*
+		 * Make sure the directory exists; note that the contents
+		 * of the buffer are undefined after mkstemp returns an
+		 * error, so we have to rewrite the whole buffer from
+		 * scratch.
+		 */
+		strbuf_add(&dir, filename.buf, dirlen - 1);
+		if (mkdir(dir.buf, 0777) && errno != EEXIST) {
+			unlink_or_warn(tmp_file.buf);
+			strbuf_release(&dir);
+			ret = -1;
+			goto cleanup;
+		}
+		strbuf_release(&dir);
+	}
+
+	ret = finalize_object_file(tmp_file.buf, filename.buf);
+
+cleanup:
+	strbuf_release(&tmp_file);
+	strbuf_release(&filename);
+	return ret;
+}
+
 int hash_object_file_literally(const void *buf, unsigned long len,
 			       const char *type, struct object_id *oid,
 			       unsigned flags)
object-store.h +9 -0
index d24915ced1..12b113ef93 100644
@@ -33,6 +33,11 @@ struct object_directory {
 	char *path;
 };
 
+struct git_zstream_reader {
+	void (*fill)(struct git_zstream *);
+	void (*use)(struct git_zstream *);
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 int write_object_file(const void *buf, unsigned long len,
 		      const char *type, struct object_id *oid);
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+			     unsigned long len, const char *type,
+			     struct object_id *oid, int dry_run);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
 			       const char *type, struct object_id *oid,
 			       unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -0
new file mode 100755
index 0000000000..7e63dfc0db
@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Han Xin
+#
+
+test_description='Test unpack-objects when receive pack'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+test_expect_success "create commit with big blobs (1.5 MB)" '
+	test-tool genrandom foo 1500000 >big-blob &&
+	test_commit --append foo big-blob &&
+	test-tool genrandom bar 1500000 >big-blob &&
+	test_commit --append bar big-blob &&
+	(
+		cd .git &&
+		find objects/?? -type f | sort
+	) >expect &&
+	git repack -ad
+'
+
+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
+	GIT_ALLOC_LIMIT=1m &&
+	export GIT_ALLOC_LIMIT
+'
+
+test_expect_success 'prepare dest repository' '
+	git init --bare dest.git &&
+	git -C dest.git config core.bigFileThreshold 2m &&
+	git -C dest.git config receive.unpacklimit 100
+'
+
+test_expect_success 'fail to push: cannot allocate' '
+	test_must_fail git push dest.git HEAD 2>err &&
+	test_i18ngrep "remote: fatal: attempting to allocate" err &&
+	(
+		cd dest.git &&
+		find objects/?? -type f | sort
+	) >actual &&
+	! test_cmp expect actual
+'
+
+test_expect_success 'set a lower bigfile threshold' '
+	git -C dest.git config core.bigFileThreshold 1m
+'
+
+test_expect_success 'unpack big object in stream' '
+	git push dest.git HEAD &&
+	git -C dest.git fsck &&
+	(
+		cd dest.git &&
+		find objects/?? -type f | sort
+	) >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'setup for unpack-objects dry-run test' '
+	PACK=$(echo main | git pack-objects --progress --revs test) &&
+	unset GIT_ALLOC_LIMIT &&
+	git init --bare unpack-test.git
+'
+
+test_expect_success 'unpack-objects dry-run with large threshold' '
+	(
+		cd unpack-test.git &&
+		git config core.bigFileThreshold 2m &&
+		git unpack-objects -n <../test-$PACK.pack
+	) &&
+	(
+		cd unpack-test.git &&
+		find objects/ -type f
+	) >actual &&
+	test_must_be_empty actual
+'
+
+test_expect_success 'unpack-objects dry-run with small threshold' '
+	(
+		cd unpack-test.git &&
+		git config core.bigFileThreshold 1m &&
+		git unpack-objects -n <../test-$PACK.pack
+	) &&
+	(
+		cd unpack-test.git &&
+		find objects/ -type f
+	) >actual &&
+	test_must_be_empty actual
+'
+
+test_done
author
#3 lore raw
Quoted 51 lines · Philip Oakley <[email protected]> 于2021年10月20日周三 下午10:43写道:
Quoted 35 lines · Han Xin

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.

---
 builtin/unpack-objects.c          |  41 +++++++-
 object-file.c                     | 149 +++++++++++++++++++++++++++---
 object-store.h                    |   9 ++
 t/t5590-receive-unpack-objects.sh |  92 ++++++++++++++++++
 4 files changed, 279 insertions(+), 12 deletions(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +2 -0
index 4a9466295b..8ac77e60a8 100644
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type,
      }
 }
 
+static void fill_stream(struct git_zstream *stream)
+{

builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 334

Quoted 10 lines
builtin/unpack-objects.c +10 -0
+     stream->next_in = fill(1);
+     stream->avail_in = len;
+}
+
+static void use_stream(struct git_zstream *stream)
+{
+     use(len - stream->avail_in);
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)

Can we use size_t for the size, and possibly nr, to improve compatibility with Windows systems where unsigned long is only 32 bits?

Quoted 4 lines

There has been some work in the past on providing large file support on Windows, which requires numerous long -> size_t changes.

Philip

Thanks for your review. I'm not sure if I should do this change in this patch, it will also change the type defined in unpack_one(),unpack_non_delta_entry, write_object() and many others.

Quoted 336 lines
Quoted 335 lines
builtin/unpack-objects.c +28 -1
+{
+     struct git_zstream_reader reader;
+     struct object_id *oid = &obj_list[nr].oid;
+
+     reader.fill = &fill_stream;
+     reader.use = &use_stream;
+
+     if (write_stream_object_file(&reader, size, type_name(OBJ_BLOB),
+                                  oid, dry_run))
+             die("failed to write object in stream");
+     if (strict && !dry_run) {
+             struct blob *blob = lookup_blob(the_repository, oid);
+             if (blob)
+                     blob->object.flags |= FLAG_WRITTEN;
+             else
+                     die("invalid blob object from stream");
+     }
+     obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
                                 unsigned nr)
 {
-     void *buf = get_data(size);
+     void *buf;
+
+     /* Write large blob in stream without allocating full buffer. */
+     if (type == OBJ_BLOB && size > big_file_threshold) {
+             write_stream_blob(nr, size);
+             return;
+     }
 
+     buf = get_data(size);
      if (!dry_run && buf)
              write_object(nr, type, buf, size);
      else
object-file.c +138 -11
index a8be899481..06c1693675 100644
@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
      return fd;
 }
 
+static int write_object_buffer(struct git_zstream *stream, git_hash_ctx *c,
+                            int fd, unsigned char *compressed,
+                            int compressed_len, const void *buf,
+                            size_t len, int flush)
+{
+     int ret;
+
+     stream->next_in = (void *)buf;
+     stream->avail_in = len;
+     do {
+             unsigned char *in0 = stream->next_in;
+             ret = git_deflate(stream, flush);
+             the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+             if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+                     die(_("unable to write loose object file"));
+             stream->next_out = compressed;
+             stream->avail_out = compressed_len;
+     } while (ret == Z_OK);
+
+     return ret;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
                            int hdrlen, const void *buf, unsigned long len,
                            time_t mtime)
@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
      the_hash_algo->update_fn(&c, hdr, hdrlen);
 
      /* Then the data itself.. */
-     stream.next_in = (void *)buf;
-     stream.avail_in = len;
-     do {
-             unsigned char *in0 = stream.next_in;
-             ret = git_deflate(&stream, Z_FINISH);
-             the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-             if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-                     die(_("unable to write loose object file"));
-             stream.next_out = compressed;
-             stream.avail_out = sizeof(compressed);
-     } while (ret == Z_OK);
+     ret = write_object_buffer(&stream, &c, fd, compressed,
+                               sizeof(compressed), buf, len,
+                               Z_FINISH);
 
      if (ret != Z_STREAM_END)
              die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
      return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                          unsigned long len, const char *type,
+                          struct object_id *oid,
+                          int dry_run)
+{
+     git_zstream istream, ostream;
+     unsigned char buf[8192], compressed[4096];
+     char hdr[MAX_HEADER_LEN];
+     int istatus, ostatus, fd = 0, hdrlen, dirlen, flush = 0;
+     int ret = 0;
+     git_hash_ctx c;
+     struct strbuf tmp_file = STRBUF_INIT;
+     struct strbuf filename = STRBUF_INIT;
+
+     /* Write tmpfile in objects dir, because oid is unknown */
+     if (!dry_run) {
+             strbuf_addstr(&filename, the_repository->objects->odb->path);
+             strbuf_addch(&filename, '/');
+             fd = create_tmpfile(&tmp_file, filename.buf);
+             if (fd < 0) {
+                     if (errno == EACCES)
+                             ret = error(_("insufficient permission for adding an object to repository database %s"),
+                                     get_object_directory());
+                     else
+                             ret = error_errno(_("unable to create temporary file"));
+                     goto cleanup;
+             }
+     }
+
+     memset(&istream, 0, sizeof(istream));
+     istream.next_out = buf;
+     istream.avail_out = sizeof(buf);
+     git_inflate_init(&istream);
+
+     if (!dry_run) {
+             /* Set it up */
+             git_deflate_init(&ostream, zlib_compression_level);
+             ostream.next_out = compressed;
+             ostream.avail_out = sizeof(compressed);
+             the_hash_algo->init_fn(&c);
+
+             /* First header */
+             hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %" PRIuMAX, type,
+                             (uintmax_t)len) + 1;
+             ostream.next_in = (unsigned char *)hdr;
+             ostream.avail_in = hdrlen;
+             while (git_deflate(&ostream, 0) == Z_OK)
+                     ; /* nothing */
+             the_hash_algo->update_fn(&c, hdr, hdrlen);
+     }
+
+     /* Then the data itself */
+     do {
+             unsigned char *last_out = istream.next_out;
+             reader->fill(&istream);
+             istatus = git_inflate(&istream, 0);
+             if (istatus == Z_STREAM_END)
+                     flush = Z_FINISH;
+             reader->use(&istream);
+             if (!dry_run)
+                     ostatus = write_object_buffer(&ostream, &c, fd, compressed,
+                                                   sizeof(compressed), last_out,
+                                                   istream.next_out - last_out,
+                                                   flush);
+             istream.next_out = buf;
+             istream.avail_out = sizeof(buf);
+     } while (istatus == Z_OK);
+
+     if (istream.total_out != len || istatus != Z_STREAM_END)
+             die( _("inflate returned %d"), istatus);
+     git_inflate_end(&istream);
+
+     if (dry_run)
+             goto cleanup;
+
+     if (ostatus != Z_STREAM_END)
+             die(_("unable to deflate new object (%d)"), ostatus);
+     ostatus = git_deflate_end_gently(&ostream);
+     if (ostatus != Z_OK)
+             die(_("deflateEnd on object failed (%d)"), ostatus);
+     the_hash_algo->final_fn(oid->hash, &c);
+     close_loose_object(fd);
+
+     /* We get the oid now */
+     loose_object_path(the_repository, &filename, oid);
+
+     dirlen = directory_size(filename.buf);
+     if (dirlen) {
+             struct strbuf dir = STRBUF_INIT;
+             /*
+              * Make sure the directory exists; note that the contents
+              * of the buffer are undefined after mkstemp returns an
+              * error, so we have to rewrite the whole buffer from
+              * scratch.
+              */
+             strbuf_add(&dir, filename.buf, dirlen - 1);
+             if (mkdir(dir.buf, 0777) && errno != EEXIST) {
+                     unlink_or_warn(tmp_file.buf);
+                     strbuf_release(&dir);
+                     ret = -1;
+                     goto cleanup;
+             }
+             strbuf_release(&dir);
+     }
+
+     ret = finalize_object_file(tmp_file.buf, filename.buf);
+
+cleanup:
+     strbuf_release(&tmp_file);
+     strbuf_release(&filename);
+     return ret;
+}
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                             const char *type, struct object_id *oid,
                             unsigned flags)
object-store.h +9 -0
index d24915ced1..12b113ef93 100644
@@ -33,6 +33,11 @@ struct object_directory {
      char *path;
 };
 
+struct git_zstream_reader {
+     void (*fill)(struct git_zstream *);
+     void (*use)(struct git_zstream *);
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
      struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 int write_object_file(const void *buf, unsigned long len,
                    const char *type, struct object_id *oid);
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                          unsigned long len, const char *type,
+                          struct object_id *oid, int dry_run);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                             const char *type, struct object_id *oid,
                             unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -0
new file mode 100755
index 0000000000..7e63dfc0db
@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Han Xin
+#
+
+test_description='Test unpack-objects when receive pack'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+test_expect_success "create commit with big blobs (1.5 MB)" '
+     test-tool genrandom foo 1500000 >big-blob &&
+     test_commit --append foo big-blob &&
+     test-tool genrandom bar 1500000 >big-blob &&
+     test_commit --append bar big-blob &&
+     (
+             cd .git &&
+             find objects/?? -type f | sort
+     ) >expect &&
+     git repack -ad
+'
+
+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
+     GIT_ALLOC_LIMIT=1m &&
+     export GIT_ALLOC_LIMIT
+'
+
+test_expect_success 'prepare dest repository' '
+     git init --bare dest.git &&
+     git -C dest.git config core.bigFileThreshold 2m &&
+     git -C dest.git config receive.unpacklimit 100
+'
+
+test_expect_success 'fail to push: cannot allocate' '
+     test_must_fail git push dest.git HEAD 2>err &&
+     test_i18ngrep "remote: fatal: attempting to allocate" err &&
+     (
+             cd dest.git &&
+             find objects/?? -type f | sort
+     ) >actual &&
+     ! test_cmp expect actual
+'
+
+test_expect_success 'set a lower bigfile threshold' '
+     git -C dest.git config core.bigFileThreshold 1m
+'
+
+test_expect_success 'unpack big object in stream' '
+     git push dest.git HEAD &&
+     git -C dest.git fsck &&
+     (
+             cd dest.git &&
+             find objects/?? -type f | sort
+     ) >actual &&
+     test_cmp expect actual
+'
+
+test_expect_success 'setup for unpack-objects dry-run test' '
+     PACK=$(echo main | git pack-objects --progress --revs test) &&
+     unset GIT_ALLOC_LIMIT &&
+     git init --bare unpack-test.git
+'
+
+test_expect_success 'unpack-objects dry-run with large threshold' '
+     (
+             cd unpack-test.git &&
+             git config core.bigFileThreshold 2m &&
+             git unpack-objects -n <../test-$PACK.pack
+     ) &&
+     (
+             cd unpack-test.git &&
+             find objects/ -type f
+     ) >actual &&
+     test_must_be_empty actual
+'
+
+test_expect_success 'unpack-objects dry-run with small threshold' '
+     (
+             cd unpack-test.git &&
+             git config core.bigFileThreshold 1m &&
+             git unpack-objects -n <../test-$PACK.pack
+     ) &&
+     (
+             cd unpack-test.git &&
+             find objects/ -type f
+     ) >actual &&
+     test_must_be_empty actual
+'
+
+test_done
#4 lore raw
Quoted 4 lines · Han Xin
Quoted 4 lines
Quoted 1 line

+static void write_stream_blob(unsigned nr, unsigned long size)

Can we use size_t for the size, and possibly nr, to improve compatibility with Windows systems where unsigned long is only 32 bits?

Quoted 8 lines
Quoted 4 lines

There has been some work in the past on providing large file support on Windows, which requires numerous long -> size_t changes.

Philip

Thanks for your review. I'm not sure if I should do this change in this patch, it will also change the type defined in unpack_one(),unpack_non_delta_entry, write_object() and many others.

I was mainly raising the issue regarding the 4GB (sometime 2GB) limitations on Windows which has been a problem for many years.

I had been thinking of not changing the nr (number of objects limit) as 2G objects is hopefully already sufficient, even for thargest of repos (though IIUC their index file size did break the 32bit size limit).

Staying with the existing types won't make the situation any worse, so from that perspective the change isn't needed.
--
Philip

author
#5 lore raw
In reply to Han Xin

Any more suggestions?

Quoted 384 lines · Han Xin <[email protected]> 于2021年10月9日周六 下午4:21写道:

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.

---
 builtin/unpack-objects.c          |  41 +++++++-
 object-file.c                     | 149 +++++++++++++++++++++++++++---
 object-store.h                    |   9 ++
 t/t5590-receive-unpack-objects.sh |  92 ++++++++++++++++++
 4 files changed, 279 insertions(+), 12 deletions(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +40 -1
index 4a9466295b..8ac77e60a8 100644
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type,
        }
 }
 
+static void fill_stream(struct git_zstream *stream)
+{
+       stream->next_in = fill(1);
+       stream->avail_in = len;
+}
+
+static void use_stream(struct git_zstream *stream)
+{
+       use(len - stream->avail_in);
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
+       struct git_zstream_reader reader;
+       struct object_id *oid = &obj_list[nr].oid;
+
+       reader.fill = &fill_stream;
+       reader.use = &use_stream;
+
+       if (write_stream_object_file(&reader, size, type_name(OBJ_BLOB),
+                                    oid, dry_run))
+               die("failed to write object in stream");
+       if (strict && !dry_run) {
+               struct blob *blob = lookup_blob(the_repository, oid);
+               if (blob)
+                       blob->object.flags |= FLAG_WRITTEN;
+               else
+                       die("invalid blob object from stream");
+       }
+       obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
                                   unsigned nr)
 {
-       void *buf = get_data(size);
+       void *buf;
+
+       /* Write large blob in stream without allocating full buffer. */
+       if (type == OBJ_BLOB && size > big_file_threshold) {
+               write_stream_blob(nr, size);
+               return;
+       }
 
+       buf = get_data(size);
        if (!dry_run && buf)
                write_object(nr, type, buf, size);
        else
object-file.c +138 -11
index a8be899481..06c1693675 100644
@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
        return fd;
 }
 
+static int write_object_buffer(struct git_zstream *stream, git_hash_ctx *c,
+                              int fd, unsigned char *compressed,
+                              int compressed_len, const void *buf,
+                              size_t len, int flush)
+{
+       int ret;
+
+       stream->next_in = (void *)buf;
+       stream->avail_in = len;
+       do {
+               unsigned char *in0 = stream->next_in;
+               ret = git_deflate(stream, flush);
+               the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+               if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+                       die(_("unable to write loose object file"));
+               stream->next_out = compressed;
+               stream->avail_out = compressed_len;
+       } while (ret == Z_OK);
+
+       return ret;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
                              int hdrlen, const void *buf, unsigned long len,
                              time_t mtime)
@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        the_hash_algo->update_fn(&c, hdr, hdrlen);
 
        /* Then the data itself.. */
-       stream.next_in = (void *)buf;
-       stream.avail_in = len;
-       do {
-               unsigned char *in0 = stream.next_in;
-               ret = git_deflate(&stream, Z_FINISH);
-               the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-               if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-                       die(_("unable to write loose object file"));
-               stream.next_out = compressed;
-               stream.avail_out = sizeof(compressed);
-       } while (ret == Z_OK);
+       ret = write_object_buffer(&stream, &c, fd, compressed,
+                                 sizeof(compressed), buf, len,
+                                 Z_FINISH);
 
        if (ret != Z_STREAM_END)
                die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
        return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                            unsigned long len, const char *type,
+                            struct object_id *oid,
+                            int dry_run)
+{
+       git_zstream istream, ostream;
+       unsigned char buf[8192], compressed[4096];
+       char hdr[MAX_HEADER_LEN];
+       int istatus, ostatus, fd = 0, hdrlen, dirlen, flush = 0;
+       int ret = 0;
+       git_hash_ctx c;
+       struct strbuf tmp_file = STRBUF_INIT;
+       struct strbuf filename = STRBUF_INIT;
+
+       /* Write tmpfile in objects dir, because oid is unknown */
+       if (!dry_run) {
+               strbuf_addstr(&filename, the_repository->objects->odb->path);
+               strbuf_addch(&filename, '/');
+               fd = create_tmpfile(&tmp_file, filename.buf);
+               if (fd < 0) {
+                       if (errno == EACCES)
+                               ret = error(_("insufficient permission for adding an object to repository database %s"),
+                                       get_object_directory());
+                       else
+                               ret = error_errno(_("unable to create temporary file"));
+                       goto cleanup;
+               }
+       }
+
+       memset(&istream, 0, sizeof(istream));
+       istream.next_out = buf;
+       istream.avail_out = sizeof(buf);
+       git_inflate_init(&istream);
+
+       if (!dry_run) {
+               /* Set it up */
+               git_deflate_init(&ostream, zlib_compression_level);
+               ostream.next_out = compressed;
+               ostream.avail_out = sizeof(compressed);
+               the_hash_algo->init_fn(&c);
+
+               /* First header */
+               hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %" PRIuMAX, type,
+                               (uintmax_t)len) + 1;
+               ostream.next_in = (unsigned char *)hdr;
+               ostream.avail_in = hdrlen;
+               while (git_deflate(&ostream, 0) == Z_OK)
+                       ; /* nothing */
+               the_hash_algo->update_fn(&c, hdr, hdrlen);
+       }
+
+       /* Then the data itself */
+       do {
+               unsigned char *last_out = istream.next_out;
+               reader->fill(&istream);
+               istatus = git_inflate(&istream, 0);
+               if (istatus == Z_STREAM_END)
+                       flush = Z_FINISH;
+               reader->use(&istream);
+               if (!dry_run)
+                       ostatus = write_object_buffer(&ostream, &c, fd, compressed,
+                                                     sizeof(compressed), last_out,
+                                                     istream.next_out - last_out,
+                                                     flush);
+               istream.next_out = buf;
+               istream.avail_out = sizeof(buf);
+       } while (istatus == Z_OK);
+
+       if (istream.total_out != len || istatus != Z_STREAM_END)
+               die( _("inflate returned %d"), istatus);
+       git_inflate_end(&istream);
+
+       if (dry_run)
+               goto cleanup;
+
+       if (ostatus != Z_STREAM_END)
+               die(_("unable to deflate new object (%d)"), ostatus);
+       ostatus = git_deflate_end_gently(&ostream);
+       if (ostatus != Z_OK)
+               die(_("deflateEnd on object failed (%d)"), ostatus);
+       the_hash_algo->final_fn(oid->hash, &c);
+       close_loose_object(fd);
+
+       /* We get the oid now */
+       loose_object_path(the_repository, &filename, oid);
+
+       dirlen = directory_size(filename.buf);
+       if (dirlen) {
+               struct strbuf dir = STRBUF_INIT;
+               /*
+                * Make sure the directory exists; note that the contents
+                * of the buffer are undefined after mkstemp returns an
+                * error, so we have to rewrite the whole buffer from
+                * scratch.
+                */
+               strbuf_add(&dir, filename.buf, dirlen - 1);
+               if (mkdir(dir.buf, 0777) && errno != EEXIST) {
+                       unlink_or_warn(tmp_file.buf);
+                       strbuf_release(&dir);
+                       ret = -1;
+                       goto cleanup;
+               }
+               strbuf_release(&dir);
+       }
+
+       ret = finalize_object_file(tmp_file.buf, filename.buf);
+
+cleanup:
+       strbuf_release(&tmp_file);
+       strbuf_release(&filename);
+       return ret;
+}
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                               const char *type, struct object_id *oid,
                               unsigned flags)
object-store.h +9 -0
index d24915ced1..12b113ef93 100644
@@ -33,6 +33,11 @@ struct object_directory {
        char *path;
 };
 
+struct git_zstream_reader {
+       void (*fill)(struct git_zstream *);
+       void (*use)(struct git_zstream *);
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
        struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 int write_object_file(const void *buf, unsigned long len,
                      const char *type, struct object_id *oid);
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                            unsigned long len, const char *type,
+                            struct object_id *oid, int dry_run);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                               const char *type, struct object_id *oid,
                               unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -1
new file mode 100755
index 0000000000..7e63dfc0db
@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Han Xin
+#
+
+test_description='Test unpack-objects when receive pack'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+test_expect_success "create commit with big blobs (1.5 MB)" '
+       test-tool genrandom foo 1500000 >big-blob &&
+       test_commit --append foo big-blob &&
+       test-tool genrandom bar 1500000 >big-blob &&
+       test_commit --append bar big-blob &&
+       (
+               cd .git &&
+               find objects/?? -type f | sort
+       ) >expect &&
+       git repack -ad
+'
+
+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
+       GIT_ALLOC_LIMIT=1m &&
+       export GIT_ALLOC_LIMIT
+'
+
+test_expect_success 'prepare dest repository' '
+       git init --bare dest.git &&
+       git -C dest.git config core.bigFileThreshold 2m &&
+       git -C dest.git config receive.unpacklimit 100
+'
+
+test_expect_success 'fail to push: cannot allocate' '
+       test_must_fail git push dest.git HEAD 2>err &&
+       test_i18ngrep "remote: fatal: attempting to allocate" err &&
+       (
+               cd dest.git &&
+               find objects/?? -type f | sort
+       ) >actual &&
+       ! test_cmp expect actual
+'
+
+test_expect_success 'set a lower bigfile threshold' '
+       git -C dest.git config core.bigFileThreshold 1m
+'
+
+test_expect_success 'unpack big object in stream' '
+       git push dest.git HEAD &&
+       git -C dest.git fsck &&
+       (
+               cd dest.git &&
+               find objects/?? -type f | sort
+       ) >actual &&
+       test_cmp expect actual
+'
+
+test_expect_success 'setup for unpack-objects dry-run test' '
+       PACK=$(echo main | git pack-objects --progress --revs test) &&
+       unset GIT_ALLOC_LIMIT &&
+       git init --bare unpack-test.git
+'
+
+test_expect_success 'unpack-objects dry-run with large threshold' '
+       (
+               cd unpack-test.git &&
+               git config core.bigFileThreshold 2m &&
+               git unpack-objects -n <../test-$PACK.pack
+       ) &&
+       (
+               cd unpack-test.git &&
+               find objects/ -type f
+       ) >actual &&
+       test_must_be_empty actual
+'
+
+test_expect_success 'unpack-objects dry-run with small threshold' '
+       (
+               cd unpack-test.git &&
+               git config core.bigFileThreshold 1m &&
+               git unpack-objects -n <../test-$PACK.pack
+       ) &&
+       (
+               cd unpack-test.git &&
+               find objects/ -type f
+       ) >actual &&
+       test_must_be_empty actual
+'
+
+test_done
--

2.33.0.1.g09a6bb964f.dirty

#6 lore raw

(replies to the alibaba-inc.com aren't getting through for me)

Quoted 9 lines · Han Xin

Any more suggestions?

Quoted 6 lines · Han Xin <[email protected]> 于2021年10月9日周六 下午4:21写道:

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

Is it possible to split the patch into smaller pieces, taking each item separately?

For large files (as above), it should be possible to stream the unpacking direct to disk, in the same way that the zlib reading is chunked. However having the same 'code' in two places would need to be addressed (the DRY principle).

At the moment on LLP64 systems (Windows) there is already a long (32bit) vs size_t (64bit) problem there (zlib stream), and the size_t problem then permeates the wider codebase.

The normal Git file operations does tend to memory map whole files, but here it looks like you can bypass that.

Quoted 4 lines
Quoted 4 lines

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

What does 'moderate' mean here? Does it mean there is a simple test that allows you to side step the whole problem?

Quoted 4 lines
Quoted 4 lines

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.

Is this "core.bigFileThreshold" the core element? If so, it is too far down the commit message. The readers have already (potentially) misread the message and reacted too soon.  Perhaps: "use core.bigFileThreshold to avoid mmap OOM limits when unpacking".

--
Philip

Quoted 369 lines
Quoted 369 lines
---
 builtin/unpack-objects.c          |  41 +++++++-
 object-file.c                     | 149 +++++++++++++++++++++++++++---
 object-store.h                    |   9 ++
 t/t5590-receive-unpack-objects.sh |  92 ++++++++++++++++++
 4 files changed, 279 insertions(+), 12 deletions(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +40 -1
index 4a9466295b..8ac77e60a8 100644
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type,
        }
 }
 
+static void fill_stream(struct git_zstream *stream)
+{
+       stream->next_in = fill(1);
+       stream->avail_in = len;
+}
+
+static void use_stream(struct git_zstream *stream)
+{
+       use(len - stream->avail_in);
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
+       struct git_zstream_reader reader;
+       struct object_id *oid = &obj_list[nr].oid;
+
+       reader.fill = &fill_stream;
+       reader.use = &use_stream;
+
+       if (write_stream_object_file(&reader, size, type_name(OBJ_BLOB),
+                                    oid, dry_run))
+               die("failed to write object in stream");
+       if (strict && !dry_run) {
+               struct blob *blob = lookup_blob(the_repository, oid);
+               if (blob)
+                       blob->object.flags |= FLAG_WRITTEN;
+               else
+                       die("invalid blob object from stream");
+       }
+       obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
                                   unsigned nr)
 {
-       void *buf = get_data(size);
+       void *buf;
+
+       /* Write large blob in stream without allocating full buffer. */
+       if (type == OBJ_BLOB && size > big_file_threshold) {
+               write_stream_blob(nr, size);
+               return;
+       }
 
+       buf = get_data(size);
        if (!dry_run && buf)
                write_object(nr, type, buf, size);
        else
object-file.c +138 -11
index a8be899481..06c1693675 100644
@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
        return fd;
 }
 
+static int write_object_buffer(struct git_zstream *stream, git_hash_ctx *c,
+                              int fd, unsigned char *compressed,
+                              int compressed_len, const void *buf,
+                              size_t len, int flush)
+{
+       int ret;
+
+       stream->next_in = (void *)buf;
+       stream->avail_in = len;
+       do {
+               unsigned char *in0 = stream->next_in;
+               ret = git_deflate(stream, flush);
+               the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+               if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+                       die(_("unable to write loose object file"));
+               stream->next_out = compressed;
+               stream->avail_out = compressed_len;
+       } while (ret == Z_OK);
+
+       return ret;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
                              int hdrlen, const void *buf, unsigned long len,
                              time_t mtime)
@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        the_hash_algo->update_fn(&c, hdr, hdrlen);
 
        /* Then the data itself.. */
-       stream.next_in = (void *)buf;
-       stream.avail_in = len;
-       do {
-               unsigned char *in0 = stream.next_in;
-               ret = git_deflate(&stream, Z_FINISH);
-               the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-               if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-                       die(_("unable to write loose object file"));
-               stream.next_out = compressed;
-               stream.avail_out = sizeof(compressed);
-       } while (ret == Z_OK);
+       ret = write_object_buffer(&stream, &c, fd, compressed,
+                                 sizeof(compressed), buf, len,
+                                 Z_FINISH);
 
        if (ret != Z_STREAM_END)
                die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
        return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                            unsigned long len, const char *type,
+                            struct object_id *oid,
+                            int dry_run)
+{
+       git_zstream istream, ostream;
+       unsigned char buf[8192], compressed[4096];
+       char hdr[MAX_HEADER_LEN];
+       int istatus, ostatus, fd = 0, hdrlen, dirlen, flush = 0;
+       int ret = 0;
+       git_hash_ctx c;
+       struct strbuf tmp_file = STRBUF_INIT;
+       struct strbuf filename = STRBUF_INIT;
+
+       /* Write tmpfile in objects dir, because oid is unknown */
+       if (!dry_run) {
+               strbuf_addstr(&filename, the_repository->objects->odb->path);
+               strbuf_addch(&filename, '/');
+               fd = create_tmpfile(&tmp_file, filename.buf);
+               if (fd < 0) {
+                       if (errno == EACCES)
+                               ret = error(_("insufficient permission for adding an object to repository database %s"),
+                                       get_object_directory());
+                       else
+                               ret = error_errno(_("unable to create temporary file"));
+                       goto cleanup;
+               }
+       }
+
+       memset(&istream, 0, sizeof(istream));
+       istream.next_out = buf;
+       istream.avail_out = sizeof(buf);
+       git_inflate_init(&istream);
+
+       if (!dry_run) {
+               /* Set it up */
+               git_deflate_init(&ostream, zlib_compression_level);
+               ostream.next_out = compressed;
+               ostream.avail_out = sizeof(compressed);
+               the_hash_algo->init_fn(&c);
+
+               /* First header */
+               hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %" PRIuMAX, type,
+                               (uintmax_t)len) + 1;
+               ostream.next_in = (unsigned char *)hdr;
+               ostream.avail_in = hdrlen;
+               while (git_deflate(&ostream, 0) == Z_OK)
+                       ; /* nothing */
+               the_hash_algo->update_fn(&c, hdr, hdrlen);
+       }
+
+       /* Then the data itself */
+       do {
+               unsigned char *last_out = istream.next_out;
+               reader->fill(&istream);
+               istatus = git_inflate(&istream, 0);
+               if (istatus == Z_STREAM_END)
+                       flush = Z_FINISH;
+               reader->use(&istream);
+               if (!dry_run)
+                       ostatus = write_object_buffer(&ostream, &c, fd, compressed,
+                                                     sizeof(compressed), last_out,
+                                                     istream.next_out - last_out,
+                                                     flush);
+               istream.next_out = buf;
+               istream.avail_out = sizeof(buf);
+       } while (istatus == Z_OK);
+
+       if (istream.total_out != len || istatus != Z_STREAM_END)
+               die( _("inflate returned %d"), istatus);
+       git_inflate_end(&istream);
+
+       if (dry_run)
+               goto cleanup;
+
+       if (ostatus != Z_STREAM_END)
+               die(_("unable to deflate new object (%d)"), ostatus);
+       ostatus = git_deflate_end_gently(&ostream);
+       if (ostatus != Z_OK)
+               die(_("deflateEnd on object failed (%d)"), ostatus);
+       the_hash_algo->final_fn(oid->hash, &c);
+       close_loose_object(fd);
+
+       /* We get the oid now */
+       loose_object_path(the_repository, &filename, oid);
+
+       dirlen = directory_size(filename.buf);
+       if (dirlen) {
+               struct strbuf dir = STRBUF_INIT;
+               /*
+                * Make sure the directory exists; note that the contents
+                * of the buffer are undefined after mkstemp returns an
+                * error, so we have to rewrite the whole buffer from
+                * scratch.
+                */
+               strbuf_add(&dir, filename.buf, dirlen - 1);
+               if (mkdir(dir.buf, 0777) && errno != EEXIST) {
+                       unlink_or_warn(tmp_file.buf);
+                       strbuf_release(&dir);
+                       ret = -1;
+                       goto cleanup;
+               }
+               strbuf_release(&dir);
+       }
+
+       ret = finalize_object_file(tmp_file.buf, filename.buf);
+
+cleanup:
+       strbuf_release(&tmp_file);
+       strbuf_release(&filename);
+       return ret;
+}
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                               const char *type, struct object_id *oid,
                               unsigned flags)
object-store.h +9 -0
index d24915ced1..12b113ef93 100644
@@ -33,6 +33,11 @@ struct object_directory {
        char *path;
 };
 
+struct git_zstream_reader {
+       void (*fill)(struct git_zstream *);
+       void (*use)(struct git_zstream *);
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
        struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 int write_object_file(const void *buf, unsigned long len,
                      const char *type, struct object_id *oid);
 
+int write_stream_object_file(struct git_zstream_reader *reader,
+                            unsigned long len, const char *type,
+                            struct object_id *oid, int dry_run);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
                               const char *type, struct object_id *oid,
                               unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -1
new file mode 100755
index 0000000000..7e63dfc0db
@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Han Xin
+#
+
+test_description='Test unpack-objects when receive pack'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+test_expect_success "create commit with big blobs (1.5 MB)" '
+       test-tool genrandom foo 1500000 >big-blob &&
+       test_commit --append foo big-blob &&
+       test-tool genrandom bar 1500000 >big-blob &&
+       test_commit --append bar big-blob &&
+       (
+               cd .git &&
+               find objects/?? -type f | sort
+       ) >expect &&
+       git repack -ad
+'
+
+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
+       GIT_ALLOC_LIMIT=1m &&
+       export GIT_ALLOC_LIMIT
+'
+
+test_expect_success 'prepare dest repository' '
+       git init --bare dest.git &&
+       git -C dest.git config core.bigFileThreshold 2m &&
+       git -C dest.git config receive.unpacklimit 100
+'
+
+test_expect_success 'fail to push: cannot allocate' '
+       test_must_fail git push dest.git HEAD 2>err &&
+       test_i18ngrep "remote: fatal: attempting to allocate" err &&
+       (
+               cd dest.git &&
+               find objects/?? -type f | sort
+       ) >actual &&
+       ! test_cmp expect actual
+'
+
+test_expect_success 'set a lower bigfile threshold' '
+       git -C dest.git config core.bigFileThreshold 1m
+'
+
+test_expect_success 'unpack big object in stream' '
+       git push dest.git HEAD &&
+       git -C dest.git fsck &&
+       (
+               cd dest.git &&
+               find objects/?? -type f | sort
+       ) >actual &&
+       test_cmp expect actual
+'
+
+test_expect_success 'setup for unpack-objects dry-run test' '
+       PACK=$(echo main | git pack-objects --progress --revs test) &&
+       unset GIT_ALLOC_LIMIT &&
+       git init --bare unpack-test.git
+'
+
+test_expect_success 'unpack-objects dry-run with large threshold' '
+       (
+               cd unpack-test.git &&
+               git config core.bigFileThreshold 2m &&
+               git unpack-objects -n <../test-$PACK.pack
+       ) &&
+       (
+               cd unpack-test.git &&
+               find objects/ -type f
+       ) >actual &&
+       test_must_be_empty actual
+'
+
+test_expect_success 'unpack-objects dry-run with small threshold' '
+       (
+               cd unpack-test.git &&
+               git config core.bigFileThreshold 1m &&
+               git unpack-objects -n <../test-$PACK.pack
+       ) &&
+       (
+               cd unpack-test.git &&
+               find objects/ -type f
+       ) >actual &&
+       test_must_be_empty actual
+'
+
+test_done
--

2.33.0.1.g09a6bb964f.dirty

author
#7 lore raw

PATCH v2 1/6 object-file: refactor write_loose_object() to support inputstream

In reply to Han Xin

From: Han Xin <[email protected]>

Refactor write_loose_object() to support inputstream, in the same way that zlib reading is chunked.

Using "in_stream" instead of "void *buf", we needn't to allocate enough memory in advance, and only part of the contents will be read when called "in_stream.read()".

---
 object-file.c  | 50 ++++++++++++++++++++++++++++++++++++++++++++++----
 object-store.h |  5 +++++
 2 files changed, 51 insertions(+), 4 deletions(-)
object-file.c +46 -4
index 02b7970274..1ad2cb579c 100644
@@ -1860,8 +1860,26 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+struct input_data_from_buffer {
+	const char *buf;
+	unsigned long len;
+};
+
+static const char *read_input_stream_from_buffer(void *data, unsigned long *len)
+{
+	struct input_data_from_buffer *input = (struct input_data_from_buffer *)data;
+
+	if (input->len == 0) {
+		*len = 0;
+		return NULL;
+	}
+	*len = input->len;
+	input->len = 0;
+	return input->buf;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
-			      int hdrlen, const void *buf, unsigned long len,
+			      int hdrlen, struct input_stream *in_stream,
 			      time_t mtime, unsigned flags)
 {
 	int fd, ret;
@@ -1871,6 +1889,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	struct object_id parano_oid;
 	static struct strbuf tmp_file = STRBUF_INIT;
 	static struct strbuf filename = STRBUF_INIT;
+	const char *buf;
+	unsigned long len;
 
 	loose_object_path(the_repository, &filename, oid);
 
@@ -1898,6 +1918,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	the_hash_algo->update_fn(&c, hdr, hdrlen);
 
 	/* Then the data itself.. */
+	buf = in_stream->read(in_stream->data, &len);
 	stream.next_in = (void *)buf;
 	stream.avail_in = len;
 	do {
@@ -1960,6 +1981,13 @@ int write_object_file_flags(const void *buf, unsigned long len,
 {
 	char hdr[MAX_HEADER_LEN];
 	int hdrlen = sizeof(hdr);
+	struct input_stream in_stream = {
+		.read = read_input_stream_from_buffer,
+		.data = (void *)&(struct input_data_from_buffer) {
+			.buf = buf,
+			.len = len,
+		},
+	};
 
 	/* Normally if we have it in the pack then we do not bother writing
 	 * it out into .git/objects/??/?{38} file.
@@ -1968,7 +1996,7 @@ int write_object_file_flags(const void *buf, unsigned long len,
 				  &hdrlen);
 	if (freshen_packed_object(oid) || freshen_loose_object(oid))
 		return 0;
-	return write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags);
+	return write_loose_object(oid, hdr, hdrlen, &in_stream, 0, flags);
 }
 
 int hash_object_file_literally(const void *buf, unsigned long len,
@@ -1977,6 +2005,13 @@ int hash_object_file_literally(const void *buf, unsigned long len,
 {
 	char *header;
 	int hdrlen, status = 0;
+	struct input_stream in_stream = {
+		.read = read_input_stream_from_buffer,
+		.data = (void *)&(struct input_data_from_buffer) {
+			.buf = buf,
+			.len = len,
+		},
+	};
 
 	/* type string, SP, %lu of the length plus NUL must fit this */
 	hdrlen = strlen(type) + MAX_HEADER_LEN;
@@ -1988,7 +2023,7 @@ int hash_object_file_literally(const void *buf, unsigned long len,
 		goto cleanup;
 	if (freshen_packed_object(oid) || freshen_loose_object(oid))
 		goto cleanup;
-	status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0);
+	status = write_loose_object(oid, header, hdrlen, &in_stream, 0, 0);
 
 cleanup:
 	free(header);
@@ -2003,14 +2038,21 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
 	char hdr[MAX_HEADER_LEN];
 	int hdrlen;
 	int ret;
+	struct input_data_from_buffer data;
+	struct input_stream in_stream = {
+		.read = read_input_stream_from_buffer,
+		.data = &data,
+	};
 
 	if (has_loose_object(oid))
 		return 0;
 	buf = read_object(the_repository, oid, &type, &len);
 	if (!buf)
 		return error(_("cannot read object for %s"), oid_to_hex(oid));
+	data.buf = buf;
+	data.len = len;
 	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
-	ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0);
+	ret = write_loose_object(oid, hdr, hdrlen, &in_stream, mtime, 0);
 	free(buf);
 
 	return ret;
object-store.h +5 -0
index 952efb6a4b..f1b67e9100 100644
@@ -34,6 +34,11 @@ struct object_directory {
 	char *path;
 };
 
+struct input_stream {
+	const char *(*read)(void* data, unsigned long *len);
+	void *data;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
#8 lore raw

Re: [PATCH v2 1/6] object-file: refactor write_loose_object() to support inputstream

Quoted 2 lines · Han Xin

From: Han Xin <[email protected]>

It would be better to provide a cover letter describing changes in v2, such as:

* Make "write_loose_object()" a public method, so we can reuse it in "unpack_non_delta_entry()".
(But I doubt we can use "write_object_file_flags()" public
function, without make this change.)

* Add an new interface "input_stream" as an argument for "write_loose_object()", so that we can feed data to "write_loose_object()" from buffer or from zlib stream.

Quoted 2 lines

Refactor write_loose_object() to support inputstream, in the same way that zlib reading is chunked.

In the beginning of your commit log, you should describe the problem, such as:

We used to read the full content of a blob into buffer in "unpack_non_delta_entry()" by calling:

    void *buf = get_data(size);

This will consume lots of memory for a very big blob object.

Quoted 16 lines

Using "in_stream" instead of "void *buf", we needn't to allocate enough memory in advance, and only part of the contents will be read when called "in_stream.read()".

---
 object-file.c  | 50 ++++++++++++++++++++++++++++++++++++++++++++++----
 object-store.h |  5 +++++
 2 files changed, 51 insertions(+), 4 deletions(-)
object-file.c +0 -0
index 02b7970274..1ad2cb579c 100644

object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1868

Quoted 9 lines
object-file.c +6 -0
@@ -1860,8 +1860,26 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
        return fd;
 }
 
+struct input_data_from_buffer {
+       const char *buf;
+       unsigned long len;
+};
+
+static const char *read_input_stream_from_buffer(void *data, unsigned long *len)

Use "const void *" for the type of return variable, just like input argument for write_loose_object()?

Quoted 19 lines
object-file.c +13 -1
+{
+       struct input_data_from_buffer *input = (struct input_data_from_buffer *)data;
+
+       if (input->len == 0) {
+               *len = 0;
+               return NULL;
+       }
+       *len = input->len;
+       input->len = 0;
+       return input->buf;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
-                             int hdrlen, const void *buf, unsigned long len,
+                             int hdrlen, struct input_stream *in_stream,
                              time_t mtime, unsigned flags)
 {
        int fd, ret;

object-file.c static int write_loose_object(const struct object_id *oid, char *hdr, · line 1892

Quoted 4 lines
object-file.c +1 -0
@@ -1871,6 +1889,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        struct object_id parano_oid;
        static struct strbuf tmp_file = STRBUF_INIT;
        static struct strbuf filename = STRBUF_INIT;
+       const char *buf;

Can we use the same prototype as the original: "const void *buf" ?

Quoted 100 lines
object-file.c +26 -3
+       unsigned long len;
 
        loose_object_path(the_repository, &filename, oid);
 
@@ -1898,6 +1918,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        the_hash_algo->update_fn(&c, hdr, hdrlen);
 
        /* Then the data itself.. */
+       buf = in_stream->read(in_stream->data, &len);
        stream.next_in = (void *)buf;
        stream.avail_in = len;
        do {
@@ -1960,6 +1981,13 @@ int write_object_file_flags(const void *buf, unsigned long len,
 {
        char hdr[MAX_HEADER_LEN];
        int hdrlen = sizeof(hdr);
+       struct input_stream in_stream = {
+               .read = read_input_stream_from_buffer,
+               .data = (void *)&(struct input_data_from_buffer) {
+                       .buf = buf,
+                       .len = len,
+               },
+       };
 
        /* Normally if we have it in the pack then we do not bother writing
         * it out into .git/objects/??/?{38} file.
@@ -1968,7 +1996,7 @@ int write_object_file_flags(const void *buf, unsigned long len,
                                  &hdrlen);
        if (freshen_packed_object(oid) || freshen_loose_object(oid))
                return 0;
-       return write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags);
+       return write_loose_object(oid, hdr, hdrlen, &in_stream, 0, flags);
 }
 
 int hash_object_file_literally(const void *buf, unsigned long len,
@@ -1977,6 +2005,13 @@ int hash_object_file_literally(const void *buf, unsigned long len,
 {
        char *header;
        int hdrlen, status = 0;
+       struct input_stream in_stream = {
+               .read = read_input_stream_from_buffer,
+               .data = (void *)&(struct input_data_from_buffer) {
+                       .buf = buf,
+                       .len = len,
+               },
+       };
 
        /* type string, SP, %lu of the length plus NUL must fit this */
        hdrlen = strlen(type) + MAX_HEADER_LEN;
@@ -1988,7 +2023,7 @@ int hash_object_file_literally(const void *buf, unsigned long len,
                goto cleanup;
        if (freshen_packed_object(oid) || freshen_loose_object(oid))
                goto cleanup;
-       status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0);
+       status = write_loose_object(oid, header, hdrlen, &in_stream, 0, 0);
 
 cleanup:
        free(header);
@@ -2003,14 +2038,21 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
        char hdr[MAX_HEADER_LEN];
        int hdrlen;
        int ret;
+       struct input_data_from_buffer data;
+       struct input_stream in_stream = {
+               .read = read_input_stream_from_buffer,
+               .data = &data,
+       };
 
        if (has_loose_object(oid))
                return 0;
        buf = read_object(the_repository, oid, &type, &len);
        if (!buf)
                return error(_("cannot read object for %s"), oid_to_hex(oid));
+       data.buf = buf;
+       data.len = len;
        hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
-       ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0);
+       ret = write_loose_object(oid, hdr, hdrlen, &in_stream, mtime, 0);
        free(buf);
 
        return ret;
object-store.h +5 -1
index 952efb6a4b..f1b67e9100 100644
@@ -34,6 +34,11 @@ struct object_directory {
        char *path;
 };
 
+struct input_stream {
+       const char *(*read)(void* data, unsigned long *len);
+       void *data;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
        struct object_directory *, 1, fspathhash, fspatheq)
 
--

2.33.1.44.g9344627884.agit.6.5.4

#9 lore raw

Re: [PATCH v2 1/6] object-file: refactor write_loose_object() to support inputstream

Quoted 18 lines · Jiang Xin
Quoted 2 lines · Han Xin

From: Han Xin <[email protected]>

It would be better to provide a cover letter describing changes in v2, such as:

* Make "write_loose_object()" a public method, so we can reuse it in "unpack_non_delta_entry()".
(But I doubt we can use "write_object_file_flags()" public
function, without make this change.)

* Add an new interface "input_stream" as an argument for "write_loose_object()", so that we can feed data to "write_loose_object()" from buffer or from zlib stream.

Quoted 2 lines

Refactor write_loose_object() to support inputstream, in the same way that zlib reading is chunked.

Quoted 8 lines

In the beginning of your commit log, you should describe the problem, such as:

We used to read the full content of a blob into buffer in "unpack_non_delta_entry()" by calling:

    void *buf = get_data(size);

This will consume lots of memory for a very big blob object.

I was not sure where "in_stream" came from---"use X insteads of Y", when X is what these patches invent and introduce, does not make a good explanation without explaining what X is, what problem X is attempting to solve and how.

Thanks for helping to clarify the proposed log message.

author
#10 lore raw

PATCH v2 2/6 object-file.c: add dry_run mode for write_loose_object()

In reply to Han Xin

From: Han Xin <[email protected]>

We will use "write_loose_object()" later to handle large blob object, which needs to work in dry_run mode.

---
 object-file.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)
object-file.c +19 -13
index 1ad2cb579c..b0838c847e 100644
@@ -1880,9 +1880,10 @@ static const char *read_input_stream_from_buffer(void *data, unsigned long *len)
 
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, struct input_stream *in_stream,
+			      int dry_run,
 			      time_t mtime, unsigned flags)
 {
-	int fd, ret;
+	int fd, ret = 0;
 	unsigned char compressed[4096];
 	git_zstream stream;
 	git_hash_ctx c;
@@ -1894,14 +1895,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
+	if (!dry_run) {
+		fd = create_tmpfile(&tmp_file, filename.buf);
+		if (fd < 0) {
+			if (flags & HASH_SILENT)
+				return -1;
+			else if (errno == EACCES)
+				return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
+			else
+				return error_errno(_("unable to create temporary file"));
+		}
 	}
 
 	/* Set it up */
@@ -1925,7 +1928,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		unsigned char *in0 = stream.next_in;
 		ret = git_deflate(&stream, Z_FINISH);
 		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
+		if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0)
 			die(_("unable to write loose object file"));
 		stream.next_out = compressed;
 		stream.avail_out = sizeof(compressed);
@@ -1943,6 +1946,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
+	if (dry_run)
+		return 0;
+
 	close_loose_object(fd);
 
 	if (mtime) {
@@ -1996,7 +2002,7 @@ int write_object_file_flags(const void *buf, unsigned long len,
 				  &hdrlen);
 	if (freshen_packed_object(oid) || freshen_loose_object(oid))
 		return 0;
-	return write_loose_object(oid, hdr, hdrlen, &in_stream, 0, flags);
+	return write_loose_object(oid, hdr, hdrlen, &in_stream, 0, 0, flags);
 }
 
 int hash_object_file_literally(const void *buf, unsigned long len,
@@ -2023,7 +2029,7 @@ int hash_object_file_literally(const void *buf, unsigned long len,
 		goto cleanup;
 	if (freshen_packed_object(oid) || freshen_loose_object(oid))
 		goto cleanup;
-	status = write_loose_object(oid, header, hdrlen, &in_stream, 0, 0);
+	status = write_loose_object(oid, header, hdrlen, &in_stream, 0, 0, 0);
 
 cleanup:
 	free(header);
@@ -2052,7 +2058,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
 	data.buf = buf;
 	data.len = len;
 	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
-	ret = write_loose_object(oid, hdr, hdrlen, &in_stream, mtime, 0);
+	ret = write_loose_object(oid, hdr, hdrlen, &in_stream, 0, mtime, 0);
 	free(buf);
 
 	return ret;
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
#11 lore raw

Re: [PATCH v2 2/6] object-file.c: add dry_run mode for write_loose_object()

Quoted 5 lines · Han Xin

From: Han Xin <[email protected]>

We will use "write_loose_object()" later to handle large blob object, which needs to work in dry_run mode.

The dry_run mode comes from "builtin/unpack-object.c", throw the buffer read from "get_data()".
So why not add "dry_run" to "get_data()" instead?

If we have a dry_run version of get_data, such as "get_data(size, dry_run)", we do not have to add dry_run mode for ” write_loose_object()".

See: git grep -A5 get_data builtin/unpack-objects.c
builtin/unpack-objects.c: void *buf = get_data(size);
builtin/unpack-objects.c-
builtin/unpack-objects.c- if (!dry_run && buf)
builtin/unpack-objects.c- write_object(nr, type, buf, size);
builtin/unpack-objects.c- else
builtin/unpack-objects.c- free(buf);
--
builtin/unpack-objects.c: delta_data = get_data(delta_size);
builtin/unpack-objects.c- if (dry_run || !delta_data) {
builtin/unpack-objects.c- free(delta_data);
builtin/unpack-objects.c- return;
builtin/unpack-objects.c- }
--
builtin/unpack-objects.c: delta_data = get_data(delta_size);
builtin/unpack-objects.c- if (dry_run || !delta_data) {
builtin/unpack-objects.c- free(delta_data);
builtin/unpack-objects.c- return;
builtin/unpack-objects.c- }

Quoted 96 lines
---
 object-file.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)
object-file.c +19 -14
index 1ad2cb579c..b0838c847e 100644
@@ -1880,9 +1880,10 @@ static const char *read_input_stream_from_buffer(void *data, unsigned long *len)
 
 static int write_loose_object(const struct object_id *oid, char *hdr,
                              int hdrlen, struct input_stream *in_stream,
+                             int dry_run,
                              time_t mtime, unsigned flags)
 {
-       int fd, ret;
+       int fd, ret = 0;
        unsigned char compressed[4096];
        git_zstream stream;
        git_hash_ctx c;
@@ -1894,14 +1895,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
        loose_object_path(the_repository, &filename, oid);
 
-       fd = create_tmpfile(&tmp_file, filename.buf);
-       if (fd < 0) {
-               if (flags & HASH_SILENT)
-                       return -1;
-               else if (errno == EACCES)
-                       return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-               else
-                       return error_errno(_("unable to create temporary file"));
+       if (!dry_run) {
+               fd = create_tmpfile(&tmp_file, filename.buf);
+               if (fd < 0) {
+                       if (flags & HASH_SILENT)
+                               return -1;
+                       else if (errno == EACCES)
+                               return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
+                       else
+                               return error_errno(_("unable to create temporary file"));
+               }
        }
 
        /* Set it up */
@@ -1925,7 +1928,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
                unsigned char *in0 = stream.next_in;
                ret = git_deflate(&stream, Z_FINISH);
                the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-               if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
+               if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0)
                        die(_("unable to write loose object file"));
                stream.next_out = compressed;
                stream.avail_out = sizeof(compressed);
@@ -1943,6 +1946,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
                die(_("confused by unstable object source data for %s"),
                    oid_to_hex(oid));
 
+       if (dry_run)
+               return 0;
+
        close_loose_object(fd);
 
        if (mtime) {
@@ -1996,7 +2002,7 @@ int write_object_file_flags(const void *buf, unsigned long len,
                                  &hdrlen);
        if (freshen_packed_object(oid) || freshen_loose_object(oid))
                return 0;
-       return write_loose_object(oid, hdr, hdrlen, &in_stream, 0, flags);
+       return write_loose_object(oid, hdr, hdrlen, &in_stream, 0, 0, flags);
 }
 
 int hash_object_file_literally(const void *buf, unsigned long len,
@@ -2023,7 +2029,7 @@ int hash_object_file_literally(const void *buf, unsigned long len,
                goto cleanup;
        if (freshen_packed_object(oid) || freshen_loose_object(oid))
                goto cleanup;
-       status = write_loose_object(oid, header, hdrlen, &in_stream, 0, 0);
+       status = write_loose_object(oid, header, hdrlen, &in_stream, 0, 0, 0);
 
 cleanup:
        free(header);
@@ -2052,7 +2058,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
        data.buf = buf;
        data.len = len;
        hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
-       ret = write_loose_object(oid, hdr, hdrlen, &in_stream, mtime, 0);
+       ret = write_loose_object(oid, hdr, hdrlen, &in_stream, 0, mtime, 0);
        free(buf);
 
        return ret;
--

2.33.1.44.g9344627884.agit.6.5.4

author
#12 lore raw

PATCH v2 3/6 object-file.c: handle nil oid in write_loose_object()

In reply to Han Xin

From: Han Xin <[email protected]>

When read input stream, oid can't get before reading all, and it will be filled after reading.

---
 object-file.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)
object-file.c +32 -2
index b0838c847e..8393659f0d 100644
@@ -1893,7 +1893,13 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	const char *buf;
 	unsigned long len;
 
-	loose_object_path(the_repository, &filename, oid);
+	if (is_null_oid(oid)) {
+		/* When oid is not determined, save tmp file to odb path. */
+		strbuf_reset(&filename);
+		strbuf_addstr(&filename, the_repository->objects->odb->path);
+		strbuf_addch(&filename, '/');
+	} else
+		loose_object_path(the_repository, &filename, oid);
 
 	if (!dry_run) {
 		fd = create_tmpfile(&tmp_file, filename.buf);
@@ -1942,7 +1948,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
 		    ret);
 	the_hash_algo->final_oid_fn(&parano_oid, &c);
-	if (!oideq(oid, &parano_oid))
+	if (!is_null_oid(oid) && !oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
@@ -1951,6 +1957,30 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	close_loose_object(fd);
 
+	if (is_null_oid(oid)) {
+		int dirlen;
+
+		/* copy oid */
+		oidcpy((struct object_id *)oid, &parano_oid);
+		/* We get the oid now */
+		loose_object_path(the_repository, &filename, oid);
+
+		dirlen = directory_size(filename.buf);
+		if (dirlen) {
+			struct strbuf dir = STRBUF_INIT;
+			/*
+			 * Make sure the directory exists; note that the
+			 * contents of the buffer are undefined after mkstemp
+			 * returns an error, so we have to rewrite the whole
+			 * buffer from scratch.
+			 */
+			strbuf_reset(&dir);
+			strbuf_add(&dir, filename.buf, dirlen - 1);
+			if (mkdir(dir.buf, 0777) && errno != EEXIST)
+				return -1;
+		}
+	}
+
 	if (mtime) {
 		struct utimbuf utb;
 		utb.actime = mtime;
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
#13 lore raw

Re: [PATCH v2 3/6] object-file.c: handle nil oid in write_loose_object()

Quoted 5 lines · Han Xin

From: Han Xin <[email protected]>

When read input stream, oid can't get before reading all, and it will be filled after reading.

Under what circumstances is the oid a null oid? Can we get the oid from “obj_list[nr].oid” ?
See unpack_non_delta_entry() of builtin/unpack-objects.c.

Quoted 68 lines
---
 object-file.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)
object-file.c +32 -3
index b0838c847e..8393659f0d 100644
@@ -1893,7 +1893,13 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        const char *buf;
        unsigned long len;
 
-       loose_object_path(the_repository, &filename, oid);
+       if (is_null_oid(oid)) {
+               /* When oid is not determined, save tmp file to odb path. */
+               strbuf_reset(&filename);
+               strbuf_addstr(&filename, the_repository->objects->odb->path);
+               strbuf_addch(&filename, '/');
+       } else
+               loose_object_path(the_repository, &filename, oid);
 
        if (!dry_run) {
                fd = create_tmpfile(&tmp_file, filename.buf);
@@ -1942,7 +1948,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
                die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
                    ret);
        the_hash_algo->final_oid_fn(&parano_oid, &c);
-       if (!oideq(oid, &parano_oid))
+       if (!is_null_oid(oid) && !oideq(oid, &parano_oid))
                die(_("confused by unstable object source data for %s"),
                    oid_to_hex(oid));
 
@@ -1951,6 +1957,30 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
        close_loose_object(fd);
 
+       if (is_null_oid(oid)) {
+               int dirlen;
+
+               /* copy oid */
+               oidcpy((struct object_id *)oid, &parano_oid);
+               /* We get the oid now */
+               loose_object_path(the_repository, &filename, oid);
+
+               dirlen = directory_size(filename.buf);
+               if (dirlen) {
+                       struct strbuf dir = STRBUF_INIT;
+                       /*
+                        * Make sure the directory exists; note that the
+                        * contents of the buffer are undefined after mkstemp
+                        * returns an error, so we have to rewrite the whole
+                        * buffer from scratch.
+                        */
+                       strbuf_reset(&dir);
+                       strbuf_add(&dir, filename.buf, dirlen - 1);
+                       if (mkdir(dir.buf, 0777) && errno != EEXIST)
+                               return -1;
+               }
+       }
+
        if (mtime) {
                struct utimbuf utb;
                utb.actime = mtime;
--

2.33.1.44.g9344627884.agit.6.5.4

author
#14 lore raw

PATCH v2 4/6 object-file.c: read input stream repeatedly in write_loose_object()

In reply to Han Xin

From: Han Xin <[email protected]>

Read input stream repeatedly in write_loose_object() unless reach the end, so that we can divide the large blob write into many small blocks.

---
 object-file.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
object-file.c +9 -5
index 8393659f0d..e333448c54 100644
@@ -1891,7 +1891,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	static struct strbuf tmp_file = STRBUF_INIT;
 	static struct strbuf filename = STRBUF_INIT;
 	const char *buf;
-	unsigned long len;
+	int flush = 0;
 
 	if (is_null_oid(oid)) {
 		/* When oid is not determined, save tmp file to odb path. */
@@ -1927,12 +1927,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	the_hash_algo->update_fn(&c, hdr, hdrlen);
 
 	/* Then the data itself.. */
-	buf = in_stream->read(in_stream->data, &len);
-	stream.next_in = (void *)buf;
-	stream.avail_in = len;
 	do {
 		unsigned char *in0 = stream.next_in;
-		ret = git_deflate(&stream, Z_FINISH);
+		if (!stream.avail_in) {
+			if ((buf = in_stream->read(in_stream->data, &stream.avail_in))) {
+				stream.next_in = (void *)buf;
+				in0 = (unsigned char *)buf;
+			} else
+				flush = Z_FINISH;
+		}
+		ret = git_deflate(&stream, flush);
 		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
 		if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0)
 			die(_("unable to write loose object file"));
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
#15 lore raw

Re: [PATCH v2 4/6] object-file.c: read input stream repeatedly in write_loose_object()

Quoted 5 lines · Han Xin

From: Han Xin <[email protected]>

Read input stream repeatedly in write_loose_object() unless reach the end, so that we can divide the large blob write into many small blocks.

In order to prepare the stream version of "write_loose_object()", we need ...

Quoted 21 lines
---
 object-file.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
object-file.c +1 -1
index 8393659f0d..e333448c54 100644
@@ -1891,7 +1891,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        static struct strbuf tmp_file = STRBUF_INIT;
        static struct strbuf filename = STRBUF_INIT;
        const char *buf;
-       unsigned long len;
+       int flush = 0;
 
        if (is_null_oid(oid)) {
                /* When oid is not determined, save tmp file to odb path. */
@@ -1927,12 +1927,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        the_hash_algo->update_fn(&c, hdr, hdrlen);

object-file.c static int write_loose_object(const struct object_id *oid, char *hdr, · line 1933

Quoted 10 lines
object-file.c +2 -4
 
        /* Then the data itself.. */
-       buf = in_stream->read(in_stream->data, &len);
-       stream.next_in = (void *)buf;
-       stream.avail_in = len;
        do {
                unsigned char *in0 = stream.next_in;
-               ret = git_deflate(&stream, Z_FINISH);
+               if (!stream.avail_in) {
+                       if ((buf = in_stream->read(in_stream->data, &stream.avail_in))) {

if ((buf = in_stream->read(in_stream->data, &stream.avail_in)) != NULL) {

Or split this long line into:

    buf = in_stream->read(in_stream->data, &stream.avail_in);
    if (buf) {

object-file.c static int write_loose_object(const struct object_id *oid, char *hdr, · line 1937

Quoted 4 lines
object-file.c +4 -0
+                               stream.next_in = (void *)buf;
+                               in0 = (unsigned char *)buf;
+                       } else
+                               flush = Z_FINISH;

Add {} around this single line, see:

https://github.com/git/git/blob/master/Documentation/CodingGuidelines#L279-L289

Quoted 8 lines
object-file.c +2 -1
+               }
+               ret = git_deflate(&stream, flush);
                the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
                if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0)
                        die(_("unable to write loose object file"));
--

2.33.1.44.g9344627884.agit.6.5.4

author
#16 lore raw

PATCH v2 5/6 object-store.h: add write_loose_object()

In reply to Han Xin

From: Han Xin <[email protected]>

For large loose object files, that should be possible to stream it direct to disk with "write_loose_object()".
Unlike "write_object_file()", you need to implement an "input_stream" instead of giving void *buf.

---
 object-file.c  | 8 ++++----
 object-store.h | 5 +++++
 2 files changed, 9 insertions(+), 4 deletions(-)
object-file.c +4 -4
index e333448c54..60eb29db97 100644
@@ -1878,10 +1878,10 @@ static const char *read_input_stream_from_buffer(void *data, unsigned long *len)
 	return input->buf;
 }
 
-static int write_loose_object(const struct object_id *oid, char *hdr,
-			      int hdrlen, struct input_stream *in_stream,
-			      int dry_run,
-			      time_t mtime, unsigned flags)
+int write_loose_object(const struct object_id *oid, char *hdr,
+		       int hdrlen, struct input_stream *in_stream,
+		       int dry_run,
+		       time_t mtime, unsigned flags)
 {
 	int fd, ret = 0;
 	unsigned char compressed[4096];
object-store.h +5 -0
index f1b67e9100..f6faa8d6d3 100644
@@ -228,6 +228,11 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,
 		     unsigned long len, const char *type,
 		     struct object_id *oid);
 
+int write_loose_object(const struct object_id *oid, char *hdr,
+		       int hdrlen, struct input_stream *in_stream,
+		       int dry_run,
+		       time_t mtime, unsigned flags);
+
 int write_object_file_flags(const void *buf, unsigned long len,
 			    const char *type, struct object_id *oid,
 			    unsigned flags);
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
author
#17 lore raw

PATCH v2 6/6 unpack-objects: unpack large object in stream

In reply to Han Xin

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use "core.bigFileThreshold" to avoid OOM limits when called "get_data()".

---
 builtin/unpack-objects.c          | 76 ++++++++++++++++++++++++-
 t/t5590-receive-unpack-objects.sh | 92 +++++++++++++++++++++++++++++++
 2 files changed, 167 insertions(+), 1 deletion(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +75 -1
index 4a9466295b..6c757d823b 100644
@@ -320,11 +320,85 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
+struct input_data_from_zstream {
+	git_zstream *zstream;
+	unsigned char buf[4096];
+	int status;
+};
+
+static const char *read_inflate_in_stream(void *data, unsigned long *readlen)
+{
+	struct input_data_from_zstream *input = data;
+	git_zstream *zstream = input->zstream;
+	void *in = fill(1);
+
+	if (!len || input->status == Z_STREAM_END) {
+		*readlen = 0;
+		return NULL;
+	}
+
+	zstream->next_out = input->buf;
+	zstream->avail_out = sizeof(input->buf);
+	zstream->next_in = in;
+	zstream->avail_in = len;
+
+	input->status = git_inflate(zstream, 0);
+	use(len - zstream->avail_in);
+	*readlen = sizeof(input->buf) - zstream->avail_out;
+
+	return (const char *)input->buf;
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
+	char hdr[32];
+	int hdrlen;
+	git_zstream zstream;
+	struct input_data_from_zstream data;
+	struct input_stream in_stream = {
+		.read = read_inflate_in_stream,
+		.data = &data,
+	};
+	struct object_id *oid = &obj_list[nr].oid;
+	int ret;
+
+	memset(&zstream, 0, sizeof(zstream));
+	memset(&data, 0, sizeof(data));
+	data.zstream = &zstream;
+	git_inflate_init(&zstream);
+
+	/* Generate the header */
+	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), (uintmax_t)size) + 1;
+
+	if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, dry_run, 0, 0)))
+		die(_("failed to write object in stream %d"), ret);
+
+	if (zstream.total_out != size || data.status != Z_STREAM_END)
+		die(_("inflate returned %d"), data.status);
+	git_inflate_end(&zstream);
+
+	if (strict && !dry_run) {
+		struct blob *blob = lookup_blob(the_repository, oid);
+		if (blob)
+			blob->object.flags |= FLAG_WRITTEN;
+		else
+			die("invalid blob object from stream");
+	}
+	obj_list[nr].obj = NULL;
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 				   unsigned nr)
 {
-	void *buf = get_data(size);
+	void *buf;
+
+	/* Write large blob in stream without allocating full buffer. */
+	if (type == OBJ_BLOB && size > big_file_threshold) {
+		write_stream_blob(nr, size);
+		return;
+	}
 
+	buf = get_data(size);
 	if (!dry_run && buf)
 		write_object(nr, type, buf, size);
 	else
t/t5590-receive-unpack-objects.sh new file +92 -0

92 lines not on this page · raw message

Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
#18 lore raw
Quoted 96 lines · Han Xin

From: Han Xin <[email protected]>

When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.

In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.

To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use "core.bigFileThreshold" to avoid OOM limits when called "get_data()".

---
 builtin/unpack-objects.c          | 76 ++++++++++++++++++++++++-
 t/t5590-receive-unpack-objects.sh | 92 +++++++++++++++++++++++++++++++
 2 files changed, 167 insertions(+), 1 deletion(-)
 create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +65 -0
index 4a9466295b..6c757d823b 100644
@@ -320,11 +320,85 @@ static void added_object(unsigned nr, enum object_type type,
        }
 }
 
+struct input_data_from_zstream {
+       git_zstream *zstream;
+       unsigned char buf[4096];
+       int status;
+};
+
+static const char *read_inflate_in_stream(void *data, unsigned long *readlen)
+{
+       struct input_data_from_zstream *input = data;
+       git_zstream *zstream = input->zstream;
+       void *in = fill(1);
+
+       if (!len || input->status == Z_STREAM_END) {
+               *readlen = 0;
+               return NULL;
+       }
+
+       zstream->next_out = input->buf;
+       zstream->avail_out = sizeof(input->buf);
+       zstream->next_in = in;
+       zstream->avail_in = len;
+
+       input->status = git_inflate(zstream, 0);
+       use(len - zstream->avail_in);
+       *readlen = sizeof(input->buf) - zstream->avail_out;
+
+       return (const char *)input->buf;
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
+       char hdr[32];
+       int hdrlen;
+       git_zstream zstream;
+       struct input_data_from_zstream data;
+       struct input_stream in_stream = {
+               .read = read_inflate_in_stream,
+               .data = &data,
+       };
+       struct object_id *oid = &obj_list[nr].oid;
+       int ret;
+
+       memset(&zstream, 0, sizeof(zstream));
+       memset(&data, 0, sizeof(data));
+       data.zstream = &zstream;
+       git_inflate_init(&zstream);
+
+       /* Generate the header */
+       hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), (uintmax_t)size) + 1;
+
+       if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, dry_run, 0, 0)))
+               die(_("failed to write object in stream %d"), ret);
+
+       if (zstream.total_out != size || data.status != Z_STREAM_END)
+               die(_("inflate returned %d"), data.status);
+       git_inflate_end(&zstream);
+
+       if (strict && !dry_run) {
+               struct blob *blob = lookup_blob(the_repository, oid);
+               if (blob)
+                       blob->object.flags |= FLAG_WRITTEN;
+               else
+                       die("invalid blob object from stream");
+       }
+       obj_list[nr].obj = NULL;

builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 396

Quoted 10 lines
builtin/unpack-objects.c +6 -1
+}
+
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
                                   unsigned nr)
 {
-       void *buf = get_data(size);
+       void *buf;
+
+       /* Write large blob in stream without allocating full buffer. */
+       if (type == OBJ_BLOB && size > big_file_threshold) {

Default size of big_file_threshold is 512m. Can we use "write_stream_blob" for all objects? Can we get a more suitable threshold through some benchmark data?

builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 399

Quoted 3 lines
builtin/unpack-objects.c +3 -0
+               write_stream_blob(nr, size);
+               return;
+       }

--
Jiang Xin

author
#19 lore raw

v3 cover unpack large objects in stream

In reply to Han Xin

From: Han Xin <[email protected]>

Although we do not recommend users push large binary files to the git repositories, it's difficult to prevent them from doing so. Once, we found a problem with a surge in memory usage on the server. The source of the problem is that a user submitted a single object with a size of 15GB. Once someone initiates a git push, the git process will immediately allocate 15G of memory, resulting in an OOM risk.

Through further analysis, we found that when we execute git unpack-objects, in unpack_non_delta_entry(), "void *buf = get_data(size);" will directly allocate memory equal to the size of the object. This is quite a scary thing, because the pre-receive hook has not been executed at this time, and we cannot avoid this by hooks.

I got inspiration from the deflate process of zlib, maybe it would be a good idea to change unpack-objects to stream deflate.

Changes since v2:
* Rewrite commit messages and make changes suggested by Jiang Xin.
* Remove the commit "object-file.c: add dry_run mode for write_loose_object()" and use a new commit "unpack-objects.c: add dry_run mode for get_data()" instead.

Han Xin (5):
object-file: refactor write_loose_object() to read buffer from stream object-file.c: handle undetermined oid in write_loose_object() object-file.c: read stream in a loop in write_loose_object() unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream

 builtin/unpack-objects.c            | 92 +++++++++++++++++++++++++--
 object-file.c                       | 98 +++++++++++++++++++++++++----
 object-store.h                      |  9 +++
 t/t5590-unpack-non-delta-objects.sh | 76 ++++++++++++++++++++++
 4 files changed, 257 insertions(+), 18 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh

Range-diff against v2:
1: 01672f50a0 ! 1: 8640b04f6d object-file: refactor write_loose_object() to support inputstream

    @@ Metadata
     Author: Han Xin <[email protected]>
     
      ## Commit message ##
    -    object-file: refactor write_loose_object() to support inputstream
    +    object-file: refactor write_loose_object() to read buffer from stream
     
    -    Refactor write_loose_object() to support inputstream, in the same way
    -    that zlib reading is chunked.
    +    We used to call "get_data()" in "unpack_non_delta_entry()" to read the
    +    entire contents of a blob object, no matter how big it is. This
    +    implementation may consume all the memory and cause OOM.
     
    -    Using "in_stream" instead of "void *buf", we needn't to allocate enough
    -    memory in advance, and only part of the contents will be read when
    -    called "in_stream.read()".
    +    This can be improved by feeding data to "write_loose_object()" in a
    +    stream. The input stream is implemented as an interface. In the first
    +    step, we make a simple implementation, feeding the entire buffer in the
    +    "stream" to "write_loose_object()" as a refactor.
     
         Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
    @@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
      	return fd;
      }
      
    -+struct input_data_from_buffer {
    -+	const char *buf;
    ++struct simple_input_stream_data {
    ++	const void *buf;
     +	unsigned long len;
     +};
     +
    -+static const char *read_input_stream_from_buffer(void *data, unsigned long *len)
    ++static const void *feed_simple_input_stream(struct input_stream *in_stream, unsigned long *len)
     +{
    -+	struct input_data_from_buffer *input = (struct input_data_from_buffer *)data;
    ++	struct simple_input_stream_data *data = in_stream->data;
     +
    -+	if (input->len == 0) {
    ++	if (data->len == 0) {
     +		*len = 0;
     +		return NULL;
     +	}
    -+	*len = input->len;
    -+	input->len = 0;
    -+	return input->buf;
    ++	*len = data->len;
    ++	data->len = 0;
    ++	return data->buf;
     +}
     +
      static int write_loose_object(const struct object_id *oid, char *hdr,
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      	struct object_id parano_oid;
      	static struct strbuf tmp_file = STRBUF_INIT;
      	static struct strbuf filename = STRBUF_INIT;
    -+	const char *buf;
    ++	const void *buf;
     +	unsigned long len;
      
      	loose_object_path(the_repository, &filename, oid);
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      	the_hash_algo->update_fn(&c, hdr, hdrlen);
      
      	/* Then the data itself.. */
    -+	buf = in_stream->read(in_stream->data, &len);
    ++	buf = in_stream->read(in_stream, &len);
      	stream.next_in = (void *)buf;
      	stream.avail_in = len;
      	do {
    @@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
      	char hdr[MAX_HEADER_LEN];
      	int hdrlen = sizeof(hdr);
     +	struct input_stream in_stream = {
    -+		.read = read_input_stream_from_buffer,
    -+		.data = (void *)&(struct input_data_from_buffer) {
    ++		.read = feed_simple_input_stream,
    ++		.data = (void *)&(struct simple_input_stream_data) {
     +			.buf = buf,
     +			.len = len,
     +		},
    @@ object-file.c: int hash_object_file_literally(const void *buf, unsigned long len
      	char *header;
      	int hdrlen, status = 0;
     +	struct input_stream in_stream = {
    -+		.read = read_input_stream_from_buffer,
    -+		.data = (void *)&(struct input_data_from_buffer) {
    ++		.read = feed_simple_input_stream,
    ++		.data = (void *)&(struct simple_input_stream_data) {
     +			.buf = buf,
     +			.len = len,
     +		},
    @@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
      	char hdr[MAX_HEADER_LEN];
      	int hdrlen;
      	int ret;
    -+	struct input_data_from_buffer data;
    ++	struct simple_input_stream_data data;
     +	struct input_stream in_stream = {
    -+		.read = read_input_stream_from_buffer,
    ++		.read = feed_simple_input_stream,
     +		.data = &data,
     +	};
      
    @@ object-store.h: struct object_directory {
      };
      
     +struct input_stream {
    -+	const char *(*read)(void* data, unsigned long *len);
    ++	const void *(*read)(struct input_stream *, unsigned long *len);
     +	void *data;
     +};
     +
2:  a309b7e391 < -:  ---------- object-file.c: add dry_run mode for write_loose_object()
3:  b0a5b53710 ! 2:  d4a2caf2bd object-file.c: handle nil oid in write_loose_object()
    @@ Metadata
     Author: Han Xin <[email protected]>
     
      ## Commit message ##
    -    object-file.c: handle nil oid in write_loose_object()
    +    object-file.c: handle undetermined oid in write_loose_object()
     
    -    When read input stream, oid can't get before reading all, and it will be
    -    filled after reading.
    +    When streaming a large blob object to "write_loose_object()", we have no
    +    chance to run "write_object_file_prepare()" to calculate the oid in
    +    advance. So we need to handle undetermined oid in function
    +    "write_loose_object()".
    +
    +    In the original implementation, we know the oid and we can write the
    +    temporary file in the same directory as the final object, but for an
    +    object with an undetermined oid, we don't know the exact directory for
    +    the object, so we have to save the temporary file in ".git/objects/"
    +    directory instead.
     
         Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
     
      ## object-file.c ##
     @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
    - 	const char *buf;
    + 	const void *buf;
      	unsigned long len;
      
     -	loose_object_path(the_repository, &filename, oid);
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
     +		strbuf_reset(&filename);
     +		strbuf_addstr(&filename, the_repository->objects->odb->path);
     +		strbuf_addch(&filename, '/');
    -+	} else
    ++	} else {
     +		loose_object_path(the_repository, &filename, oid);
    ++	}
      
    - 	if (!dry_run) {
    - 		fd = create_tmpfile(&tmp_file, filename.buf);
    + 	fd = create_tmpfile(&tmp_file, filename.buf);
    + 	if (fd < 0) {
     @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
      		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
      		    ret);
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      		die(_("confused by unstable object source data for %s"),
      		    oid_to_hex(oid));
      
    -@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
    - 
      	close_loose_object(fd);
      
     +	if (is_null_oid(oid)) {
     +		int dirlen;
     +
    -+		/* copy oid */
     +		oidcpy((struct object_id *)oid, &parano_oid);
    -+		/* We get the oid now */
     +		loose_object_path(the_repository, &filename, oid);
     +
    ++		/* We finally know the object path, and create the missing dir. */
     +		dirlen = directory_size(filename.buf);
     +		if (dirlen) {
     +			struct strbuf dir = STRBUF_INIT;
    -+			/*
    -+			 * Make sure the directory exists; note that the
    -+			 * contents of the buffer are undefined after mkstemp
    -+			 * returns an error, so we have to rewrite the whole
    -+			 * buffer from scratch.
    -+			 */
    -+			strbuf_reset(&dir);
     +			strbuf_add(&dir, filename.buf, dirlen - 1);
     +			if (mkdir(dir.buf, 0777) && errno != EEXIST)
     +				return -1;
    ++			if (adjust_shared_perm(dir.buf))
    ++				return -1;
    ++			strbuf_release(&dir);
     +		}
     +	}
     +
4:  09d438b692 ! 3:  2575900449 object-file.c: read input stream repeatedly in write_loose_object()
    @@ Metadata
     Author: Han Xin <[email protected]>
     
      ## Commit message ##
    -    object-file.c: read input stream repeatedly in write_loose_object()
    +    object-file.c: read stream in a loop in write_loose_object()
     
    -    Read input stream repeatedly in write_loose_object() unless reach the
    -    end, so that we can divide the large blob write into many small blocks.
    +    In order to prepare the stream version of "write_loose_object()", read
    +    the input stream in a loop in "write_loose_object()", so that we can
    +    feed the contents of large blob object to "write_loose_object()" using
    +    a small fixed buffer.
     
    +    Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
     
      ## object-file.c ##
     @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
      	static struct strbuf tmp_file = STRBUF_INIT;
      	static struct strbuf filename = STRBUF_INIT;
    - 	const char *buf;
    + 	const void *buf;
     -	unsigned long len;
     +	int flush = 0;
      
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      	the_hash_algo->update_fn(&c, hdr, hdrlen);
      
      	/* Then the data itself.. */
    --	buf = in_stream->read(in_stream->data, &len);
    +-	buf = in_stream->read(in_stream, &len);
     -	stream.next_in = (void *)buf;
     -	stream.avail_in = len;
      	do {
      		unsigned char *in0 = stream.next_in;
     -		ret = git_deflate(&stream, Z_FINISH);
     +		if (!stream.avail_in) {
    -+			if ((buf = in_stream->read(in_stream->data, &stream.avail_in))) {
    ++			buf = in_stream->read(in_stream, &stream.avail_in);
    ++			if (buf) {
     +				stream.next_in = (void *)buf;
     +				in0 = (unsigned char *)buf;
    -+			} else
    ++			} else {
     +				flush = Z_FINISH;
    ++			}
     +		}
     +		ret = git_deflate(&stream, flush);
      		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
    - 		if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0)
    + 		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
      			die(_("unable to write loose object file"));
5:  9fb188d437 < -:  ---------- object-store.h: add write_loose_object()
-:  ---------- > 4:  ca93ecc780 unpack-objects.c: add dry_run mode for get_data()
6:  80468a6fbc ! 5:  39a072ee2a unpack-objects: unpack large object in stream
    @@ Metadata
     Author: Han Xin <[email protected]>
     
      ## Commit message ##
    -    unpack-objects: unpack large object in stream
    +    unpack-objects: unpack_non_delta_entry() read data in a stream
     
    -    When calling "unpack_non_delta_entry()", will allocate full memory for
    -    the whole size of the unpacked object and write the buffer to loose file
    -    on disk. This may lead to OOM for the git-unpack-objects process when
    -    unpacking a very large object.
    +    We used to call "get_data()" in "unpack_non_delta_entry()" to read the
    +    entire contents of a blob object, no matter how big it is. This
    +    implementation may consume all the memory and cause OOM.
     
    -    In function "unpack_delta_entry()", will also allocate full memory to
    -    buffer the whole delta, but since there will be no delta for an object
    -    larger than "core.bigFileThreshold", this issue is moderate.
    +    By implementing a zstream version of input_stream interface, we can use
    +    a small fixed buffer for "unpack_non_delta_entry()".
     
    -    To resolve the OOM issue in "git-unpack-objects", we can unpack large
    -    object to file in stream, and use "core.bigFileThreshold" to avoid OOM
    -    limits when called "get_data()".
    +    However, unpack non-delta objects from a stream instead of from an entrie
    +    buffer will have 10% performance penalty. Therefore, only unpack object
    +    larger than the "big_file_threshold" in zstream. See the following
    +    benchmarks:
     
    +        $ hyperfine \
    +        --prepare 'rm -rf dest.git && git init --bare dest.git' \
    +        'git -C dest.git unpack-objects <binary_320M.pack'
    +        Benchmark 1: git -C dest.git unpack-objects <binary_320M.pack
    +          Time (mean ± σ):     10.029 s ±  0.270 s    [User: 8.265 s, System: 1.522 s]
    +          Range (min … max):    9.786 s … 10.603 s    10 runs
    +
    +        $ hyperfine \
    +        --prepare 'rm -rf dest.git && git init --bare dest.git' \
    +        'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack'
    +        Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack
    +          Time (mean ± σ):     10.859 s ±  0.774 s    [User: 8.813 s, System: 1.898 s]
    +          Range (min … max):    9.884 s … 12.192 s    10 runs
    +
    +        $ hyperfine \
    +        --prepare 'rm -rf dest.git && git init --bare dest.git' \
    +        'git -C dest.git unpack-objects <binary_96M.pack'
    +        Benchmark 1: git -C dest.git unpack-objects <binary_96M.pack
    +          Time (mean ± σ):      2.678 s ±  0.037 s    [User: 2.205 s, System: 0.450 s]
    +          Range (min … max):    2.639 s …  2.743 s    10 runs
    +
    +        $ hyperfine \
    +        --prepare 'rm -rf dest.git && git init --bare dest.git' \
    +        'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack'
    +        Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack
    +          Time (mean ± σ):      2.819 s ±  0.124 s    [User: 2.216 s, System: 0.564 s]
    +          Range (min … max):    2.679 s …  3.125 s    10 runs
    +
    +    Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
     
      ## builtin/unpack-objects.c ##
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
      	}
      }
      
    -+struct input_data_from_zstream {
    ++struct input_zstream_data {
     +	git_zstream *zstream;
     +	unsigned char buf[4096];
     +	int status;
     +};
     +
    -+static const char *read_inflate_in_stream(void *data, unsigned long *readlen)
    ++static const void *feed_input_zstream(struct input_stream *in_stream, unsigned long *readlen)
     +{
    -+	struct input_data_from_zstream *input = data;
    -+	git_zstream *zstream = input->zstream;
    ++	struct input_zstream_data *data = in_stream->data;
    ++	git_zstream *zstream = data->zstream;
     +	void *in = fill(1);
     +
    -+	if (!len || input->status == Z_STREAM_END) {
    ++	if (!len || data->status == Z_STREAM_END) {
     +		*readlen = 0;
     +		return NULL;
     +	}
     +
    -+	zstream->next_out = input->buf;
    -+	zstream->avail_out = sizeof(input->buf);
    ++	zstream->next_out = data->buf;
    ++	zstream->avail_out = sizeof(data->buf);
     +	zstream->next_in = in;
     +	zstream->avail_in = len;
     +
    -+	input->status = git_inflate(zstream, 0);
    ++	data->status = git_inflate(zstream, 0);
     +	use(len - zstream->avail_in);
    -+	*readlen = sizeof(input->buf) - zstream->avail_out;
    ++	*readlen = sizeof(data->buf) - zstream->avail_out;
     +
    -+	return (const char *)input->buf;
    ++	return data->buf;
     +}
     +
     +static void write_stream_blob(unsigned nr, unsigned long size)
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	char hdr[32];
     +	int hdrlen;
     +	git_zstream zstream;
    -+	struct input_data_from_zstream data;
    ++	struct input_zstream_data data;
     +	struct input_stream in_stream = {
    -+		.read = read_inflate_in_stream,
    ++		.read = feed_input_zstream,
     +		.data = &data,
     +	};
     +	struct object_id *oid = &obj_list[nr].oid;
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	/* Generate the header */
     +	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), (uintmax_t)size) + 1;
     +
    -+	if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, dry_run, 0, 0)))
    ++	if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, 0, 0)))
     +		die(_("failed to write object in stream %d"), ret);
     +
     +	if (zstream.total_out != size || data.status != Z_STREAM_END)
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
      static void unpack_non_delta_entry(enum object_type type, unsigned long size,
      				   unsigned nr)
      {
    --	void *buf = get_data(size);
    +-	void *buf = get_data(size, dry_run);
     +	void *buf;
     +
     +	/* Write large blob in stream without allocating full buffer. */
    -+	if (type == OBJ_BLOB && size > big_file_threshold) {
    ++	if (!dry_run && type == OBJ_BLOB && size > big_file_threshold) {
     +		write_stream_blob(nr, size);
     +		return;
     +	}
      
    -+	buf = get_data(size);
    ++	buf = get_data(size, dry_run);
      	if (!dry_run && buf)
      		write_object(nr, type, buf, size);
      	else
     
    - ## t/t5590-receive-unpack-objects.sh (new) ##
    + ## object-file.c ##
    +@@ object-file.c: static const void *feed_simple_input_stream(struct input_stream *in_stream, unsi
    + 	return data->buf;
    + }
    + 
    +-static int write_loose_object(const struct object_id *oid, char *hdr,
    +-			      int hdrlen, struct input_stream *in_stream,
    +-			      time_t mtime, unsigned flags)
    ++int write_loose_object(const struct object_id *oid, char *hdr,
    ++		       int hdrlen, struct input_stream *in_stream,
    ++		       time_t mtime, unsigned flags)
    + {
    + 	int fd, ret;
    + 	unsigned char compressed[4096];
    +
    + ## object-store.h ##
    +@@ object-store.h: int hash_object_file(const struct git_hash_algo *algo, const void *buf,
    + 		     unsigned long len, const char *type,
    + 		     struct object_id *oid);
    + 
    ++int write_loose_object(const struct object_id *oid, char *hdr,
    ++		       int hdrlen, struct input_stream *in_stream,
    ++		       time_t mtime, unsigned flags);
    ++
    + int write_object_file_flags(const void *buf, unsigned long len,
    + 			    const char *type, struct object_id *oid,
    + 			    unsigned flags);
    +
    + ## t/t5590-unpack-non-delta-objects.sh (new) ##
     @@
     +#!/bin/sh
     +#
    @@ t/t5590-receive-unpack-objects.sh (new)
     +		cd .git &&
     +		find objects/?? -type f | sort
     +	) >expect &&
    -+	git repack -ad
    ++	PACK=$(echo main | git pack-objects --progress --revs test)
     +'
     +
     +test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
    @@ t/t5590-receive-unpack-objects.sh (new)
     +	git -C dest.git config receive.unpacklimit 100
     +'
     +
    -+test_expect_success 'fail to push: cannot allocate' '
    -+	test_must_fail git push dest.git HEAD 2>err &&
    -+	test_i18ngrep "remote: fatal: attempting to allocate" err &&
    ++test_expect_success 'fail to unpack-objects: cannot allocate' '
    ++	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
    ++	test_i18ngrep "fatal: attempting to allocate" err &&
     +	(
     +		cd dest.git &&
     +		find objects/?? -type f | sort
    @@ t/t5590-receive-unpack-objects.sh (new)
     +'
     +
     +test_expect_success 'unpack big object in stream' '
    -+	git push dest.git HEAD &&
    ++	git -C dest.git unpack-objects <test-$PACK.pack &&
     +	git -C dest.git fsck &&
     +	(
     +		cd dest.git &&
    @@ t/t5590-receive-unpack-objects.sh (new)
     +'
     +
     +test_expect_success 'setup for unpack-objects dry-run test' '
    -+	PACK=$(echo main | git pack-objects --progress --revs test) &&
    -+	unset GIT_ALLOC_LIMIT &&
     +	git init --bare unpack-test.git
     +'
     +
    -+test_expect_success 'unpack-objects dry-run with large threshold' '
    -+	(
    -+		cd unpack-test.git &&
    -+		git config core.bigFileThreshold 2m &&
    -+		git unpack-objects -n <../test-$PACK.pack
    -+	) &&
    -+	(
    -+		cd unpack-test.git &&
    -+		find objects/ -type f
    -+	) >actual &&
    -+	test_must_be_empty actual
    -+'
    -+
    -+test_expect_success 'unpack-objects dry-run with small threshold' '
    ++test_expect_success 'unpack-objects dry-run' '
     +	(
     +		cd unpack-test.git &&
    -+		git config core.bigFileThreshold 1m &&
     +		git unpack-objects -n <../test-$PACK.pack
     +	) &&
     +	(
Signature · 1 line
2.34.0.6.g676eedc724
author
#20 lore raw

Re: [PATCH v3 0/5] unpack large objects in stream

Quoted 9 lines · Han Xin

From: Han Xin <[email protected]>

Although we do not recommend users push large binary files to the git repositories, it's difficult to prevent them from doing so. Once, we found a problem with a surge in memory usage on the server. The source of the problem is that a user submitted a single object with a size of 15GB. Once someone initiates a git push, the git process will immediately allocate 15G of memory, resulting in an OOM risk.

Quoted 8 lines

Through further analysis, we found that when we execute git unpack-objects, in unpack_non_delta_entry(), "void *buf = get_data(size);" will directly allocate memory equal to the size of the object. This is quite a scary thing, because the pre-receive hook has not been executed at this time, and we cannot avoid this by hooks.

I got inspiration from the deflate process of zlib, maybe it would be a good idea to change unpack-objects to stream deflate.

Hi, Jeff.

I hope you can share with me how Github solves this problem.

As you said in your reply at:
https://lore.kernel.org/git/[email protected]/
"we don't have a match in unpack-objects, but we always run index-pack on incoming packs".

In the original implementation of "index-pack", for objects larger than big_file_threshold, "fixed_buf" with a size of 8192 will be used to complete the calculation of "oid".

I tried the implementation in jk/no-more-unpack-objects, as you noted:
/* XXX This will expand too-large objects! */
if (!data)
data = new_data = get_data_from_pack(obj_entry);
If the conditions of --unpack are given, there will be risks here. When I create an object larger than 1GB and execute index-pack, the result is as follows:
$GIT_ALLOC_LIMIT=1024m git index-pack --unpack --stdin <large.pack
fatal: attempting to allocate 1228800001 over limit 1073741824

Looking forward to your reply.

#21 lore raw

Re: [PATCH v3 0/5] unpack large objects in stream

Quoted 23 lines · Han Xin
Quoted 9 lines · Han Xin

From: Han Xin <[email protected]>

Although we do not recommend users push large binary files to the git repositories, it's difficult to prevent them from doing so. Once, we found a problem with a surge in memory usage on the server. The source of the problem is that a user submitted a single object with a size of 15GB. Once someone initiates a git push, the git process will immediately allocate 15G of memory, resulting in an OOM risk.

Quoted 8 lines

Through further analysis, we found that when we execute git unpack-objects, in unpack_non_delta_entry(), "void *buf = get_data(size);" will directly allocate memory equal to the size of the object. This is quite a scary thing, because the pre-receive hook has not been executed at this time, and we cannot avoid this by hooks.

I got inspiration from the deflate process of zlib, maybe it would be a good idea to change unpack-objects to stream deflate.

Hi, Jeff.

I hope you can share with me how Github solves this problem.

Quoted 8 lines

As you said in your reply at:
https://lore.kernel.org/git/[email protected]/
"we don't have a match in unpack-objects, but we always run index-pack on incoming packs".

In the original implementation of "index-pack", for objects larger than big_file_threshold, "fixed_buf" with a size of 8192 will be used to complete the calculation of "oid".

We set transfer.unpackLimit to "1", so we never run unpack-objects at all. We always run index-pack, and every push, no matter how small, results in a pack.

We also set GIT_ALLOC_LIMIT to limit any single allocation. We also have custom code in index-pack to detect large objects (where our definition of "large" is 100MB by default):

- for large blobs, we do index it as normal, writing the oid out to a file which is then processed by a pre-receive hook (since people often push up large files accidentally, the hook generates a nice error message, including finding the path at which the blob is referenced)

- for other large objects, we die immediately (with an error message). 100MB commit messages aren't a common user error, and it closes off a whole set of possible integer-overflow parsing attacks (e.g., index-pack in strict-mode will run every tree through fsck_tree(), so there's otherwise nothing stopping you from having a 4GB filename in a tree).

Quoted 9 lines

I tried the implementation in jk/no-more-unpack-objects, as you noted:
/* XXX This will expand too-large objects! */
if (!data)
data = new_data = get_data_from_pack(obj_entry);
If the conditions of --unpack are given, there will be risks here. When I create an object larger than 1GB and execute index-pack, the result is as follows:
$GIT_ALLOC_LIMIT=1024m git index-pack --unpack --stdin <large.pack
fatal: attempting to allocate 1228800001 over limit 1073741824

Yeah, that issue was one of the reasons I never sent the "index-pack --unpack" code to the list. We don't actually use those patches at GitHub. It was something I was working on for upstream but never finished.

-Peff

author
#22 lore raw

Re: [PATCH v3 0/5] unpack large objects in stream

Quoted 14 lines · Jeff King

We set transfer.unpackLimit to "1", so we never run unpack-objects at all. We always run index-pack, and every push, no matter how small, results in a pack.

We also set GIT_ALLOC_LIMIT to limit any single allocation. We also have custom code in index-pack to detect large objects (where our definition of "large" is 100MB by default):

- for large blobs, we do index it as normal, writing the oid out to a file which is then processed by a pre-receive hook (since people often push up large files accidentally, the hook generates a nice error message, including finding the path at which the blob is referenced)

Quoted 6 lines

- for other large objects, we die immediately (with an error message). 100MB commit messages aren't a common user error, and it closes off a whole set of possible integer-overflow parsing attacks (e.g., index-pack in strict-mode will run every tree through fsck_tree(), so there's otherwise nothing stopping you from having a 4GB filename in a tree).

Thank you very much for sharing.

The way Github handles it reminds me of what Shawn Pearce introduced in "Scaling up JGit". I guess "mulit-pack-index" and "bitmap" must play an important role in this.

I will seriously consider this solution, thanks a lot.

author
#23 lore raw

v4 cover unpack large objects in stream

In reply to Han Xin

From: Han Xin <[email protected]>

Changes since v3:
* Add "size" to "struct input_stream" which used by following commits.

* Increase the buffer size of "struct input_zstream_data" from 4096 to 8192, which is consistent with the "fixed_buf" in the "index-pack.c".

* Refactor "read stream in a loop in write_loose_object()" which introduced a performance problem reported by Derrick Stolee[1].

* Rewrite benchmarks in "unpack-objects: unpack_non_delta_entry() read data in a stream" with sugguestions by Derrick Stolee[1] and Ævar Arnfjörð Bjarmason[2].
Now use "scalar.git" to benchmark, which contains more than 28000 objects and 96 objects larger than 16kB.

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/

Han Xin (5):
object-file: refactor write_loose_object() to read buffer from stream object-file.c: handle undetermined oid in write_loose_object() object-file.c: read stream in a loop in write_loose_object() unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream

 builtin/unpack-objects.c            |  93 +++++++++++++++++++++++--
 object-file.c                       | 102 ++++++++++++++++++++++++----
 object-store.h                      |  10 +++
 t/t5590-unpack-non-delta-objects.sh |  76 +++++++++++++++++++++
 4 files changed, 262 insertions(+), 19 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh

Range-diff against v3:
1: 8640b04f6d ! 1: af707ef304 object-file: refactor write_loose_object() to read buffer from stream

    @@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
     +			.buf = buf,
     +			.len = len,
     +		},
    ++		.size = len,
     +	};
      
      	/* Normally if we have it in the pack then we do not bother writing
    @@ object-file.c: int hash_object_file_literally(const void *buf, unsigned long len
     +			.buf = buf,
     +			.len = len,
     +		},
    ++		.size = len,
     +	};
      
      	/* type string, SP, %lu of the length plus NUL must fit this */
    @@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
      	if (has_loose_object(oid))
      		return 0;
      	buf = read_object(the_repository, oid, &type, &len);
    ++	in_stream.size = len;
      	if (!buf)
      		return error(_("cannot read object for %s"), oid_to_hex(oid));
     +	data.buf = buf;
    @@ object-store.h: struct object_directory {
     +struct input_stream {
     +	const void *(*read)(struct input_stream *, unsigned long *len);
     +	void *data;
    ++	size_t size;
     +};
     +
      KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
2:  d4a2caf2bd = 2:  321ad90d8e object-file.c: handle undetermined oid in write_loose_object()
3:  2575900449 ! 3:  1992ac39af object-file.c: read stream in a loop in write_loose_object()
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
     -		ret = git_deflate(&stream, Z_FINISH);
     +		if (!stream.avail_in) {
     +			buf = in_stream->read(in_stream, &stream.avail_in);
    -+			if (buf) {
    -+				stream.next_in = (void *)buf;
    -+				in0 = (unsigned char *)buf;
    -+			} else {
    ++			stream.next_in = (void *)buf;
    ++			in0 = (unsigned char *)buf;
    ++			/* All data has been read. */
    ++			if (in_stream->size + hdrlen == stream.total_in + stream.avail_in)
     +				flush = Z_FINISH;
    -+			}
     +		}
     +		ret = git_deflate(&stream, flush);
      		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
      		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
      			die(_("unable to write loose object file"));
    + 		stream.next_out = compressed;
    + 		stream.avail_out = sizeof(compressed);
    +-	} while (ret == Z_OK);
    ++	} while (ret == Z_OK || ret == Z_BUF_ERROR);
    + 
    + 	if (ret != Z_STREAM_END)
    + 		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
4:  ca93ecc780 = 4:  c41eb06533 unpack-objects.c: add dry_run mode for get_data()
5:  39a072ee2a ! 5:  9427775bdc unpack-objects: unpack_non_delta_entry() read data in a stream

@@ Commit message
larger than the "big_file_threshold" in zstream. See the following benchmarks:

- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -C dest.git unpack-objects <binary_320M.pack'
- Benchmark 1: git -C dest.git unpack-objects <binary_320M.pack
- Time (mean ± σ): 10.029 s ± 0.270 s [User: 8.265 s, System: 1.522 s]
- Range (min … max): 9.786 s … 10.603 s 10 runs
+ hyperfine \
+ --setup \
+ 'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ -n 'old' 'git -C dest.git unpack-objects <small.pack' \
+ -n 'new' 'new/git -C dest.git unpack-objects <small.pack' \
+ -n 'new (small threshold)' \
+ 'new/git -c core.bigfilethreshold=16k -C dest.git unpack-objects <small.pack'
+ Benchmark 1: old
+ Time (mean ± σ): 6.075 s ± 0.069 s [User: 5.047 s, System: 0.991 s]
+ Range (min … max): 6.018 s … 6.189 s 10 runs

- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack'
- Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack
- Time (mean ± σ): 10.859 s ± 0.774 s [User: 8.813 s, System: 1.898 s]
- Range (min … max): 9.884 s … 12.192 s 10 runs
+ Benchmark 2: new
+ Time (mean ± σ): 6.090 s ± 0.033 s [User: 5.075 s, System: 0.976 s]
+ Range (min … max): 6.030 s … 6.142 s 10 runs

- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -C dest.git unpack-objects <binary_96M.pack'
- Benchmark 1: git -C dest.git unpack-objects <binary_96M.pack
- Time (mean ± σ): 2.678 s ± 0.037 s [User: 2.205 s, System: 0.450 s]
- Range (min … max): 2.639 s … 2.743 s 10 runs
+ Benchmark 3: new (small threshold)
+ Time (mean ± σ): 6.755 s ± 0.029 s [User: 5.150 s, System: 1.560 s]
+ Range (min … max): 6.711 s … 6.809 s 10 runs

- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack'
- Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack
- Time (mean ± σ): 2.819 s ± 0.124 s [User: 2.216 s, System: 0.564 s]
- Range (min … max): 2.679 s … 3.125 s 10 runs
+ Summary
+ 'old' ran
+ 1.00 ± 0.01 times faster than 'new'
+ 1.11 ± 0.01 times faster than 'new (small threshold)'

+ Helped-by: Derrick Stolee <[email protected]>

@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type

+struct input_zstream_data {
+ git_zstream *zstream;
-+ unsigned char buf[4096];
++ unsigned char buf[8192];
+ int status;
+};
+
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ struct input_stream in_stream = {
+ .read = feed_input_zstream,
+ .data = &data,
++ .size = size,
+ };
+ struct object_id *oid = &obj_list[nr].oid;
+ int ret;

Signature · 1 line
2.34.0
#24 lore raw

Re: [PATCH v4 0/5] unpack large objects in stream

Quoted 10 lines · Han Xin

From: Han Xin <[email protected]>

Changes since v3:
* Add "size" to "struct input_stream" which used by following commits.

* Increase the buffer size of "struct input_zstream_data" from 4096 to 8192, which is consistent with the "fixed_buf" in the "index-pack.c".

* Refactor "read stream in a loop in write_loose_object()" which introduced a performance problem reported by Derrick Stolee[1].

Thank you for finding the issue. It seems simple enough to add that size information and regain the performance back to nearly no overhead. Your hyperfine statistics are within noise, which is great. Thanks!

-Stolee

author
#25 lore raw

v5 cover unpack large blobs in stream

In reply to Han Xin

From: Han Xin <[email protected]>

Changes since v4:
* Refactor to "struct input_stream" implementations so that we can reduce the changes to "write_loose_object()" sugguest by Ævar Arnfjörð Bjarmason.

* Add a new flag called "HASH_STREAM" to support this feature.

* Add a new config "core.bigFileStreamingThreshold" instread of "core.bigFileThreshold" sugguest by Ævar Arnfjörð Bjarmason[1].

* Roll destination repository preparement into a function in "t5590-unpack-non-delta-objects.sh", so that we can run testcases with --run=setup,3,4.

1. https://lore.kernel.org/git/[email protected]/

Han Xin (6):
object-file: refactor write_loose_object() to support read from stream object-file.c: handle undetermined oid in write_loose_object() object-file.c: read stream in a loop in write_loose_object() unpack-objects.c: add dry_run mode for get_data() object-file.c: make "write_object_file_flags()" to support "HASH_STREAM"
unpack-objects: unpack_non_delta_entry() read data in a stream

 Documentation/config/core.txt       | 11 ++++
 builtin/unpack-objects.c            | 86 +++++++++++++++++++++++++++--
 cache.h                             |  2 +
 config.c                            |  5 ++
 environment.c                       |  1 +
 object-file.c                       | 73 +++++++++++++++++++-----
 object-store.h                      |  5 ++
 t/t5590-unpack-non-delta-objects.sh | 70 +++++++++++++++++++++++
 8 files changed, 234 insertions(+), 19 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh

Range-diff against v4:

1:  af707ef304 < -:  ---------- object-file: refactor write_loose_object() to read buffer from stream
2:  321ad90d8e < -:  ---------- object-file.c: handle undetermined oid in write_loose_object()
3:  1992ac39af < -:  ---------- object-file.c: read stream in a loop in write_loose_object()
-:  ---------- > 1:  f3595e68cc object-file: refactor write_loose_object() to support read from stream
-:  ---------- > 2:  c25fdd1fe5 object-file.c: handle undetermined oid in write_loose_object()
-:  ---------- > 3:  ed226f2f9f object-file.c: read stream in a loop in write_loose_object()
4:  c41eb06533 ! 4:  2f91e540f6 unpack-objects.c: add dry_run mode for get_data()
    @@ builtin/unpack-objects.c: static void use(int bytes)
      {
      	git_zstream stream;
     -	void *buf = xmallocz(size);
    -+	unsigned long bufsize = dry_run ? 4096 : size;
    ++	unsigned long bufsize = dry_run ? 8192 : size;
     +	void *buf = xmallocz(bufsize);
      
      	memset(&stream, 0, sizeof(stream));
-:  ---------- > 5:  7698938eac object-file.c: make "write_object_file_flags()" to support "HASH_STREAM"
5:  9427775bdc ! 6:  103bb1db06 unpack-objects: unpack_non_delta_entry() read data in a stream
    @@ Commit message
     
         However, unpack non-delta objects from a stream instead of from an entrie
         buffer will have 10% performance penalty. Therefore, only unpack object
    -    larger than the "big_file_threshold" in zstream. See the following
    +    larger than the "core.BigFileStreamingThreshold" in zstream. See the following
         benchmarks:
     
             hyperfine \
               --setup \
               'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
    -          --prepare 'rm -rf dest.git && git init --bare dest.git' \
    -          -n 'old' 'git -C dest.git unpack-objects <small.pack' \
    -          -n 'new' 'new/git -C dest.git unpack-objects <small.pack' \
    -          -n 'new (small threshold)' \
    -          'new/git -c core.bigfilethreshold=16k -C dest.git unpack-objects <small.pack'
    -        Benchmark 1: old
    -          Time (mean ± σ):      6.075 s ±  0.069 s    [User: 5.047 s, System: 0.991 s]
    -          Range (min … max):    6.018 s …  6.189 s    10 runs
    -
    -        Benchmark 2: new
    -          Time (mean ± σ):      6.090 s ±  0.033 s    [User: 5.075 s, System: 0.976 s]
    -          Range (min … max):    6.030 s …  6.142 s    10 runs
    -
    -        Benchmark 3: new (small threshold)
    -          Time (mean ± σ):      6.755 s ±  0.029 s    [User: 5.150 s, System: 1.560 s]
    -          Range (min … max):    6.711 s …  6.809 s    10 runs
    +          --prepare 'rm -rf dest.git && git init --bare dest.git'
     
             Summary
    -          'old' ran
    -            1.00 ± 0.01 times faster than 'new'
    -            1.11 ± 0.01 times faster than 'new (small threshold)'
    +          './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
    +            1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1'
    +            1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0'
    +            1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master'
    +            1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0'
    +            1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'
     
    +    Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
         Helped-by: Derrick Stolee <[email protected]>
         Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
     
    + ## Documentation/config/core.txt ##
    +@@ Documentation/config/core.txt: be delta compressed, but larger binary media files won't be.
    + +
    + Common unit suffixes of 'k', 'm', or 'g' are supported.
    + 
    ++core.bigFileStreamingThreshold::
    ++	Files larger than this will be streamed out to a temporary
    ++	object file while being hashed, which will when be renamed
    ++	in-place to a loose object, particularly if the
    ++	`core.bigFileThreshold' setting dictates that they're always
    ++	written out as loose objects.
    +++
    ++Default is 128 MiB on all platforms.
    +++
    ++Common unit suffixes of 'k', 'm', or 'g' are supported.
    ++
    + core.excludesFile::
    + 	Specifies the pathname to the file that contains patterns to
    + 	describe paths that are not meant to be tracked, in addition
    +
      ## builtin/unpack-objects.c ##
     @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
      	}
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +
     +static void write_stream_blob(unsigned nr, unsigned long size)
     +{
    -+	char hdr[32];
    -+	int hdrlen;
     +	git_zstream zstream;
     +	struct input_zstream_data data;
     +	struct input_stream in_stream = {
     +		.read = feed_input_zstream,
     +		.data = &data,
    -+		.size = size,
     +	};
    -+	struct object_id *oid = &obj_list[nr].oid;
     +	int ret;
     +
     +	memset(&zstream, 0, sizeof(zstream));
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	data.zstream = &zstream;
     +	git_inflate_init(&zstream);
     +
    -+	/* Generate the header */
    -+	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), (uintmax_t)size) + 1;
    -+
    -+	if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, 0, 0)))
    ++	if ((ret = write_object_file_flags(&in_stream, size, type_name(OBJ_BLOB) ,&obj_list[nr].oid, HASH_STREAM)))
     +		die(_("failed to write object in stream %d"), ret);
     +
     +	if (zstream.total_out != size || data.status != Z_STREAM_END)
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	git_inflate_end(&zstream);
     +
     +	if (strict && !dry_run) {
    -+		struct blob *blob = lookup_blob(the_repository, oid);
    ++		struct blob *blob = lookup_blob(the_repository, &obj_list[nr].oid);
     +		if (blob)
     +			blob->object.flags |= FLAG_WRITTEN;
     +		else
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	void *buf;
     +
     +	/* Write large blob in stream without allocating full buffer. */
    -+	if (!dry_run && type == OBJ_BLOB && size > big_file_threshold) {
    ++	if (!dry_run && type == OBJ_BLOB && size > big_file_streaming_threshold) {
     +		write_stream_blob(nr, size);
     +		return;
     +	}
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
      		write_object(nr, type, buf, size);
      	else
     
    - ## object-file.c ##
    -@@ object-file.c: static const void *feed_simple_input_stream(struct input_stream *in_stream, unsi
    - 	return data->buf;
    - }
    + ## cache.h ##
    +@@ cache.h: extern size_t packed_git_window_size;
    + extern size_t packed_git_limit;
    + extern size_t delta_base_cache_limit;
    + extern unsigned long big_file_threshold;
    ++extern unsigned long big_file_streaming_threshold;
    + extern unsigned long pack_size_limit_cfg;
      
    --static int write_loose_object(const struct object_id *oid, char *hdr,
    --			      int hdrlen, struct input_stream *in_stream,
    --			      time_t mtime, unsigned flags)
    -+int write_loose_object(const struct object_id *oid, char *hdr,
    -+		       int hdrlen, struct input_stream *in_stream,
    -+		       time_t mtime, unsigned flags)
    - {
    - 	int fd, ret;
    - 	unsigned char compressed[4096];
    + /*
     
    - ## object-store.h ##
    -@@ object-store.h: int hash_object_file(const struct git_hash_algo *algo, const void *buf,
    - 		     unsigned long len, const char *type,
    - 		     struct object_id *oid);
    + ## config.c ##
    +@@ config.c: static int git_default_core_config(const char *var, const char *value, void *cb)
    + 		return 0;
    + 	}
      
    -+int write_loose_object(const struct object_id *oid, char *hdr,
    -+		       int hdrlen, struct input_stream *in_stream,
    -+		       time_t mtime, unsigned flags);
    ++	if (!strcmp(var, "core.bigfilestreamingthreshold")) {
    ++		big_file_streaming_threshold = git_config_ulong(var, value);
    ++		return 0;
    ++	}
     +
    - int write_object_file_flags(const void *buf, unsigned long len,
    - 			    const char *type, struct object_id *oid,
    - 			    unsigned flags);
    + 	if (!strcmp(var, "core.packedgitlimit")) {
    + 		packed_git_limit = git_config_ulong(var, value);
    + 		return 0;
    +
    + ## environment.c ##
    +@@ environment.c: size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
    + size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
    + size_t delta_base_cache_limit = 96 * 1024 * 1024;
    + unsigned long big_file_threshold = 512 * 1024 * 1024;
    ++unsigned long big_file_streaming_threshold = 128 * 1024 * 1024;
    + int pager_use_color = 1;
    + const char *editor_program;
    + const char *askpass_program;
     
      ## t/t5590-unpack-non-delta-objects.sh (new) ##
     @@
    @@ t/t5590-unpack-non-delta-objects.sh (new)
     +
     +. ./test-lib.sh
     +
    -+test_expect_success "create commit with big blobs (1.5 MB)" '
    ++prepare_dest () {
    ++	test_when_finished "rm -rf dest.git" &&
    ++	git init --bare dest.git &&
    ++	git -C dest.git config core.bigFileStreamingThreshold $1
    ++	git -C dest.git config core.bigFileThreshold $1
    ++}
    ++
    ++test_expect_success "setup repo with big blobs (1.5 MB)" '
     +	test-tool genrandom foo 1500000 >big-blob &&
     +	test_commit --append foo big-blob &&
     +	test-tool genrandom bar 1500000 >big-blob &&
    @@ t/t5590-unpack-non-delta-objects.sh (new)
     +		cd .git &&
     +		find objects/?? -type f | sort
     +	) >expect &&
    -+	PACK=$(echo main | git pack-objects --progress --revs test)
    ++	PACK=$(echo main | git pack-objects --revs test)
     +'
     +
    -+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
    ++test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
     +	GIT_ALLOC_LIMIT=1m &&
     +	export GIT_ALLOC_LIMIT
     +'
     +
    -+test_expect_success 'prepare dest repository' '
    -+	git init --bare dest.git &&
    -+	git -C dest.git config core.bigFileThreshold 2m &&
    -+	git -C dest.git config receive.unpacklimit 100
    -+'
    -+
     +test_expect_success 'fail to unpack-objects: cannot allocate' '
    ++	prepare_dest 2m &&
     +	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
    -+	test_i18ngrep "fatal: attempting to allocate" err &&
    ++	grep "fatal: attempting to allocate" err &&
     +	(
     +		cd dest.git &&
     +		find objects/?? -type f | sort
     +	) >actual &&
    ++	test_file_not_empty actual &&
     +	! test_cmp expect actual
     +'
     +
    -+test_expect_success 'set a lower bigfile threshold' '
    -+	git -C dest.git config core.bigFileThreshold 1m
    -+'
    -+
     +test_expect_success 'unpack big object in stream' '
    ++	prepare_dest 1m &&
     +	git -C dest.git unpack-objects <test-$PACK.pack &&
     +	git -C dest.git fsck &&
     +	(
    @@ t/t5590-unpack-non-delta-objects.sh (new)
     +	test_cmp expect actual
     +'
     +
    -+test_expect_success 'setup for unpack-objects dry-run test' '
    -+	git init --bare unpack-test.git
    -+'
    -+
     +test_expect_success 'unpack-objects dry-run' '
    ++	prepare_dest 1m &&
    ++	git -C dest.git unpack-objects -n <test-$PACK.pack &&
     +	(
    -+		cd unpack-test.git &&
    -+		git unpack-objects -n <../test-$PACK.pack
    -+	) &&
    -+	(
    -+		cd unpack-test.git &&
    ++		cd dest.git &&
     +		find objects/ -type f
     +	) >actual &&
     +	test_must_be_empty actual
Signature · 1 line
2.34.0
author
#26 lore raw

v5 cover unpack large blobs in stream

From: Han Xin <[email protected]>

Changes since v5:
* Refactor write_loose_object() to reuse in stream version sugguest by Ævar Arnfjörð Bjarmason [1].

* Add a new testcase into t5590-unpack-non-delta-objects to cover the case of unpacking existing objects.

* Fix code formatting in unpack-objects.c sugguest by Ævar Arnfjörð Bjarmason [2].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/

Han Xin (6):
object-file.c: release strbuf in write_loose_object() object-file.c: refactor object header generation into a function object-file.c: refactor write_loose_object() to reuse in stream
version
object-file.c: make "write_object_file_flags()" to support read in
stream
unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream

 Documentation/config/core.txt       |  11 ++
 builtin/unpack-objects.c            |  94 ++++++++++++-
 cache.h                             |   2 +
 config.c                            |   5 +
 environment.c                       |   1 +
 object-file.c                       | 207 +++++++++++++++++++++++-----
 object-store.h                      |   5 +
 t/t5590-unpack-non-delta-objects.sh |  87 ++++++++++++
 8 files changed, 370 insertions(+), 42 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh

Range-diff against v5:

1:  f3595e68cc < -:  ---------- object-file: refactor write_loose_object() to support read from stream
2:  c25fdd1fe5 < -:  ---------- object-file.c: handle undetermined oid in write_loose_object()
3:  ed226f2f9f < -:  ---------- object-file.c: read stream in a loop in write_loose_object()
-:  ---------- > 1:  59d35dac5f object-file.c: release strbuf in write_loose_object()
-:  ---------- > 2:  2174a6cbad object-file.c: refactor object header generation into a function
-:  ---------- > 3:  8a704ecc59 object-file.c: refactor write_loose_object() to reuse in stream version
-:  ---------- > 4:  96f05632a2 object-file.c: make "write_object_file_flags()" to support read in stream
4:  2f91e540f6 ! 5:  1acbb6e849 unpack-objects.c: add dry_run mode for get_data()
    @@ builtin/unpack-objects.c: static void use(int bytes)
      {
      	git_zstream stream;
     -	void *buf = xmallocz(size);
    -+	unsigned long bufsize = dry_run ? 8192 : size;
    -+	void *buf = xmallocz(bufsize);
    ++	unsigned long bufsize;
    ++	void *buf;
      
      	memset(&stream, 0, sizeof(stream));
    ++	if (dry_run && size > 8192)
    ++		bufsize = 8192;
    ++	else
    ++		bufsize = size;
    ++	buf = xmallocz(bufsize);
      
      	stream.next_out = buf;
     -	stream.avail_out = size;
5:  7698938eac < -:  ---------- object-file.c: make "write_object_file_flags()" to support "HASH_STREAM"
6:  92d69cb84a ! 6:  476aaba527 unpack-objects: unpack_non_delta_entry() read data in a stream
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	int status;
     +};
     +
    -+static const void *feed_input_zstream(struct input_stream *in_stream, unsigned long *readlen)
    ++static const void *feed_input_zstream(const struct input_stream *in_stream,
    ++				      unsigned long *readlen)
     +{
     +	struct input_zstream_data *data = in_stream->data;
     +	git_zstream *zstream = data->zstream;
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +		.read = feed_input_zstream,
     +		.data = &data,
     +	};
    -+	int ret;
     +
     +	memset(&zstream, 0, sizeof(zstream));
     +	memset(&data, 0, sizeof(data));
     +	data.zstream = &zstream;
     +	git_inflate_init(&zstream);
     +
    -+	if ((ret = write_object_file_flags(&in_stream, size, type_name(OBJ_BLOB) ,&obj_list[nr].oid, HASH_STREAM)))
    -+		die(_("failed to write object in stream %d"), ret);
    ++	if (write_object_file_flags(&in_stream, size,
    ++				    type_name(OBJ_BLOB),
    ++				    &obj_list[nr].oid,
    ++				    HASH_STREAM))
    ++		die(_("failed to write object in stream"));
     +
     +	if (zstream.total_out != size || data.status != Z_STREAM_END)
     +		die(_("inflate returned %d"), data.status);
     +	git_inflate_end(&zstream);
     +
    -+	if (strict && !dry_run) {
    ++	if (strict) {
     +		struct blob *blob = lookup_blob(the_repository, &obj_list[nr].oid);
     +		if (blob)
     +			blob->object.flags |= FLAG_WRITTEN;
     +		else
    -+			die("invalid blob object from stream");
    ++			die(_("invalid blob object from stream"));
     +	}
     +	obj_list[nr].obj = NULL;
     +}
    @@ t/t5590-unpack-non-delta-objects.sh (new)
     +prepare_dest () {
     +	test_when_finished "rm -rf dest.git" &&
     +	git init --bare dest.git &&
    -+	git -C dest.git config core.bigFileStreamingThreshold $1
    ++	git -C dest.git config core.bigFileStreamingThreshold $1 &&
     +	git -C dest.git config core.bigFileThreshold $1
     +}
     +
    @@ t/t5590-unpack-non-delta-objects.sh (new)
     +	test_cmp expect actual
     +'
     +
    ++test_expect_success 'unpack big object in stream with existing oids' '
    ++	prepare_dest 1m &&
    ++	git -C dest.git index-pack --stdin <test-$PACK.pack &&
    ++	(
    ++		cd dest.git &&
    ++		find objects/?? -type f | sort
    ++	) >actual &&
    ++	test_must_be_empty actual &&
    ++	git -C dest.git unpack-objects <test-$PACK.pack &&
    ++	git -C dest.git fsck &&
    ++	(
    ++		cd dest.git &&
    ++		find objects/?? -type f | sort
    ++	) >actual &&
    ++	test_must_be_empty actual
    ++'
    ++
     +test_expect_success 'unpack-objects dry-run' '
     +	prepare_dest 1m &&
     +	git -C dest.git unpack-objects -n <test-$PACK.pack &&
Signature · 1 line
2.34.1.52.gfcc2252aea.agit.6.5.6
author
#27 lore raw

v7 cover unpack large blobs in stream

From: Han Xin <[email protected]>

Changes since v6:
* Remove "object-file.c: release strbuf in write_loose_object()" which is not needed anymore. Thanks to René Scharfe[1] for reporting this.

* Reorder the patch series and put "unpack-objects.c: add dry_run mode for get_data()" and its testcases to the front.

* Replace "refactor object header generation into a function" with "object-file API: add a format_object_header() function" sugguested by Ævar Arnfjörð Bjarmason[2].

* Export "write_stream_object_file()" instead of "reusing write_object_file_flags()" sugguested by René Scharfe[3]. The new flag "HASH_STREAM" has been removed.

* Fix the directory creation error and the "strbuf dir" leak in "write_stream_object_file()".

* Change "unsigned long size" to "size_t size" in "write_stream_blob()" and "get_data()" in "unpack-objects.c".

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/[email protected]/

Han Xin (4):
unpack-objects.c: add dry_run mode for get_data() object-file.c: refactor write_loose_object() to reuse in stream
version
object-file.c: add "write_stream_object_file()" to support read in
stream
unpack-objects: unpack_non_delta_entry() read data in a stream

Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function

 Documentation/config/core.txt       |  11 ++
 builtin/index-pack.c                |   3 +-
 builtin/unpack-objects.c            |  94 ++++++++++++-
 bulk-checkin.c                      |   4 +-
 cache.h                             |  22 +++
 config.c                            |   5 +
 environment.c                       |   1 +
 http-push.c                         |   2 +-
 object-file.c                       | 199 ++++++++++++++++++++++------
 object-store.h                      |   9 ++
 t/t5590-unpack-non-delta-objects.sh |  91 +++++++++++++
 11 files changed, 392 insertions(+), 49 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh

Range-diff against v6:

1:  59d35dac5f < -:  ---------- object-file.c: release strbuf in write_loose_object()
2:  2174a6cbad < -:  ---------- object-file.c: refactor object header generation into a function
5:  1acbb6e849 ! 1:  a8f232f553 unpack-objects.c: add dry_run mode for get_data()
    @@ builtin/unpack-objects.c: static void use(int bytes)
      }
      
     -static void *get_data(unsigned long size)
    -+static void *get_data(unsigned long size, int dry_run)
    ++static void *get_data(size_t size, int dry_run)
      {
      	git_zstream stream;
     -	void *buf = xmallocz(size);
    -+	unsigned long bufsize;
    ++	size_t bufsize;
     +	void *buf;
      
      	memset(&stream, 0, sizeof(stream));
    @@ builtin/unpack-objects.c: static void unpack_delta_entry(enum object_type type,
      		if (dry_run || !delta_data) {
      			free(delta_data);
      			return;
    +
    + ## t/t5590-unpack-non-delta-objects.sh (new) ##
    +@@
    ++#!/bin/sh
    ++#
    ++# Copyright (c) 2021 Han Xin
    ++#
    ++
    ++test_description='Test unpack-objects with non-delta objects'
    ++
    ++GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
    ++export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
    ++
    ++. ./test-lib.sh
    ++
    ++prepare_dest () {
    ++	test_when_finished "rm -rf dest.git" &&
    ++	git init --bare dest.git
    ++}
    ++
    ++test_expect_success "setup repo with big blobs (1.5 MB)" '
    ++	test-tool genrandom foo 1500000 >big-blob &&
    ++	test_commit --append foo big-blob &&
    ++	test-tool genrandom bar 1500000 >big-blob &&
    ++	test_commit --append bar big-blob &&
    ++	(
    ++		cd .git &&
    ++		find objects/?? -type f | sort
    ++	) >expect &&
    ++	PACK=$(echo main | git pack-objects --revs test)
    ++'
    ++
    ++test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
    ++	GIT_ALLOC_LIMIT=1m &&
    ++	export GIT_ALLOC_LIMIT
    ++'
    ++
    ++test_expect_success 'fail to unpack-objects: cannot allocate' '
    ++	prepare_dest &&
    ++	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
    ++	grep "fatal: attempting to allocate" err &&
    ++	(
    ++		cd dest.git &&
    ++		find objects/?? -type f | sort
    ++	) >actual &&
    ++	test_file_not_empty actual &&
    ++	! test_cmp expect actual
    ++'
    ++
    ++test_expect_success 'unpack-objects dry-run' '
    ++	prepare_dest &&
    ++	git -C dest.git unpack-objects -n <test-$PACK.pack &&
    ++	(
    ++		cd dest.git &&
    ++		find objects/ -type f
    ++	) >actual &&
    ++	test_must_be_empty actual
    ++'
    ++
    ++test_done
-:  ---------- > 2:  0d2e0f3a00 object-file API: add a format_object_header() function
3:  8a704ecc59 ! 3:  a571b8f16c object-file.c: refactor write_loose_object() to reuse in stream version
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      	loose_object_path(the_repository, &filename, oid);
      
     -	fd = create_tmpfile(&tmp_file, filename.buf);
    -+	fd = create_tmpfile(&tmp_file, filename.buf, flags);
    - 	if (fd < 0) {
    +-	if (fd < 0) {
     -		if (flags & HASH_SILENT)
    --			ret = -1;
    +-			return -1;
     -		else if (errno == EACCES)
    --			ret = error(_("insufficient permission for adding an "
    --				      "object to repository database %s"),
    --				    get_object_directory());
    +-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
     -		else
    --			ret = error_errno(_("unable to create temporary file"));
    -+		ret = -1;
    - 		goto cleanup;
    - 	}
    - 
    +-			return error_errno(_("unable to create temporary file"));
    +-	}
    +-
     -	/* Set it up */
     -	git_deflate_init(&stream, zlib_compression_level);
     -	stream.next_out = compressed;
     -	stream.avail_out = sizeof(compressed);
     -	the_hash_algo->init_fn(&c);
    --
    ++	fd = create_tmpfile(&tmp_file, filename.buf, flags);
    ++	if (fd < 0)
    ++		return -1;
    + 
     -	/* First header.. */
     -	stream.next_in = (unsigned char *)hdr;
     -	stream.avail_in = hdrlen;
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
     -			warning_errno(_("failed utime() on %s"), tmp_file.buf);
     -	}
     -
    --	ret = finalize_object_file(tmp_file.buf, filename.buf);
    -+	ret = finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
    - cleanup:
    - 	strbuf_release(&filename);
    - 	strbuf_release(&tmp_file);
    +-	return finalize_object_file(tmp_file.buf, filename.buf);
    ++	return finalize_object_file_with_mtime(tmp_file.buf, filename.buf,
    ++					       mtime, flags);
    + }
    + 
    + static int freshen_loose_object(const struct object_id *oid)
4:  96f05632a2 ! 4:  1de06a8f5c object-file.c: make "write_object_file_flags()" to support read in stream
    @@ Metadata
     Author: Han Xin <[email protected]>
     
      ## Commit message ##
    -    object-file.c: make "write_object_file_flags()" to support read in stream
    +    object-file.c: add "write_stream_object_file()" to support read in stream
     
         We used to call "get_data()" in "unpack_non_delta_entry()" to read the
         entire contents of a blob object, no matter how big it is. This
         implementation may consume all the memory and cause OOM.
     
    -    This can be improved by feeding data to "stream_loose_object()" in a
    -    stream. The input stream is implemented as an interface.
    -
    -    When streaming a large blob object to "write_loose_object()", we have no
    -    chance to run "write_object_file_prepare()" to calculate the oid in
    -    advance. So we need to handle undetermined oid in a new function called
    -    "stream_loose_object()".
    +    This can be improved by feeding data to "write_stream_object_file()"
    +    in a stream. The input stream is implemented as an interface.
     
    +    The difference with "write_loose_object()" is that we have no chance
    +    to run "write_object_file_prepare()" to calculate the oid in advance.
         In "write_loose_object()", we know the oid and we can write the
         temporary file in the same directory as the final object, but for an
         object with an undetermined oid, we don't know the exact directory for
         the object, so we have to save the temporary file in ".git/objects/"
         directory instead.
     
    -    We will reuse "write_object_file_flags()" in "unpack_non_delta_entry()" to
    -    read the entire data contents in stream, so a new flag "HASH_STREAM" is
    -    added. When read in stream, we needn't prepare the "oid" before
    -    "write_loose_object()", only generate the header.
         "freshen_packed_object()" or "freshen_loose_object()" will be called
    -    inside "stream_loose_object()" after obtaining the "oid".
    +    inside "write_stream_object_file()" after obtaining the "oid".
     
    +    Helped-by: René Scharfe <[email protected]>
         Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
         Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
     
    - ## cache.h ##
    -@@ cache.h: int ie_modified(struct index_state *, const struct cache_entry *, struct stat *,
    - #define HASH_FORMAT_CHECK 2
    - #define HASH_RENORMALIZE  4
    - #define HASH_SILENT 8
    -+#define HASH_STREAM 16
    - int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
    - int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags);
    - 
    -
      ## object-file.c ##
     @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
      	return 1;
      }
      
    -+static int stream_loose_object(struct object_id *oid, char *hdr, int hdrlen,
    -+			       const struct input_stream *in_stream,
    -+			       unsigned long len, time_t mtime, unsigned flags)
    ++int write_stream_object_file(struct input_stream *in_stream, size_t len,
    ++			     enum object_type type, time_t mtime,
    ++			     unsigned flags, struct object_id *oid)
     +{
    -+	int fd, ret, err = 0, flush = 0;
    ++	int fd, ret, flush = 0;
     +	unsigned char compressed[4096];
     +	git_zstream stream;
     +	git_hash_ctx c;
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +	static struct strbuf tmp_file = STRBUF_INIT;
     +	static struct strbuf filename = STRBUF_INIT;
     +	int dirlen;
    ++	char hdr[MAX_HEADER_LEN];
    ++	int hdrlen = sizeof(hdr);
     +
    ++	/* Since "filename" is defined as static, it will be reused. So reset it
    ++	 * first before using it. */
    ++	strbuf_reset(&filename);
     +	/* When oid is not determined, save tmp file to odb path. */
     +	strbuf_addf(&filename, "%s/", get_object_directory());
     +
     +	fd = create_tmpfile(&tmp_file, filename.buf, flags);
    -+	if (fd < 0) {
    -+		err = -1;
    -+		goto cleanup;
    -+	}
    ++	if (fd < 0)
    ++		return -1;
    ++
    ++	hdrlen = format_object_header(hdr, hdrlen, type, len);
     +
     +	/* Set it up and write header */
     +	setup_stream_and_header(&stream, compressed, sizeof(compressed),
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +
     +	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
     +		unlink_or_warn(tmp_file.buf);
    -+		goto cleanup;
    ++		return 0;
     +	}
     +
     +	loose_object_path(the_repository, &filename, oid);
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +		struct strbuf dir = STRBUF_INIT;
     +		strbuf_add(&dir, filename.buf, dirlen - 1);
     +
    -+		if (mkdir_in_gitdir(dir.buf) < 0) {
    -+			err = -1;
    -+			goto cleanup;
    ++		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
    ++			ret = error_errno(_("unable to create directory %s"), dir.buf);
    ++			strbuf_release(&dir);
    ++			return ret;
     +		}
    ++		strbuf_release(&dir);
     +	}
     +
    -+	err = finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
    -+cleanup:
    -+	strbuf_release(&tmp_file);
    -+	strbuf_release(&filename);
    -+	return err;
    ++	return finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
     +}
     +
      int write_object_file_flags(const void *buf, unsigned long len,
      			    const char *type, struct object_id *oid,
      			    unsigned flags)
    -@@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
    - 	char hdr[MAX_HEADER_LEN];
    - 	int hdrlen = sizeof(hdr);
    - 
    -+	/* When streaming a large blob object (marked as HASH_STREAM),
    -+	 * we have no chance to run "write_object_file_prepare()" to
    -+	 * calculate the "oid" in advance.  Call "stream_loose_object()"
    -+	 * to write loose object in stream.
    -+	 */
    -+	if (flags & HASH_STREAM) {
    -+		hdrlen = generate_object_header(hdr, hdrlen, type, len);
    -+		return stream_loose_object(oid, hdr, hdrlen, buf, len, 0, flags);
    -+	}
    -+
    - 	/* Normally if we have it in the pack then we do not bother writing
    - 	 * it out into .git/objects/??/?{38} file.
    - 	 */
     
      ## object-store.h ##
     @@ object-store.h: struct object_directory {
    @@ object-store.h: struct object_directory {
      };
      
     +struct input_stream {
    -+	const void *(*read)(const struct input_stream *, unsigned long *len);
    ++	const void *(*read)(struct input_stream *, unsigned long *len);
     +	void *data;
     +};
     +
      KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
      	struct object_directory *, 1, fspathhash, fspatheq)
      
    +@@ object-store.h: static inline int write_object_file(const void *buf, unsigned long len,
    + 	return write_object_file_flags(buf, len, type, oid, 0);
    + }
    + 
    ++int write_stream_object_file(struct input_stream *in_stream, size_t len,
    ++			     enum object_type type, time_t mtime,
    ++			     unsigned flags, struct object_id *oid);
    ++
    + int hash_object_file_literally(const void *buf, unsigned long len,
    + 			       const char *type, struct object_id *oid,
    + 			       unsigned flags);
6:  476aaba527 ! 5:  e7b4e426ef unpack-objects: unpack_non_delta_entry() read data in a stream
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	int status;
     +};
     +
    -+static const void *feed_input_zstream(const struct input_stream *in_stream,
    ++static const void *feed_input_zstream(struct input_stream *in_stream,
     +				      unsigned long *readlen)
     +{
     +	struct input_zstream_data *data = in_stream->data;
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	return data->buf;
     +}
     +
    -+static void write_stream_blob(unsigned nr, unsigned long size)
    ++static void write_stream_blob(unsigned nr, size_t size)
     +{
     +	git_zstream zstream;
     +	struct input_zstream_data data;
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	data.zstream = &zstream;
     +	git_inflate_init(&zstream);
     +
    -+	if (write_object_file_flags(&in_stream, size,
    -+				    type_name(OBJ_BLOB),
    -+				    &obj_list[nr].oid,
    -+				    HASH_STREAM))
    ++	if (write_stream_object_file(&in_stream, size, OBJ_BLOB, 0, 0,
    ++				     &obj_list[nr].oid))
     +		die(_("failed to write object in stream"));
     +
     +	if (zstream.total_out != size || data.status != Z_STREAM_END)
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	git_inflate_end(&zstream);
     +
     +	if (strict) {
    -+		struct blob *blob = lookup_blob(the_repository, &obj_list[nr].oid);
    ++		struct blob *blob =
    ++			lookup_blob(the_repository, &obj_list[nr].oid);
     +		if (blob)
     +			blob->object.flags |= FLAG_WRITTEN;
     +		else
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
     +	void *buf;
     +
     +	/* Write large blob in stream without allocating full buffer. */
    -+	if (!dry_run && type == OBJ_BLOB && size > big_file_streaming_threshold) {
    ++	if (!dry_run && type == OBJ_BLOB &&
    ++	    size > big_file_streaming_threshold) {
     +		write_stream_blob(nr, size);
     +		return;
     +	}
    @@ environment.c: size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
      const char *editor_program;
      const char *askpass_program;
     
    - ## t/t5590-unpack-non-delta-objects.sh (new) ##
    -@@
    -+#!/bin/sh
    -+#
    -+# Copyright (c) 2021 Han Xin
    -+#
    -+
    -+test_description='Test unpack-objects when receive pack'
    -+
    -+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
    -+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
    -+
    -+. ./test-lib.sh
    -+
    -+prepare_dest () {
    -+	test_when_finished "rm -rf dest.git" &&
    -+	git init --bare dest.git &&
    -+	git -C dest.git config core.bigFileStreamingThreshold $1 &&
    -+	git -C dest.git config core.bigFileThreshold $1
    -+}
    -+
    -+test_expect_success "setup repo with big blobs (1.5 MB)" '
    -+	test-tool genrandom foo 1500000 >big-blob &&
    -+	test_commit --append foo big-blob &&
    -+	test-tool genrandom bar 1500000 >big-blob &&
    -+	test_commit --append bar big-blob &&
    -+	(
    -+		cd .git &&
    -+		find objects/?? -type f | sort
    -+	) >expect &&
    -+	PACK=$(echo main | git pack-objects --revs test)
    -+'
    -+
    -+test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
    -+	GIT_ALLOC_LIMIT=1m &&
    -+	export GIT_ALLOC_LIMIT
    -+'
    -+
    -+test_expect_success 'fail to unpack-objects: cannot allocate' '
    + ## t/t5590-unpack-non-delta-objects.sh ##
    +@@ t/t5590-unpack-non-delta-objects.sh: export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
    + prepare_dest () {
    + 	test_when_finished "rm -rf dest.git" &&
    + 	git init --bare dest.git
    ++	if test -n "$1"
    ++	then
    ++		git -C dest.git config core.bigFileStreamingThreshold $1
    ++		git -C dest.git config core.bigFileThreshold $1
    ++	fi
    + }
    + 
    + test_expect_success "setup repo with big blobs (1.5 MB)" '
    +@@ t/t5590-unpack-non-delta-objects.sh: test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
    + '
    + 
    + test_expect_success 'fail to unpack-objects: cannot allocate' '
    +-	prepare_dest &&
     +	prepare_dest 2m &&
    -+	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
    -+	grep "fatal: attempting to allocate" err &&
    -+	(
    -+		cd dest.git &&
    -+		find objects/?? -type f | sort
    -+	) >actual &&
    -+	test_file_not_empty actual &&
    -+	! test_cmp expect actual
    -+'
    -+
    + 	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
    + 	grep "fatal: attempting to allocate" err &&
    + 	(
    +@@ t/t5590-unpack-non-delta-objects.sh: test_expect_success 'fail to unpack-objects: cannot allocate' '
    + 	! test_cmp expect actual
    + '
    + 
     +test_expect_success 'unpack big object in stream' '
     +	prepare_dest 1m &&
    ++	mkdir -p dest.git/objects/05 &&
     +	git -C dest.git unpack-objects <test-$PACK.pack &&
     +	git -C dest.git fsck &&
     +	(
    @@ t/t5590-unpack-non-delta-objects.sh (new)
     +	test_must_be_empty actual
     +'
     +
    -+test_expect_success 'unpack-objects dry-run' '
    -+	prepare_dest 1m &&
    -+	git -C dest.git unpack-objects -n <test-$PACK.pack &&
    -+	(
    -+		cd dest.git &&
    -+		find objects/ -type f
    -+	) >actual &&
    -+	test_must_be_empty actual
    -+'
    -+
    -+test_done
    + test_expect_success 'unpack-objects dry-run' '
    + 	prepare_dest &&
    + 	git -C dest.git unpack-objects -n <test-$PACK.pack &&
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
author
#28 lore raw

PATCH v7 1/5 unpack-objects.c: add dry_run mode for get_data()

In reply to Han Xin

From: Han Xin <[email protected]>

In dry_run mode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even in dry_run mode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" in dry_run mode.

---
 builtin/unpack-objects.c            | 23 +++++++++---
 t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh
builtin/unpack-objects.c +17 -6
index 4a9466295b..9104eb48da 100644
@@ -96,15 +96,21 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
-static void *get_data(unsigned long size)
+static void *get_data(size_t size, int dry_run)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	size_t bufsize;
+	void *buf;
 
 	memset(&stream, 0, sizeof(stream));
+	if (dry_run && size > 8192)
+		bufsize = 8192;
+	else
+		bufsize = size;
+	buf = xmallocz(bufsize);
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -124,6 +130,11 @@ static void *get_data(unsigned long size)
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
 	return buf;
@@ -323,7 +334,7 @@ static void added_object(unsigned nr, enum object_type type,
 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 				   unsigned nr)
 {
-	void *buf = get_data(size);
+	void *buf = get_data(size, dry_run);
 
 	if (!dry_run && buf)
 		write_object(nr, type, buf, size);
@@ -357,7 +368,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 	if (type == OBJ_REF_DELTA) {
 		oidread(&base_oid, fill(the_hash_algo->rawsz));
 		use(the_hash_algo->rawsz);
-		delta_data = get_data(delta_size);
+		delta_data = get_data(delta_size, dry_run);
 		if (dry_run || !delta_data) {
 			free(delta_data);
 			return;
@@ -396,7 +407,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
 			die("offset value out of bound for delta base object");
 
-		delta_data = get_data(delta_size);
+		delta_data = get_data(delta_size, dry_run);
 		if (dry_run || !delta_data) {
 			free(delta_data);
 			return;
t/t5590-unpack-non-delta-objects.sh new file +57 -0

57 lines not on this page · raw message

Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
#29 lore raw

Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()

Quoted 28 lines · Han Xin

From: Han Xin <[email protected]>

In dry_run mode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even in dry_run mode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" in dry_run mode.

---
 builtin/unpack-objects.c            | 23 +++++++++---
 t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh
builtin/unpack-objects.c +1 -1
index 4a9466295b..9104eb48da 100644
@@ -96,15 +96,21 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
-static void *get_data(unsigned long size)
+static void *get_data(size_t size, int dry_run)
 {
 	git_zstream stream;

builtin/unpack-objects.c static void use(int bytes) · line 110

Quoted 10 lines
builtin/unpack-objects.c +7 -1
-	void *buf = xmallocz(size);
+	size_t bufsize;
+	void *buf;
 
 	memset(&stream, 0, sizeof(stream));
+	if (dry_run && size > 8192)
+		bufsize = 8192;
+	else
+		bufsize = size;
+	buf = xmallocz(bufsize);

Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "dry_run".

Removing that "&& size > 8192" makes all the tests pass still, so there seems to be some missing coverage here in any case.

Quoted 53 lines
t/t5590-unpack-non-delta-objects.sh new file +47 -0
new file mode 100755
index 0000000000..48c4fb1ba3
@@ -0,0 +1,57 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Han Xin
+#
+
+test_description='Test unpack-objects with non-delta objects'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+prepare_dest () {
+	test_when_finished "rm -rf dest.git" &&
+	git init --bare dest.git
+}
+
+test_expect_success "setup repo with big blobs (1.5 MB)" '
+	test-tool genrandom foo 1500000 >big-blob &&
+	test_commit --append foo big-blob &&
+	test-tool genrandom bar 1500000 >big-blob &&
+	test_commit --append bar big-blob &&
+	(
+		cd .git &&
+		find objects/?? -type f | sort
+	) >expect &&
+	PACK=$(echo main | git pack-objects --revs test)
+'
+
+test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
+	GIT_ALLOC_LIMIT=1m &&
+	export GIT_ALLOC_LIMIT
+'
+
+test_expect_success 'fail to unpack-objects: cannot allocate' '
+	prepare_dest &&
+	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
+	grep "fatal: attempting to allocate" err &&
+	(
+		cd dest.git &&
+		find objects/?? -type f | sort
+	) >actual &&
+	test_file_not_empty actual &&
+	! test_cmp expect actual
+'
+
+test_expect_success 'unpack-objects dry-run' '

t/t5590-unpack-non-delta-objects.sh line 57

Quoted 10 lines
t/t5590-unpack-non-delta-objects.sh new file +10 -0
+	prepare_dest &&
+	git -C dest.git unpack-objects -n <test-$PACK.pack &&
+	(
+		cd dest.git &&
+		find objects/ -type f
+	) >actual &&
+	test_must_be_empty actual
+'
+
+test_done

I commented on this "find" usage in an earlier round, I think there's a much easier way to do this. You're really just going back and forth between checking whether or not all the objects are loose.

I think that the below fix-up on top of this series is a better way to do that, and more accurate. I.e. in your test here you check "! test_cmp", which means that we could have some packed and some loose, but really what you're meaning to check is a flip-flop between "all loose?" and "no loose?.

In addition to that there was no reason to hardcode "main", we can just use HEAD. All in all I think the below fix-up makes sense:

t/t5590-unpack-non-delta-objects.sh +15 -36
index 8436cbf8db6..d78bb89225d 100755
@@ -5,9 +5,6 @@
 
 test_description='Test unpack-objects with non-delta objects'
 
-GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
-export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
-
 . ./test-lib.sh
 
 prepare_dest () {
@@ -20,16 +17,22 @@ prepare_dest () {
 	fi
 }
 
+assert_no_loose () {
+	glob=dest.git/objects/?? &&
+	echo "$glob" >expect &&
+	echo $glob >actual &&
+	test_cmp expect actual
+}
+
 test_expect_success "setup repo with big blobs (1.5 MB)" '
 	test-tool genrandom foo 1500000 >big-blob &&
 	test_commit --append foo big-blob &&
 	test-tool genrandom bar 1500000 >big-blob &&
 	test_commit --append bar big-blob &&
-	(
-		cd .git &&
-		find objects/?? -type f | sort
-	) >expect &&
-	PACK=$(echo main | git pack-objects --revs test)
+
+	# Everything is loose
+	rmdir .git/objects/pack &&
+	PACK=$(echo HEAD | git pack-objects --revs test)
 '
 
 test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
@@ -41,51 +44,27 @@ test_expect_success 'fail to unpack-objects: cannot allocate' '
 	prepare_dest 2m &&
 	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
 	grep "fatal: attempting to allocate" err &&
-	(
-		cd dest.git &&
-		find objects/?? -type f | sort
-	) >actual &&
-	test_file_not_empty actual &&
-	! test_cmp expect actual
+	rmdir dest.git/objects/pack
 '
 
 test_expect_success 'unpack big object in stream' '
 	prepare_dest 1m &&
 	mkdir -p dest.git/objects/05 &&
 	git -C dest.git unpack-objects <test-$PACK.pack &&
-	git -C dest.git fsck &&
-	(
-		cd dest.git &&
-		find objects/?? -type f | sort
-	) >actual &&
-	test_cmp expect actual
+	rmdir dest.git/objects/pack
 '
 
 test_expect_success 'unpack big object in stream with existing oids' '
 	prepare_dest 1m &&
 	git -C dest.git index-pack --stdin <test-$PACK.pack &&
-	(
-		cd dest.git &&
-		find objects/?? -type f | sort
-	) >actual &&
-	test_must_be_empty actual &&
 	git -C dest.git unpack-objects <test-$PACK.pack &&
-	git -C dest.git fsck &&
-	(
-		cd dest.git &&
-		find objects/?? -type f | sort
-	) >actual &&
-	test_must_be_empty actual
+	assert_no_loose
 '
 
 test_expect_success 'unpack-objects dry-run' '
 	prepare_dest &&
 	git -C dest.git unpack-objects -n <test-$PACK.pack &&
-	(
-		cd dest.git &&
-		find objects/ -type f
-	) >actual &&
-	test_must_be_empty actual
+	assert_no_loose
 '
 
 test_done
#30 lore raw

Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()

Am 21.12.21 um 15:09 schrieb Ævar Arnfjörð Bjarmason:

Quoted 35 lines · Han Xin
Quoted 33 lines

From: Han Xin <[email protected]>

In dry_run mode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even in dry_run mode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" in dry_run mode.

---
 builtin/unpack-objects.c            | 23 +++++++++---
 t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh
builtin/unpack-objects.c +3 -2
index 4a9466295b..9104eb48da 100644
@@ -96,15 +96,21 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
-static void *get_data(unsigned long size)
+static void *get_data(size_t size, int dry_run)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	size_t bufsize;
+	void *buf;
 
Quoted 10 lines

builtin/unpack-objects.c static void use(int bytes) · line 110

Quoted 6 lines
builtin/unpack-objects.c +5 -0
 	memset(&stream, 0, sizeof(stream));
+	if (dry_run && size > 8192)
+		bufsize = 8192;
+	else
+		bufsize = size;
+	buf = xmallocz(bufsize);

Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "dry_run".

This patch reduces the memory usage of dry runs, as its commit message says. The memory usage of one type of actual (non-dry) unpack is reduced by patch 5.

Quoted 2 lines

Removing that "&& size > 8192" makes all the tests pass still, so there seems to be some missing coverage here in any case.

How would you test that an 8KB buffer is allocated even though a smaller one would suffice? And why? Wasting a few KB shouldn't be noticeable.

René

#31 lore raw

Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()

Quoted 51 lines · René Scharfe

Am 21.12.21 um 15:09 schrieb Ævar Arnfjörð Bjarmason:

Quoted 35 lines · Han Xin
Quoted 33 lines

From: Han Xin <[email protected]>

In dry_run mode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even in dry_run mode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" in dry_run mode.

---
 builtin/unpack-objects.c            | 23 +++++++++---
 t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh
builtin/unpack-objects.c +3 -2
index 4a9466295b..9104eb48da 100644
@@ -96,15 +96,21 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
-static void *get_data(unsigned long size)
+static void *get_data(size_t size, int dry_run)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	size_t bufsize;
+	void *buf;
 
Quoted 10 lines

builtin/unpack-objects.c static void use(int bytes) · line 110

Quoted 6 lines
builtin/unpack-objects.c +5 -0
 	memset(&stream, 0, sizeof(stream));
+	if (dry_run && size > 8192)
+		bufsize = 8192;
+	else
+		bufsize = size;
+	buf = xmallocz(bufsize);

Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "dry_run".

This patch reduces the memory usage of dry runs, as its commit message says. The memory usage of one type of actual (non-dry) unpack is reduced by patch 5.

Quoted 5 lines
Quoted 2 lines

Removing that "&& size > 8192" makes all the tests pass still, so there seems to be some missing coverage here in any case.

How would you test that an 8KB buffer is allocated even though a smaller one would suffice? And why? Wasting a few KB shouldn't be noticeable.

That doesn't sound like it needs to be tested. I was just trying to grok what this was all doing. Thanks!

#32 lore raw

Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()

In reply to René Scharfe
Quoted 10 lines · René Scharfe

Am 21.12.21 um 15:09 schrieb Ævar Arnfjörð Bjarmason:

Quoted 3 lines

Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "dry_run".

This patch reduces the memory usage of dry runs, as its commit message says. The memory usage of one type of actual (non-dry) unpack is reduced by patch 5.

For Han Xin and me, it is very challenging to write better commit log in English. Since the commit is moved to the beginning, the commit log should be rewritten as follows:

unpack-objects.c: low memory footprint for get_data() in dry_run mode

As the name implies, "get_data(size)" will allocate and return a given size of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to resolve unpack issue of large blob objects, refactor "get_data()" to reduce memory footprint for dry_run mode. Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all.

Therefore, add the flag "dry_run" as an additional parameter of "get_data()" and reuse a small buffer in dry_run mode. Because in dry_run mode, the return buffer is not the entire data that the user wants, for this reason, we will release the buffer and return NULL.

Han Xin, I think you can try to free the allocated buffer for dry_run mode inside "get_data()".

--
Jiang Xin

#33 lore raw

Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()

In reply to Ævar Arnfjörð Bjarmason
Quoted 31 lines · Ævar Arnfjörð Bjarmason

On Tue, Dec 21 2021, Han Xin wrote:

I commented on this "find" usage in an earlier round, I think there's a much easier way to do this. You're really just going back and forth between checking whether or not all the objects are loose.

I think that the below fix-up on top of this series is a better way to do that, and more accurate. I.e. in your test here you check "! test_cmp", which means that we could have some packed and some loose, but really what you're meaning to check is a flip-flop between "all loose?" and "no loose?.

In addition to that there was no reason to hardcode "main", we can just use HEAD. All in all I think the below fix-up makes sense:

t/t5590-unpack-non-delta-objects.sh +0 -3
index 8436cbf8db6..d78bb89225d 100755
@@ -5,9 +5,6 @@
 
 test_description='Test unpack-objects with non-delta objects'
 
-GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
-export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
-
 . ./test-lib.sh
 
 prepare_dest () {

t/t5590-unpack-non-delta-objects.sh prepare_dest () { · line 23

Quoted 7 lines
t/t5590-unpack-non-delta-objects.sh +4 -0
@@ -20,16 +17,22 @@ prepare_dest () {
        fi
 }
 
+assert_no_loose () {
+       glob=dest.git/objects/?? &&
+       echo "$glob" >expect &&
+       echo $glob >actual &&

Incompatible for zsh. This may work:

    eval "echo $glob" >actual &&

--
Jiang Xin

#34 lore raw

Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()

In reply to Han Xin
Quoted 22 lines · Han Xin

From: Han Xin <[email protected]>

In dry_run mode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even in dry_run mode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" in dry_run mode.

---
 builtin/unpack-objects.c            | 23 +++++++++---
 t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)
 create mode 100755 t/t5590-unpack-non-delta-objects.sh
builtin/unpack-objects.c +0 -0
index 4a9466295b..9104eb48da 100644

builtin/unpack-objects.c static void use(int bytes) · line 99

Quoted 5 lines
builtin/unpack-objects.c +1 -1
@@ -96,15 +96,21 @@ static void use(int bytes)
        display_throughput(progress, consumed_bytes);
 }
 
-static void *get_data(unsigned long size)
+static void *get_data(size_t size, int dry_run)

After a offline talk with Han Xin, we feel it is not necessary to pass "dry_run" as a argument, use the file-scope static variable directly in "get_data()".

builtin/unpack-objects.c static void use(int bytes) · line 106

Quoted 8 lines
builtin/unpack-objects.c +3 -1
 {
        git_zstream stream;
-       void *buf = xmallocz(size);
+       size_t bufsize;
+       void *buf;
 
        memset(&stream, 0, sizeof(stream));
+       if (dry_run && size > 8192)

Use the file-scope static variable "dry_run".

author
#35 lore raw

PATCH v7 2/5 object-file API: add a format_object_header() function

In reply to Han Xin

From: Ævar Arnfjörð Bjarmason <[email protected]>

Add a convenience function to wrap the xsnprintf() command that generates loose object headers. This code was copy/pasted in various parts of the codebase, let's define it in one place and re-use it from there.

All except one caller of it had a valid "enum object_type" for us, it's only write_object_file_prepare() which might need to deal with "git hash-object --literally" and a potential garbage type. Let's have the primary API use an "enum object_type", and define an *_extended() function that can take an arbitrary "const char *" for the type.

See [1] for the discussion that prompted this patch, i.e. new code in object-file.c that wanted to copy/paste the xsnprintf() invocation.

1. https://lore.kernel.org/git/[email protected]/

---
 builtin/index-pack.c |  3 +--
 bulk-checkin.c       |  4 ++--
 cache.h              | 21 +++++++++++++++++++++
 http-push.c          |  2 +-
 object-file.c        | 14 +++++++++++---
 5 files changed, 36 insertions(+), 8 deletions(-)
builtin/index-pack.c +1 -2
index c23d01de7d..4a765ddae6 100644
@@ -449,8 +449,7 @@ static void *unpack_entry_data(off_t offset, unsigned long size,
 	int hdrlen;
 
 	if (!is_delta_type(type)) {
-		hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX,
-				   type_name(type),(uintmax_t)size) + 1;
+		hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)size);
 		the_hash_algo->init_fn(&c);
 		the_hash_algo->update_fn(&c, hdr, hdrlen);
 	} else
bulk-checkin.c +2 -2
index 8785b2ac80..1733a1de4f 100644
@@ -220,8 +220,8 @@ static int deflate_to_pack(struct bulk_checkin_state *state,
 	if (seekback == (off_t) -1)
 		return error("cannot find the current offset");
 
-	header_len = xsnprintf((char *)obuf, sizeof(obuf), "%s %" PRIuMAX,
-			       type_name(type), (uintmax_t)size) + 1;
+	header_len = format_object_header((char *)obuf, sizeof(obuf),
+					 type, (uintmax_t)size);
 	the_hash_algo->init_fn(&ctx);
 	the_hash_algo->update_fn(&ctx, obuf, header_len);
 
cache.h +21 -0
index cfba463aa9..64071a8d80 100644
@@ -1310,6 +1310,27 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
 						    unsigned long bufsiz,
 						    struct strbuf *hdrbuf);
 
+/**
+ * format_object_header() is a thin wrapper around s xsnprintf() that
+ * writes the initial "<type> <obj-len>" part of the loose object
+ * header. It returns the size that snprintf() returns + 1.
+ *
+ * The format_object_header_extended() function allows for writing a
+ * type_name that's not one of the "enum object_type" types. This is
+ * used for "git hash-object --literally". Pass in a OBJ_NONE as the
+ * type, and a non-NULL "type_str" to do that.
+ *
+ * format_object_header() is a convenience wrapper for
+ * format_object_header_extended().
+ */
+int format_object_header_extended(char *str, size_t size, enum object_type type,
+				 const char *type_str, size_t objsize);
+static inline int format_object_header(char *str, size_t size,
+				      enum object_type type, size_t objsize)
+{
+	return format_object_header_extended(str, size, type, NULL, objsize);
+}
+
 /**
  * parse_loose_header() parses the starting "<type> <len>\0" of an
  * object. If it doesn't follow that format -1 is returned. To check
http-push.c +1 -1
index 3309aaf004..f55e316ff4 100644
@@ -363,7 +363,7 @@ static void start_put(struct transfer_request *request)
 	git_zstream stream;
 
 	unpacked = read_object_file(&request->obj->oid, &type, &len);
-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
+	hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)len);
 
 	/* Set it up */
 	git_deflate_init(&stream, zlib_compression_level);
object-file.c +11 -3
index eb1426f98c..6bba4766f9 100644
@@ -1006,6 +1006,14 @@ void *xmmap(void *start, size_t length,
 	return ret;
 }
 
+int format_object_header_extended(char *str, size_t size, enum object_type type,
+				 const char *typestr, size_t objsize)
+{
+	const char *s = type == OBJ_NONE ? typestr : type_name(type);
+
+	return xsnprintf(str, size, "%s %"PRIuMAX, s, (uintmax_t)objsize) + 1;
+}
+
 /*
  * With an in-core object data in "map", rehash it to make sure the
  * object name actually matches "oid" to detect object corruption.
@@ -1034,7 +1042,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid,
 		return -1;
 
 	/* Generate the header */
-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(obj_type), (uintmax_t)size) + 1;
+	hdrlen = format_object_header(hdr, sizeof(hdr), obj_type, size);
 
 	/* Sha1.. */
 	r->hash_algo->init_fn(&c);
@@ -1734,7 +1742,7 @@ static void write_object_file_prepare(const struct git_hash_algo *algo,
 	git_hash_ctx c;
 
 	/* Generate the header */
-	*hdrlen = xsnprintf(hdr, *hdrlen, "%s %"PRIuMAX , type, (uintmax_t)len)+1;
+	*hdrlen = format_object_header_extended(hdr, *hdrlen, OBJ_NONE, type, len);
 
 	/* Sha1.. */
 	algo->init_fn(&c);
@@ -2006,7 +2014,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
 	buf = read_object(the_repository, oid, &type, &len);
 	if (!buf)
 		return error(_("cannot read object for %s"), oid_to_hex(oid));
-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
+	hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
 	ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0);
 	free(buf);
 
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
#36 lore raw

Re: [PATCH v7 2/5] object-file API: add a format_object_header() function

Am 21.12.21 um 12:51 schrieb Han Xin:

Quoted 33 lines

From: Ævar Arnfjörð Bjarmason <[email protected]>

Add a convenience function to wrap the xsnprintf() command that generates loose object headers. This code was copy/pasted in various parts of the codebase, let's define it in one place and re-use it from there.

All except one caller of it had a valid "enum object_type" for us, it's only write_object_file_prepare() which might need to deal with "git hash-object --literally" and a potential garbage type. Let's have the primary API use an "enum object_type", and define an *_extended() function that can take an arbitrary "const char *" for the type.

See [1] for the discussion that prompted this patch, i.e. new code in object-file.c that wanted to copy/paste the xsnprintf() invocation.

1. https://lore.kernel.org/git/[email protected]/

---
 builtin/index-pack.c |  3 +--
 bulk-checkin.c       |  4 ++--
 cache.h              | 21 +++++++++++++++++++++
 http-push.c          |  2 +-
 object-file.c        | 14 +++++++++++---
 5 files changed, 36 insertions(+), 8 deletions(-)
builtin/index-pack.c +0 -0
index c23d01de7d..4a765ddae6 100644

builtin/index-pack.c static void *unpack_entry_data(off_t offset, unsigned long size, · line 452

Quoted 6 lines
builtin/index-pack.c +1 -2
@@ -449,8 +449,7 @@ static void *unpack_entry_data(off_t offset, unsigned long size,
 	int hdrlen;
 
 	if (!is_delta_type(type)) {
-		hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX,
-				   type_name(type),(uintmax_t)size) + 1;
+		hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)size);
                                                                      ^^^^^^^^^^^

This explicit cast is unnecessary. It was needed with xsnprintf(), but that implementation detail is handled inside the new helper function.

(format_object_header() takes a size_t; even if unsigned long would be wider than that on some weird architecture, casting the size to uintmax_t will not avoid the implicit truncation happening during the function call.)

Quoted 8 lines
builtin/index-pack.c +0 -0
 		the_hash_algo->init_fn(&c);
 		the_hash_algo->update_fn(&c, hdr, hdrlen);
 	} else
bulk-checkin.c +0 -0
index 8785b2ac80..1733a1de4f 100644

bulk-checkin.c static int deflate_to_pack(struct bulk_checkin_state *state, · line 224

Quoted 7 lines
bulk-checkin.c +2 -2
@@ -220,8 +220,8 @@ static int deflate_to_pack(struct bulk_checkin_state *state,
 	if (seekback == (off_t) -1)
 		return error("cannot find the current offset");
 
-	header_len = xsnprintf((char *)obuf, sizeof(obuf), "%s %" PRIuMAX,
-			       type_name(type), (uintmax_t)size) + 1;
+	header_len = format_object_header((char *)obuf, sizeof(obuf),
+					 type, (uintmax_t)size);
                                               ^^^^^^^^^^^

Same here, just that size is already of type size_t, so a cast makes even less sense.

Quoted 40 lines
bulk-checkin.c +0 -0
 	the_hash_algo->init_fn(&ctx);
 	the_hash_algo->update_fn(&ctx, obuf, header_len);
 
cache.h +21 -0
index cfba463aa9..64071a8d80 100644
@@ -1310,6 +1310,27 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
 						    unsigned long bufsiz,
 						    struct strbuf *hdrbuf);
 
+/**
+ * format_object_header() is a thin wrapper around s xsnprintf() that
+ * writes the initial "<type> <obj-len>" part of the loose object
+ * header. It returns the size that snprintf() returns + 1.
+ *
+ * The format_object_header_extended() function allows for writing a
+ * type_name that's not one of the "enum object_type" types. This is
+ * used for "git hash-object --literally". Pass in a OBJ_NONE as the
+ * type, and a non-NULL "type_str" to do that.
+ *
+ * format_object_header() is a convenience wrapper for
+ * format_object_header_extended().
+ */
+int format_object_header_extended(char *str, size_t size, enum object_type type,
+				 const char *type_str, size_t objsize);
+static inline int format_object_header(char *str, size_t size,
+				      enum object_type type, size_t objsize)
+{
+	return format_object_header_extended(str, size, type, NULL, objsize);
+}
+
 /**
  * parse_loose_header() parses the starting "<type> <len>\0" of an
  * object. If it doesn't follow that format -1 is returned. To check
http-push.c +0 -0
index 3309aaf004..f55e316ff4 100644

http-push.c static void start_put(struct transfer_request *request) · line 366

Quoted 5 lines
http-push.c +1 -1
@@ -363,7 +363,7 @@ static void start_put(struct transfer_request *request)
 	git_zstream stream;
 
 	unpacked = read_object_file(&request->obj->oid, &type, &len);
-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
+	hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)len);
                                                              ^^^^^^^^^^^

Same here; len is of type unsigned long.

Quoted 8 lines
http-push.c +0 -0
 	/* Set it up */
 	git_deflate_init(&stream, zlib_compression_level);
object-file.c +0 -0
index eb1426f98c..6bba4766f9 100644

object-file.c void *xmmap(void *start, size_t length, · line 1014

Quoted 9 lines
object-file.c +6 -0
@@ -1006,6 +1006,14 @@ void *xmmap(void *start, size_t length,
 	return ret;
 }
 
+int format_object_header_extended(char *str, size_t size, enum object_type type,
+				 const char *typestr, size_t objsize)
+{
+	const char *s = type == OBJ_NONE ? typestr : type_name(type);
+
+	return xsnprintf(str, size, "%s %"PRIuMAX, s, (uintmax_t)objsize) + 1;
                                                      ^^^^^^^^^^^

This cast is necessary to match PRIuMAX. And that is used because the z modifier (as in e.g. printf("%zu", sizeof(size_t));) was only added in C99 and not all platforms may have it. (Perhaps this cautious approach is worth revisiting separately, now that some time has passed, but this patch series should still use PRIuMAX, as it does.)

Quoted 24 lines
object-file.c +4 -2
+}
+
 /*
  * With an in-core object data in "map", rehash it to make sure the
  * object name actually matches "oid" to detect object corruption.
@@ -1034,7 +1042,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid,
 		return -1;
 
 	/* Generate the header */
-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(obj_type), (uintmax_t)size) + 1;
+	hdrlen = format_object_header(hdr, sizeof(hdr), obj_type, size);
 
 	/* Sha1.. */
 	r->hash_algo->init_fn(&c);
@@ -1734,7 +1742,7 @@ static void write_object_file_prepare(const struct git_hash_algo *algo,
 	git_hash_ctx c;
 
 	/* Generate the header */
-	*hdrlen = xsnprintf(hdr, *hdrlen, "%s %"PRIuMAX , type, (uintmax_t)len)+1;
+	*hdrlen = format_object_header_extended(hdr, *hdrlen, OBJ_NONE, type, len);
 
 	/* Sha1.. */
 	algo->init_fn(&c);

object-file.c int force_object_loose(const struct object_id *oid, time_t mtime) · line 2020

Quoted 8 lines
object-file.c +1 -1
@@ -2006,7 +2014,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
 	buf = read_object(the_repository, oid, &type, &len);
 	if (!buf)
 		return error(_("cannot read object for %s"), oid_to_hex(oid));
-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
+	hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
 	ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0);
 	free(buf);
 

No explicit cast in these three cases -- good. They all pass an unsigned long as last parameter btw.

René

#37 lore raw

C99 %z (was: [PATCH v7 2/5] object-file API: add a format_object_header() function)

Quoted 14 lines · René Scharfe

Am 21.12.21 um 12:51 schrieb Han Xin:

Quoted 13 lines

From: Ævar Arnfjörð Bjarmason <[email protected]>
[...]

builtin/index-pack.c +0 -0
 		the_hash_algo->init_fn(&c);
 		the_hash_algo->update_fn(&c, hdr, hdrlen);
 	} else
bulk-checkin.c +0 -0
index 8785b2ac80..1733a1de4f 100644
@@ -220,8 +220,8 @@ static int deflate_to_pack(struct bulk_checkin_state *state,
 	if (seekback == (off_t) -1)
 		return error("cannot find the current offset");
 
Quoted 7 lines

bulk-checkin.c static int deflate_to_pack(struct bulk_checkin_state *state, · line 224

Quoted 4 lines
bulk-checkin.c +2 -2
-	header_len = xsnprintf((char *)obuf, sizeof(obuf), "%s %" PRIuMAX,
-			       type_name(type), (uintmax_t)size) + 1;
+	header_len = format_object_header((char *)obuf, sizeof(obuf),
+					 type, (uintmax_t)size);
                                               ^^^^^^^^^^^

Same here, just that size is already of type size_t, so a cast makes even less sense.

Thanks, this and the below is something I made sure to include in a re-roll I'm about to send (to do these cleanups in object-file.c separately from Han Xin's series).

Quoted 7 lines

bulk-checkin.c static int deflate_to_pack(struct bulk_checkin_state *state,

Quoted 6 lines
bulk-checkin.c +6 -0
+int format_object_header_extended(char *str, size_t size, enum object_type type,
+				 const char *typestr, size_t objsize)
+{
+	const char *s = type == OBJ_NONE ? typestr : type_name(type);
+
+	return xsnprintf(str, size, "%s %"PRIuMAX, s, (uintmax_t)objsize) + 1;
                                                      ^^^^^^^^^^^
Quoted 5 lines

This cast is necessary to match PRIuMAX. And that is used because the z modifier (as in e.g. printf("%zu", sizeof(size_t));) was only added in C99 and not all platforms may have it. (Perhaps this cautious approach is worth revisiting separately, now that some time has passed, but this patch series should still use PRIuMAX, as it does.)

I tried to use %z recently and found that the CI breaks on Windows, but this was a few months ago. But I think the status of that particular C99 feature is that we can't use it freely, unfortunately. I may be wrong about that, I haven't looked it any detail beyond running those CI errors.

#38 lore raw

Re: [PATCH v7 2/5] object-file API: add a format_object_header() function

In reply to Han Xin
Quoted 21 lines · Han Xin

From: Ævar Arnfjörð Bjarmason <[email protected]>

Add a convenience function to wrap the xsnprintf() command that generates loose object headers. This code was copy/pasted in various parts of the codebase, let's define it in one place and re-use it from there.

All except one caller of it had a valid "enum object_type" for us, it's only write_object_file_prepare() which might need to deal with "git hash-object --literally" and a potential garbage type. Let's have the primary API use an "enum object_type", and define an *_extended() function that can take an arbitrary "const char *" for the type.

See [1] for the discussion that prompted this patch, i.e. new code in object-file.c that wanted to copy/paste the xsnprintf() invocation.

1. https://lore.kernel.org/git/[email protected]/

Quoted 7 lines
---
 builtin/index-pack.c |  3 +--
 bulk-checkin.c       |  4 ++--
 cache.h              | 21 +++++++++++++++++++++
 http-push.c          |  2 +-
 object-file.c        | 14 +++++++++++---
 5 files changed, 36 insertions(+), 8 deletions(-)

After a offline review with Han Xin, we feel it's better to move this fixup commit to the end of this series, and this commit will also fix an additional "xsnprintf()" we introduced in this series.

--
Jiang Xin

author
#39 lore raw

PATCH v7 3/5 object-file.c: refactor write_loose_object() to reuse in stream version

In reply to Han Xin

From: Han Xin <[email protected]>

We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.

This can be improved by feeding data to "stream_loose_object()" in stream instead of read into the whole buf.

As this new method "stream_loose_object()" has many similarities with "write_loose_object()", we split up "write_loose_object()" into some steps:
1. Figuring out a path for the (temp) object file.
2. Creating the tempfile.
3. Setting up zlib and write header.
4. Write object data and handle errors.
5. Optionally, do someting after write, maybe force a loose object if
"mtime".

---
 object-file.c | 100 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 65 insertions(+), 35 deletions(-)
object-file.c +65 -35
index 6bba4766f9..e048f3d39e 100644
@@ -1751,6 +1751,25 @@ static void write_object_file_prepare(const struct git_hash_algo *algo,
 	algo->final_oid_fn(oid, &c);
 }
 
+/*
+ * Move the just written object with proper mtime into its final resting place.
+ */
+static int finalize_object_file_with_mtime(const char *tmpfile,
+					   const char *filename,
+					   time_t mtime,
+					   unsigned flags)
+{
+	struct utimbuf utb;
+
+	if (mtime) {
+		utb.actime = mtime;
+		utb.modtime = mtime;
+		if (utime(tmpfile, &utb) < 0 && !(flags & HASH_SILENT))
+			warning_errno(_("failed utime() on %s"), tmpfile);
+	}
+	return finalize_object_file(tmpfile, filename);
+}
+
 /*
  * Move the just written object into its final resting place.
  */
@@ -1836,7 +1855,8 @@ static inline int directory_size(const char *filename)
  * We want to avoid cross-directory filename renames, because those
  * can have problems on various filesystems (FAT, NFS, Coda).
  */
-static int create_tmpfile(struct strbuf *tmp, const char *filename)
+static int create_tmpfile(struct strbuf *tmp, const char *filename,
+			  unsigned flags)
 {
 	int fd, dirlen = directory_size(filename);
 
@@ -1844,7 +1864,9 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	strbuf_add(tmp, filename, dirlen);
 	strbuf_addstr(tmp, "tmp_obj_XXXXXX");
 	fd = git_mkstemp_mode(tmp->buf, 0444);
-	if (fd < 0 && dirlen && errno == ENOENT) {
+	do {
+		if (fd >= 0 || !dirlen || errno != ENOENT)
+			break;
 		/*
 		 * Make sure the directory exists; note that the contents
 		 * of the buffer are undefined after mkstemp returns an
@@ -1854,17 +1876,48 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 		strbuf_reset(tmp);
 		strbuf_add(tmp, filename, dirlen - 1);
 		if (mkdir(tmp->buf, 0777) && errno != EEXIST)
-			return -1;
+			break;
 		if (adjust_shared_perm(tmp->buf))
-			return -1;
+			break;
 
 		/* Try again */
 		strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
 		fd = git_mkstemp_mode(tmp->buf, 0444);
+	} while (0);
+
+	if (fd < 0 && !(flags & HASH_SILENT)) {
+		if (errno == EACCES)
+			return error(_("insufficient permission for adding an "
+				       "object to repository database %s"),
+				     get_object_directory());
+		else
+			return error_errno(_("unable to create temporary file"));
 	}
+
 	return fd;
 }
 
+static void setup_stream_and_header(git_zstream *stream,
+				    unsigned char *compressed,
+				    unsigned long compressed_size,
+				    git_hash_ctx *c,
+				    char *hdr,
+				    int hdrlen)
+{
+	/* Set it up */
+	git_deflate_init(stream, zlib_compression_level);
+	stream->next_out = compressed;
+	stream->avail_out = compressed_size;
+	the_hash_algo->init_fn(c);
+
+	/* First header.. */
+	stream->next_in = (unsigned char *)hdr;
+	stream->avail_in = hdrlen;
+	while (git_deflate(stream, 0) == Z_OK)
+		; /* nothing */
+	the_hash_algo->update_fn(c, hdr, hdrlen);
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1879,28 +1932,13 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
-	}
-
-	/* Set it up */
-	git_deflate_init(&stream, zlib_compression_level);
-	stream.next_out = compressed;
-	stream.avail_out = sizeof(compressed);
-	the_hash_algo->init_fn(&c);
+	fd = create_tmpfile(&tmp_file, filename.buf, flags);
+	if (fd < 0)
+		return -1;
 
-	/* First header.. */
-	stream.next_in = (unsigned char *)hdr;
-	stream.avail_in = hdrlen;
-	while (git_deflate(&stream, 0) == Z_OK)
-		; /* nothing */
-	the_hash_algo->update_fn(&c, hdr, hdrlen);
+	/* Set it up and write header */
+	setup_stream_and_header(&stream, compressed, sizeof(compressed),
+				&c, hdr, hdrlen);
 
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
@@ -1929,16 +1967,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	close_loose_object(fd);
 
-	if (mtime) {
-		struct utimbuf utb;
-		utb.actime = mtime;
-		utb.modtime = mtime;
-		if (utime(tmp_file.buf, &utb) < 0 &&
-		    !(flags & HASH_SILENT))
-			warning_errno(_("failed utime() on %s"), tmp_file.buf);
-	}
-
-	return finalize_object_file(tmp_file.buf, filename.buf);
+	return finalize_object_file_with_mtime(tmp_file.buf, filename.buf,
+					       mtime, flags);
 }
 
 static int freshen_loose_object(const struct object_id *oid)
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
#40 lore raw

Re: [PATCH v7 3/5] object-file.c: refactor write_loose_object() to reuse in stream version

Quoted 12 lines · Han Xin

From: Han Xin <[email protected]>
[...]

object-file.c +2 -2
@@ -1854,17 +1876,48 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 		strbuf_reset(tmp);
 		strbuf_add(tmp, filename, dirlen - 1);
 		if (mkdir(tmp->buf, 0777) && errno != EEXIST)
-			return -1;
+			break;
 		if (adjust_shared_perm(tmp->buf))
-			return -1;
+			break;
 

object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1892

Quoted 10 lines
object-file.c +7 -0
 		/* Try again */
 		strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
 		fd = git_mkstemp_mode(tmp->buf, 0444);
+	} while (0);
+
+	if (fd < 0 && !(flags & HASH_SILENT)) {
+		if (errno == EACCES)
+			return error(_("insufficient permission for adding an "
+				       "object to repository database %s"),
+				     get_object_directory());

This should be an error_errno() instead, ...

object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1894

Quoted 2 lines
object-file.c +2 -0
+		else
+			return error_errno(_("unable to create temporary file"));

...and we can just fold this whole if/else into one condition with a briefer message, e.g.:

    error_errno(_("unable to add object to '%s'"), get_object_directory());

Or whatever, unless there's another bug here where you inverted these conditions, and the "else" really should not use "error_errno" but "error".... (I don't know...)

#41 lore raw

Re: [PATCH v7 3/5] object-file.c: refactor write_loose_object() to reuse in stream version

Quoted 16 lines · Ævar Arnfjörð Bjarmason
Quoted 12 lines · Han Xin

From: Han Xin <[email protected]>
[...]

object-file.c +2 -2
@@ -1854,17 +1876,48 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
              strbuf_reset(tmp);
              strbuf_add(tmp, filename, dirlen - 1);
              if (mkdir(tmp->buf, 0777) && errno != EEXIST)
-                     return -1;
+                     break;
              if (adjust_shared_perm(tmp->buf))
-                     return -1;
+                     break;
 
Quoted 12 lines

object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1892

Quoted 10 lines
object-file.c +7 -0
              /* Try again */
              strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
              fd = git_mkstemp_mode(tmp->buf, 0444);
+     } while (0);
+
+     if (fd < 0 && !(flags & HASH_SILENT)) {
+             if (errno == EACCES)
+                     return error(_("insufficient permission for adding an "
+                                    "object to repository database %s"),
+                                  get_object_directory());

This should be an error_errno() instead, ...

We already know the errno (EACCESS) and output a decent error message, so using error() is OK. BTW, it's just a refactor by copy & paste.

Quoted 12 lines

object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1894

Quoted 3 lines
object-file.c +2 -0
+             else
+                     return error_errno(_("unable to create temporary file"));

...and we can just fold this whole if/else into one condition with a briefer message, e.g.:

    error_errno(_("unable to add object to '%s'"), get_object_directory());

Or whatever, unless there's another bug here where you inverted these conditions, and the "else" really should not use "error_errno" but "error".... (I don't know...)

author
#42 lore raw

PATCH v7 4/5 object-file.c: add "write_stream_object_file()" to support read in stream

In reply to Han Xin

From: Han Xin <[email protected]>

We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.

This can be improved by feeding data to "write_stream_object_file()" in a stream. The input stream is implemented as an interface.

The difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object, so we have to save the temporary file in ".git/objects/" directory instead.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "write_stream_object_file()" after obtaining the "oid".

---
 object-file.c  | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++
 object-store.h |  9 ++++++
 2 files changed, 94 insertions(+)
object-file.c +85 -0
index e048f3d39e..d0573e2a61 100644
@@ -1989,6 +1989,91 @@ static int freshen_packed_object(const struct object_id *oid)
 	return 1;
 }
 
+int write_stream_object_file(struct input_stream *in_stream, size_t len,
+			     enum object_type type, time_t mtime,
+			     unsigned flags, struct object_id *oid)
+{
+	int fd, ret, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct object_id parano_oid;
+	static struct strbuf tmp_file = STRBUF_INIT;
+	static struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen = sizeof(hdr);
+
+	/* Since "filename" is defined as static, it will be reused. So reset it
+	 * first before using it. */
+	strbuf_reset(&filename);
+	/* When oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+
+	fd = create_tmpfile(&tmp_file, filename.buf, flags);
+	if (fd < 0)
+		return -1;
+
+	hdrlen = format_object_header(hdr, hdrlen, type, len);
+
+	/* Set it up and write header */
+	setup_stream_and_header(&stream, compressed, sizeof(compressed),
+				&c, hdr, hdrlen);
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+		if (!stream.avail_in) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (len + hdrlen == stream.total_in + stream.avail_in)
+				flush = Z_FINISH;
+		}
+		ret = git_deflate(&stream, flush);
+		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
+		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
+			die(_("unable to write loose object file"));
+		stream.next_out = compressed;
+		stream.avail_out = sizeof(compressed);
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (ret != Z_STREAM_END)
+		die(_("unable to deflate new object streamingly (%d)"), ret);
+	ret = git_deflate_end_gently(&stream);
+	if (ret != Z_OK)
+		die(_("deflateEnd on object streamingly failed (%d)"), ret);
+	the_hash_algo->final_oid_fn(&parano_oid, &c);
+
+	close_loose_object(fd);
+
+	oidcpy(oid, &parano_oid);
+
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		return 0;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen - 1);
+
+		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
+			ret = error_errno(_("unable to create directory %s"), dir.buf);
+			strbuf_release(&dir);
+			return ret;
+		}
+		strbuf_release(&dir);
+	}
+
+	return finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
+}
+
 int write_object_file_flags(const void *buf, unsigned long len,
 			    const char *type, struct object_id *oid,
 			    unsigned flags)
object-store.h +9 -0
index 952efb6a4b..061b0cb2ba 100644
@@ -34,6 +34,11 @@ struct object_directory {
 	char *path;
 };
 
+struct input_stream {
+	const void *(*read)(struct input_stream *, unsigned long *len);
+	void *data;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -232,6 +237,10 @@ static inline int write_object_file(const void *buf, unsigned long len,
 	return write_object_file_flags(buf, len, type, oid, 0);
 }
 
+int write_stream_object_file(struct input_stream *in_stream, size_t len,
+			     enum object_type type, time_t mtime,
+			     unsigned flags, struct object_id *oid);
+
 int hash_object_file_literally(const void *buf, unsigned long len,
 			       const char *type, struct object_id *oid,
 			       unsigned flags);
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
#43 lore raw

Re: [PATCH v7 4/5] object-file.c: add "write_stream_object_file()" to support read in stream

Quoted 12 lines · Han Xin

From: Han Xin <[email protected]>
[...]

object-file.c +10 -0
+int write_stream_object_file(struct input_stream *in_stream, size_t len,
+			     enum object_type type, time_t mtime,
+			     unsigned flags, struct object_id *oid)
+{
+	int fd, ret, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct object_id parano_oid;
+	static struct strbuf tmp_file = STRBUF_INIT;

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2011

Quoted 10 lines
object-file.c +10 -0
+	static struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen = sizeof(hdr);
+
+	/* Since "filename" is defined as static, it will be reused. So reset it
+	 * first before using it. */
+	strbuf_reset(&filename);
+	/* When oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());

I realize this is somewhat following the pattern of code you moved around earlier, but FWIW I think these sorts of comments are really over-doing it. I.e. we try not to comment on things that are obvious from the code itself.

Also René's comment on v6 still applies here:

Given that this function is only used for huge objects I think making the strbufs non-static and releasing them is the best choice here.

I thin just making them non-static and doing a strbuf_release() as he suggested is best here.

Quoted 25 lines
object-file.c +25 -0
+
+	fd = create_tmpfile(&tmp_file, filename.buf, flags);
+	if (fd < 0)
+		return -1;
+
+	hdrlen = format_object_header(hdr, hdrlen, type, len);
+
+	/* Set it up and write header */
+	setup_stream_and_header(&stream, compressed, sizeof(compressed),
+				&c, hdr, hdrlen);
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+		if (!stream.avail_in) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (len + hdrlen == stream.total_in + stream.avail_in)
+				flush = Z_FINISH;
+		}
+		ret = git_deflate(&stream, flush);
+		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
+		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2046

Quoted 10 lines
object-file.c +10 -0
+			die(_("unable to write loose object file"));
+		stream.next_out = compressed;
+		stream.avail_out = sizeof(compressed);
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (ret != Z_STREAM_END)
+		die(_("unable to deflate new object streamingly (%d)"), ret);
+	ret = git_deflate_end_gently(&stream);
+	if (ret != Z_OK)
+		die(_("deflateEnd on object streamingly failed (%d)"), ret);

nit: let's say "unable to stream deflate new object" or something, and not use the confusing (invented?) word "streamingly".

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2051

Quoted 5 lines
object-file.c +5 -0
+	the_hash_algo->final_oid_fn(&parano_oid, &c);
+
+	close_loose_object(fd);
+
+	oidcpy(oid, &parano_oid);

I see there's still quite a bit of duplication between this and write_loose_object(), but maybe it's not easy to factor out.

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2064

Quoted 12 lines
object-file.c +12 -0
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		return 0;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen - 1);

Just a minor nit, but I noticed we could have this on top, i.e. this "remove the slash" is now what 1/3 users of it wan:

	 object-file.c | 10 +++++-----
	 1 file changed, 5 insertions(+), 5 deletions(-)
	
	diff --git a/object-file.c b/object-file.c
	index 77a3217fd0e..b0dea96906e 100644
	--- a/object-file.c
	+++ b/object-file.c
	@@ -1878,13 +1878,13 @@ static void close_loose_object(int fd)
	 		die_errno(_("error when closing loose object file"));
	 }
	 
	-/* Size of directory component, including the ending '/' */
	+/* Size of directory component, excluding the ending '/' */
	 static inline int directory_size(const char *filename)
	 {
	 	const char *s = strrchr(filename, '/');
	 	if (!s)
	 		return 0;
	-	return s - filename + 1;
	+	return s - filename;
	 }
	 
	 /*
	@@ -1901,7 +1901,7 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename,
	 
	 	strbuf_reset(tmp);
	 	strbuf_add(tmp, filename, dirlen);
	-	strbuf_addstr(tmp, "tmp_obj_XXXXXX");
	+	strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
	 	fd = git_mkstemp_mode(tmp->buf, 0444);
	 	do {
	 		if (fd >= 0 || !dirlen || errno != ENOENT)
	@@ -1913,7 +1913,7 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename,
	 		 * scratch.
	 		 */
	 		strbuf_reset(tmp);
	-		strbuf_add(tmp, filename, dirlen - 1);
	+		strbuf_add(tmp, filename, dirlen);
	 		if (mkdir(tmp->buf, 0777) && errno != EEXIST)
	 			break;
	 		if (adjust_shared_perm(tmp->buf))
	@@ -2100,7 +2100,7 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
	 	dirlen = directory_size(filename.buf);
	 	if (dirlen) {
	 		struct strbuf dir = STRBUF_INIT;
	-		strbuf_add(&dir, filename.buf, dirlen - 1);
	+		strbuf_add(&dir, filename.buf, dirlen);
	 
	 		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
	 			ret = error_errno(_("unable to create directory %s"), dir.buf);

On my platform (linux) it's not needed either way, a "mkdir foo" works as well as "mkdir foo/", but maybe some oS's have trouble with it.

#44 lore raw

Re: [PATCH v7 4/5] object-file.c: add "write_stream_object_file()" to support read in stream

Quoted 30 lines · Ævar Arnfjörð Bjarmason
Quoted 14 lines · Han Xin
(quoted diff) +14 -0
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+		if (!stream.avail_in) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (len + hdrlen == stream.total_in + stream.avail_in)
+				flush = Z_FINISH;
+		}
+		ret = git_deflate(&stream, flush);
+		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
+		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)

Review

Quoted 10 lines
(quoted diff) +10 -0
+			die(_("unable to write loose object file"));
+		stream.next_out = compressed;
+		stream.avail_out = sizeof(compressed);
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (ret != Z_STREAM_END)
+		die(_("unable to deflate new object streamingly (%d)"), ret);
+	ret = git_deflate_end_gently(&stream);
+	if (ret != Z_OK)
+		die(_("deflateEnd on object streamingly failed (%d)"), ret);

nit: let's say "unable to stream deflate new object" or something, and not use the confusing (invented?) word "streamingly".

Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+	the_hash_algo->final_oid_fn(&parano_oid, &c);
+
+	close_loose_object(fd);
+
+	oidcpy(oid, &parano_oid);

I see there's still quite a bit of duplication between this and write_loose_object(), but maybe it's not easy to factor out.

For what it's worth I tried to do that and the result doesn't really seem worth it. I.e. something like the below. The inner loop of the do/while looks like it could get a similar treatment, but likewise doesn't seem worth the effort.

object-file.c +52 -27
index b0dea96906e..7fc2363cfa1 100644
@@ -1957,6 +1957,46 @@ static void setup_stream_and_header(git_zstream *stream,
 	the_hash_algo->update_fn(c, hdr, hdrlen);
 }
 
+static int start_loose_object_common(struct strbuf *tmp_file,
+				     const char *filename, unsigned flags,
+				     git_zstream *stream,
+				     unsigned char *buf, size_t buflen,
+				     git_hash_ctx *c,
+				     enum object_type type, size_t len,
+				     char *hdr, int *hdrlen)
+{
+	int fd;
+
+	fd = create_tmpfile(tmp_file, filename, flags);
+	if (fd < 0)
+		return -1;
+
+	if (type != OBJ_NONE)
+		*hdrlen = format_object_header(hdr, *hdrlen, type, len);
+
+	/* Set it up and write header */
+	setup_stream_and_header(stream, buf, buflen, c, hdr, *hdrlen);
+
+	return fd;
+
+}
+
+static void end_loose_object_common(int ret, git_hash_ctx *c,
+				    git_zstream *stream,
+				    struct object_id *parano_oid,
+				    const struct object_id *expected_oid,
+				    const char *zstream_end_fmt,
+				    const char *z_ok_fmt)
+{
+	if (ret != Z_STREAM_END)
+		die(_(zstream_end_fmt), ret, expected_oid);
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		die(_(z_ok_fmt), ret, expected_oid);
+	the_hash_algo->final_oid_fn(parano_oid, c);
+}
+
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1970,15 +2010,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	static struct strbuf filename = STRBUF_INIT;
 
 	loose_object_path(the_repository, &filename, oid);
-
-	fd = create_tmpfile(&tmp_file, filename.buf, flags);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, OBJ_NONE, 0, hdr, &hdrlen);
 	if (fd < 0)
 		return -1;
 
-	/* Set it up and write header */
-	setup_stream_and_header(&stream, compressed, sizeof(compressed),
-				&c, hdr, hdrlen);
-
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
 	stream.avail_in = len;
@@ -1992,14 +2029,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		stream.avail_out = sizeof(compressed);
 	} while (ret == Z_OK);
 
-	if (ret != Z_STREAM_END)
-		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
-		    ret);
-	ret = git_deflate_end_gently(&stream);
-	if (ret != Z_OK)
-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
-		    ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
+	end_loose_object_common(ret, &c, &stream, &parano_oid, oid,
+				N_("unable to deflate new object %s (%d)"),
+				N_("deflateEnd on object %s failed (%d)"));
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
@@ -2049,16 +2081,12 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
 	/* When oid is not determined, save tmp file to odb path. */
 	strbuf_addf(&filename, "%s/", get_object_directory());
 
-	fd = create_tmpfile(&tmp_file, filename.buf, flags);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, type, len, hdr, &hdrlen);
 	if (fd < 0)
 		return -1;
 
-	hdrlen = format_object_header(hdr, hdrlen, type, len);
-
-	/* Set it up and write header */
-	setup_stream_and_header(&stream, compressed, sizeof(compressed),
-				&c, hdr, hdrlen);
-
 	/* Then the data itself.. */
 	do {
 		unsigned char *in0 = stream.next_in;
@@ -2078,12 +2106,9 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
 		stream.avail_out = sizeof(compressed);
 	} while (ret == Z_OK || ret == Z_BUF_ERROR);
 
-	if (ret != Z_STREAM_END)
-		die(_("unable to deflate new object streamingly (%d)"), ret);
-	ret = git_deflate_end_gently(&stream);
-	if (ret != Z_OK)
-		die(_("deflateEnd on object streamingly failed (%d)"), ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
+	end_loose_object_common(ret, &c, &stream, &parano_oid, NULL,
+				N_("unable to deflate new object streamingly (%d)"),
+				N_("deflateEnd on object streamingly failed (%d)"));
 
 	close_loose_object(fd);
 
author
#45 lore raw

PATCH v7 5/5 unpack-objects: unpack_non_delta_entry() read data in a stream

In reply to Han Xin

From: Han Xin <[email protected]>

We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.

By implementing a zstream version of input_stream interface, we can use a small fixed buffer for "unpack_non_delta_entry()".

However, unpack non-delta objects from a stream instead of from an entrie buffer will have 10% performance penalty. Therefore, only unpack object larger than the "core.BigFileStreamingThreshold" in zstream. See the following benchmarks:

hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git'

Summary
'./git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1' 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0' 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master' 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0' 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'

---
 Documentation/config/core.txt       | 11 +++++
 builtin/unpack-objects.c            | 73 ++++++++++++++++++++++++++++-
 cache.h                             |  1 +
 config.c                            |  5 ++
 environment.c                       |  1 +
 t/t5590-unpack-non-delta-objects.sh | 36 +++++++++++++-
 6 files changed, 125 insertions(+), 2 deletions(-)
Documentation/config/core.txt +11 -0
index c04f62a54a..601b7a2418 100644
@@ -424,6 +424,17 @@ be delta compressed, but larger binary media files won't be.
 +
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
+core.bigFileStreamingThreshold::
+	Files larger than this will be streamed out to a temporary
+	object file while being hashed, which will when be renamed
+	in-place to a loose object, particularly if the
+	`core.bigFileThreshold' setting dictates that they're always
+	written out as loose objects.
++
+Default is 128 MiB on all platforms.
++
+Common unit suffixes of 'k', 'm', or 'g' are supported.
+
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
 	describe paths that are not meant to be tracked, in addition
builtin/unpack-objects.c +72 -1

83 lines not on this page · raw message

cache.h +1 -0
index 64071a8d80..8c9123cb5d 100644
@@ -974,6 +974,7 @@ extern size_t packed_git_window_size;
 extern size_t packed_git_limit;
 extern size_t delta_base_cache_limit;
 extern unsigned long big_file_threshold;
+extern unsigned long big_file_streaming_threshold;
 extern unsigned long pack_size_limit_cfg;
 
 /*
config.c +5 -0
index c5873f3a70..7b122a142a 100644
@@ -1408,6 +1408,11 @@ static int git_default_core_config(const char *var, const char *value, void *cb)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.bigfilestreamingthreshold")) {
+		big_file_streaming_threshold = git_config_ulong(var, value);
+		return 0;
+	}
+
 	if (!strcmp(var, "core.packedgitlimit")) {
 		packed_git_limit = git_config_ulong(var, value);
 		return 0;
environment.c +1 -0
index 0d06a31024..04bba593de 100644
@@ -47,6 +47,7 @@ size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
 size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
 size_t delta_base_cache_limit = 96 * 1024 * 1024;
 unsigned long big_file_threshold = 512 * 1024 * 1024;
+unsigned long big_file_streaming_threshold = 128 * 1024 * 1024;
 int pager_use_color = 1;
 const char *editor_program;
 const char *askpass_program;
t/t5590-unpack-non-delta-objects.sh +35 -1
index 48c4fb1ba3..8436cbf8db 100755
@@ -13,6 +13,11 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
 prepare_dest () {
 	test_when_finished "rm -rf dest.git" &&
 	git init --bare dest.git
+	if test -n "$1"
+	then
+		git -C dest.git config core.bigFileStreamingThreshold $1
+		git -C dest.git config core.bigFileThreshold $1
+	fi
 }
 
 test_expect_success "setup repo with big blobs (1.5 MB)" '
@@ -33,7 +38,7 @@ test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
 '
 
 test_expect_success 'fail to unpack-objects: cannot allocate' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
 	grep "fatal: attempting to allocate" err &&
 	(
@@ -44,6 +49,35 @@ test_expect_success 'fail to unpack-objects: cannot allocate' '
 	! test_cmp expect actual
 '
 
+test_expect_success 'unpack big object in stream' '
+	prepare_dest 1m &&
+	mkdir -p dest.git/objects/05 &&
+	git -C dest.git unpack-objects <test-$PACK.pack &&
+	git -C dest.git fsck &&
+	(
+		cd dest.git &&
+		find objects/?? -type f | sort
+	) >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'unpack big object in stream with existing oids' '
+	prepare_dest 1m &&
+	git -C dest.git index-pack --stdin <test-$PACK.pack &&
+	(
+		cd dest.git &&
+		find objects/?? -type f | sort
+	) >actual &&
+	test_must_be_empty actual &&
+	git -C dest.git unpack-objects <test-$PACK.pack &&
+	git -C dest.git fsck &&
+	(
+		cd dest.git &&
+		find objects/?? -type f | sort
+	) >actual &&
+	test_must_be_empty actual
+'
+
 test_expect_success 'unpack-objects dry-run' '
 	prepare_dest &&
 	git -C dest.git unpack-objects -n <test-$PACK.pack &&
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
#46 lore raw

Re: [PATCH v7 5/5] unpack-objects: unpack_non_delta_entry() read data in a stream

Quoted 101 lines · Han Xin

From: Han Xin <[email protected]>

We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.

By implementing a zstream version of input_stream interface, we can use a small fixed buffer for "unpack_non_delta_entry()".

However, unpack non-delta objects from a stream instead of from an entrie buffer will have 10% performance penalty. Therefore, only unpack object larger than the "core.BigFileStreamingThreshold" in zstream. See the following benchmarks:

hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git'

Summary
'./git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1' 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0' 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master' 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0' 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'

---
 Documentation/config/core.txt       | 11 +++++
 builtin/unpack-objects.c            | 73 ++++++++++++++++++++++++++++-
 cache.h                             |  1 +
 config.c                            |  5 ++
 environment.c                       |  1 +
 t/t5590-unpack-non-delta-objects.sh | 36 +++++++++++++-
 6 files changed, 125 insertions(+), 2 deletions(-)
Documentation/config/core.txt +11 -0
index c04f62a54a..601b7a2418 100644
@@ -424,6 +424,17 @@ be delta compressed, but larger binary media files won't be.
 +
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
+core.bigFileStreamingThreshold::
+	Files larger than this will be streamed out to a temporary
+	object file while being hashed, which will when be renamed
+	in-place to a loose object, particularly if the
+	`core.bigFileThreshold' setting dictates that they're always
+	written out as loose objects.
++
+Default is 128 MiB on all platforms.
++
+Common unit suffixes of 'k', 'm', or 'g' are supported.
+
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
 	describe paths that are not meant to be tracked, in addition
builtin/unpack-objects.c +31 -0
index 9104eb48da..72d8616e00 100644
@@ -331,11 +331,82 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
+struct input_zstream_data {
+	git_zstream *zstream;
+	unsigned char buf[8192];
+	int status;
+};
+
+static const void *feed_input_zstream(struct input_stream *in_stream,
+				      unsigned long *readlen)
+{
+	struct input_zstream_data *data = in_stream->data;
+	git_zstream *zstream = data->zstream;
+	void *in = fill(1);
+
+	if (!len || data->status == Z_STREAM_END) {
+		*readlen = 0;
+		return NULL;
+	}
+
+	zstream->next_out = data->buf;
+	zstream->avail_out = sizeof(data->buf);
+	zstream->next_in = in;
+	zstream->avail_in = len;
+
+	data->status = git_inflate(zstream, 0);
+	use(len - zstream->avail_in);
+	*readlen = sizeof(data->buf) - zstream->avail_out;
+
+	return data->buf;
+}
+
+static void write_stream_blob(unsigned nr, size_t size)

builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 374

Quoted 10 lines
builtin/unpack-objects.c +10 -0
+{
+	git_zstream zstream;
+	struct input_zstream_data data;
+	struct input_stream in_stream = {
+		.read = feed_input_zstream,
+		.data = &data,
+	};
+
+	memset(&zstream, 0, sizeof(zstream));
+	memset(&data, 0, sizeof(data));

nit/style: both of these memset can be replaced by "{ 0 }", e.g. "git_zstream zstream = { 0 }".

builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 379

Quoted 5 lines
builtin/unpack-objects.c +5 -0
+	data.zstream = &zstream;
+	git_inflate_init(&zstream);
+
+	if (write_stream_object_file(&in_stream, size, OBJ_BLOB, 0, 0,
+				     &obj_list[nr].oid))

So at the end of this series we never pass in anything but blob here, mtime is always 0 etc. So there was no reason to create a factored out finalize_object_file_with_mtime() earlier in the series.

Well, I don't mind the finalize_object_file_with_mtime() exiting, but let's not pretend this is more generalized than it is. We're unlikely to ever want to do this for non-blobs.

This on top of this series (and my local WIP fixups as I'm reviewing this, so it won't cleanly apply, but the idea should be clear) makes this simpler:

	diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c
	index 2f8d34a2e47..a3a1d4b266f 100644
	--- a/builtin/unpack-objects.c
	+++ b/builtin/unpack-objects.c
	@@ -375,8 +375,7 @@ static void write_stream_blob(unsigned nr, size_t size)
	 	data.zstream = &zstream;
	 	git_inflate_init(&zstream);
	 
	-	if (write_stream_object_file(&in_stream, size, OBJ_BLOB, 0, 0,
	-				     &obj_list[nr].oid))
	+	if (write_stream_object_file(&in_stream, size, &obj_list[nr].oid))
	 		die(_("failed to write object in stream"));
	 
	 	if (zstream.total_out != size || data.status != Z_STREAM_END)
	diff --git a/object-file.c b/object-file.c
	index 7fc2363cfa1..0572b34fc5a 100644
	--- a/object-file.c
	+++ b/object-file.c
	@@ -2061,8 +2061,7 @@ static int freshen_packed_object(const struct object_id *oid)
	 }
	 
	 int write_stream_object_file(struct input_stream *in_stream, size_t len,
	-			     enum object_type type, time_t mtime,
	-			     unsigned flags, struct object_id *oid)
	+			     struct object_id *oid)
	 {
	 	int fd, ret, flush = 0;
	 	unsigned char compressed[4096];
	@@ -2081,9 +2080,9 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
	 	/* When oid is not determined, save tmp file to odb path. */
	 	strbuf_addf(&filename, "%s/", get_object_directory());
	 
	-	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
	+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
	 				       &stream, compressed, sizeof(compressed),
	-				       &c, type, len, hdr, &hdrlen);
	+				       &c, OBJ_BLOB, len, hdr, &hdrlen);
	 	if (fd < 0)
	 		return -1;
	 
	@@ -2135,7 +2134,7 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
	 		strbuf_release(&dir);
	 	}
	 
	-	return finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
	+	return finalize_object_file(tmp_file.buf, filename.buf);
	 }
	 
	 int write_object_file_flags(const void *buf, unsigned long len,
	diff --git a/object-store.h b/object-store.h
	index 87d370d39ca..1362b58a4d3 100644
	--- a/object-store.h
	+++ b/object-store.h
	@@ -257,8 +257,7 @@ int hash_write_object_file_literally(const void *buf, unsigned long len,
	 				     unsigned flags);
	 
	 int write_stream_object_file(struct input_stream *in_stream, size_t len,
	-			     enum object_type type, time_t mtime,
	-			     unsigned flags, struct object_id *oid);
	+			     struct object_id *oid);
	 
	 /*
	  * Add an object file to the in-memory object store, without writing it
Quoted 18 lines
builtin/unpack-objects.c +1 -0
+		die(_("failed to write object in stream"));
environment.c +1 -0
index 0d06a31024..04bba593de 100644
@@ -47,6 +47,7 @@ size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
 size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
 size_t delta_base_cache_limit = 96 * 1024 * 1024;
 unsigned long big_file_threshold = 512 * 1024 * 1024;
+unsigned long big_file_streaming_threshold = 128 * 1024 * 1024;
 int pager_use_color = 1;
 const char *editor_program;
 const char *askpass_program;
t/t5590-unpack-non-delta-objects.sh +0 -0
index 48c4fb1ba3..8436cbf8db 100755

t/t5590-unpack-non-delta-objects.sh export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME · line 20

Quoted 8 lines
t/t5590-unpack-non-delta-objects.sh +5 -0
@@ -13,6 +13,11 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
 prepare_dest () {
 	test_when_finished "rm -rf dest.git" &&
 	git init --bare dest.git
+	if test -n "$1"
+	then
+		git -C dest.git config core.bigFileStreamingThreshold $1
+		git -C dest.git config core.bigFileThreshold $1
+	fi

All of this new code is missing "&&" to chain & test forfailures.

#47 lore raw

Re: [PATCH v7 5/5] unpack-objects: unpack_non_delta_entry() read data in a stream

In reply to Han Xin
Quoted 46 lines · Han Xin

From: Han Xin <[email protected]>

We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.

By implementing a zstream version of input_stream interface, we can use a small fixed buffer for "unpack_non_delta_entry()".

However, unpack non-delta objects from a stream instead of from an entrie buffer will have 10% performance penalty. Therefore, only unpack object larger than the "core.BigFileStreamingThreshold" in zstream. See the following benchmarks:

hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git'

Summary
'./git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1' 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0' 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master' 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0' 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'

---
 Documentation/config/core.txt       | 11 +++++
 builtin/unpack-objects.c            | 73 ++++++++++++++++++++++++++++-
 cache.h                             |  1 +
 config.c                            |  5 ++
 environment.c                       |  1 +
 t/t5590-unpack-non-delta-objects.sh | 36 +++++++++++++-
 6 files changed, 125 insertions(+), 2 deletions(-)
Documentation/config/core.txt +0 -0
index c04f62a54a..601b7a2418 100644

Documentation/config/core.txt be delta compressed, but larger binary media files won't be. · line 432

Quoted 9 lines
Documentation/config/core.txt +6 -0
@@ -424,6 +424,17 @@ be delta compressed, but larger binary media files won't be.
 +
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
+core.bigFileStreamingThreshold::
+       Files larger than this will be streamed out to a temporary
+       object file while being hashed, which will when be renamed
+       in-place to a loose object, particularly if the
+       `core.bigFileThreshold' setting dictates that they're always
+       written out as loose objects.

Han Xin told me the reason to introduce another git config variable, but I feel it not good to introduce an application specific config variable as "core.XXX" and parsing it in "config.c".

So in patch v8, will still reuse the config variable "core.bigFileThreshold", and will introduce an application specific config variable, such as unpack.bigFileThreshold and parse the new config in "builtin/unpack-objects.c".

--
Jiang Xin

author
#48 lore raw

v8 cover unpack large blobs in stream

In reply to Han Xin

From: Han Xin <[email protected]>

Changes since v7:
* Use functions "assert_no_loose()" and "assert_no_pack()" to do tests instead of "find" sugguseted by Ævar Arnfjörð Bjarmason[1].

* "get_data()" now use the global "dry_run" and it will release the buf before returning.

* Add a new commit "object-file.c: remove the slash for directory_size()" sugguseted by Ævar Arnfjörð Bjarmason[2].

* Add "int is_finished" to "struct input_stream" who will tell us if there is next buffer in the stream.

* Remove the config "core.bigFileStreamingThreshold" introduced in v5, and keep using "core.bigFileThreshold". Until now, the config variable has been used in the cases listed in "unpack-objects: unpack_non_delta_entry() read data in a stream", this new case belongs to the packfile category.

* Remove unnecessary explicit cast in "object-file API: add a format_object_header() function" sugguseted by René Scharfe[3].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/[email protected]/

Han Xin (5):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: remove the slash for directory_size() object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream

Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function

 builtin/index-pack.c            |   3 +-
 builtin/unpack-objects.c        | 110 +++++++++++--
 bulk-checkin.c                  |   4 +-
 cache.h                         |  21 +++
 http-push.c                     |   2 +-
 object-file.c                   | 272 ++++++++++++++++++++++++++------
 object-store.h                  |   9 ++
 t/t5329-unpack-large-objects.sh |  69 ++++++++
 8 files changed, 422 insertions(+), 68 deletions(-)
 create mode 100755 t/t5329-unpack-large-objects.sh

Range-diff against v7:

1:  a8f232f553 < -:  ---------- unpack-objects.c: add dry_run mode for get_data()
-:  ---------- > 1:  bd34da5816 unpack-objects: low memory footprint for get_data() in dry_run mode
3:  a571b8f16c ! 2:  f9a4365a7d object-file.c: refactor write_loose_object() to reuse in stream version
    @@ Metadata
     Author: Han Xin <[email protected]>
     
      ## Commit message ##
    -    object-file.c: refactor write_loose_object() to reuse in stream version
    +    object-file.c: refactor write_loose_object() to several steps
     
    -    We used to call "get_data()" in "unpack_non_delta_entry()" to read the
    -    entire contents of a blob object, no matter how big it is. This
    -    implementation may consume all the memory and cause OOM.
    +    When writing a large blob using "write_loose_object()", we have to pass
    +    a buffer with the whole content of the blob, and this behavior will
    +    consume lots of memory and may cause OOM. We will introduce a stream
    +    version function ("stream_loose_object()") in latter commit to resolve
    +    this issue.
     
    -    This can be improved by feeding data to "stream_loose_object()" in
    -    stream instead of read into the whole buf.
    +    Before introducing a stream vesion function for writing loose object,
    +    do some refactoring on "write_loose_object()" to reuse code for both
    +    versions.
     
    -    As this new method "stream_loose_object()" has many similarities with
    -    "write_loose_object()", we split up "write_loose_object()" into some
    -    steps:
    -     1. Figuring out a path for the (temp) object file.
    -     2. Creating the tempfile.
    -     3. Setting up zlib and write header.
    -     4. Write object data and handle errors.
    -     5. Optionally, do someting after write, maybe force a loose object if
    -    "mtime".
    +    Rewrite "write_loose_object()" as follows:
    +
    +     1. Figure out a path for the (temp) object file. This step is only
    +        used in "write_loose_object()".
    +
    +     2. Move common steps for starting to write loose objects into a new
    +        function "start_loose_object_common()".
    +
    +     3. Compress data.
    +
    +     4. Move common steps for ending zlib stream into a new funciton
    +        "end_loose_object_common()".
    +
    +     5. Close fd and finalize the object file.
     
         Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
    +    Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
     
      ## object-file.c ##
    @@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
      	return fd;
      }
      
    -+static void setup_stream_and_header(git_zstream *stream,
    -+				    unsigned char *compressed,
    -+				    unsigned long compressed_size,
    -+				    git_hash_ctx *c,
    -+				    char *hdr,
    -+				    int hdrlen)
    ++static int start_loose_object_common(struct strbuf *tmp_file,
    ++				     const char *filename, unsigned flags,
    ++				     git_zstream *stream,
    ++				     unsigned char *buf, size_t buflen,
    ++				     git_hash_ctx *c,
    ++				     enum object_type type, size_t len,
    ++				     char *hdr, int hdrlen)
     +{
    -+	/* Set it up */
    ++	int fd;
    ++
    ++	fd = create_tmpfile(tmp_file, filename, flags);
    ++	if (fd < 0)
    ++		return -1;
    ++
    ++	/*  Setup zlib stream for compression */
     +	git_deflate_init(stream, zlib_compression_level);
    -+	stream->next_out = compressed;
    -+	stream->avail_out = compressed_size;
    ++	stream->next_out = buf;
    ++	stream->avail_out = buflen;
     +	the_hash_algo->init_fn(c);
     +
    -+	/* First header.. */
    ++	/*  Start to feed header to zlib stream */
     +	stream->next_in = (unsigned char *)hdr;
     +	stream->avail_in = hdrlen;
     +	while (git_deflate(stream, 0) == Z_OK)
     +		; /* nothing */
     +	the_hash_algo->update_fn(c, hdr, hdrlen);
    ++
    ++	return fd;
    ++}
    ++
    ++static void end_loose_object_common(int ret, git_hash_ctx *c,
    ++				    git_zstream *stream,
    ++				    struct object_id *parano_oid,
    ++				    const struct object_id *expected_oid,
    ++				    const char *die_msg1_fmt,
    ++				    const char *die_msg2_fmt)
    ++{
    ++	if (ret != Z_STREAM_END)
    ++		die(_(die_msg1_fmt), ret, expected_oid);
    ++	ret = git_deflate_end_gently(stream);
    ++	if (ret != Z_OK)
    ++		die(_(die_msg2_fmt), ret, expected_oid);
    ++	the_hash_algo->final_oid_fn(parano_oid, c);
     +}
     +
      static int write_loose_object(const struct object_id *oid, char *hdr,
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
     -	stream.next_out = compressed;
     -	stream.avail_out = sizeof(compressed);
     -	the_hash_algo->init_fn(&c);
    -+	fd = create_tmpfile(&tmp_file, filename.buf, flags);
    -+	if (fd < 0)
    -+		return -1;
    - 
    +-
     -	/* First header.. */
     -	stream.next_in = (unsigned char *)hdr;
     -	stream.avail_in = hdrlen;
     -	while (git_deflate(&stream, 0) == Z_OK)
     -		; /* nothing */
     -	the_hash_algo->update_fn(&c, hdr, hdrlen);
    -+	/* Set it up and write header */
    -+	setup_stream_and_header(&stream, compressed, sizeof(compressed),
    -+				&c, hdr, hdrlen);
    ++	/* Common steps for write_loose_object and stream_loose_object to
    ++	 * start writing loose oject:
    ++	 *
    ++	 *  - Create tmpfile for the loose object.
    ++	 *  - Setup zlib stream for compression.
    ++	 *  - Start to feed header to zlib stream.
    ++	 */
    ++	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
    ++				       &stream, compressed, sizeof(compressed),
    ++				       &c, OBJ_NONE, 0, hdr, hdrlen);
    ++	if (fd < 0)
    ++		return -1;
      
      	/* Then the data itself.. */
      	stream.next_in = (void *)buf;
     @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
    + 		stream.avail_out = sizeof(compressed);
    + 	} while (ret == Z_OK);
    + 
    +-	if (ret != Z_STREAM_END)
    +-		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
    +-		    ret);
    +-	ret = git_deflate_end_gently(&stream);
    +-	if (ret != Z_OK)
    +-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
    +-		    ret);
    +-	the_hash_algo->final_oid_fn(&parano_oid, &c);
    ++	/* Common steps for write_loose_object and stream_loose_object to
    ++	 * end writing loose oject:
    ++	 *
    ++	 *  - End the compression of zlib stream.
    ++	 *  - Get the calculated oid to "parano_oid".
    ++	 */
    ++	end_loose_object_common(ret, &c, &stream, &parano_oid, oid,
    ++				N_("unable to deflate new object %s (%d)"),
    ++				N_("deflateEnd on object %s failed (%d)"));
    ++
    + 	if (!oideq(oid, &parano_oid))
    + 		die(_("confused by unstable object source data for %s"),
    + 		    oid_to_hex(oid));
      
      	close_loose_object(fd);
-:  ---------- > 3:  18dd21122d object-file.c: remove the slash for directory_size()
-:  ---------- > 4:  964715451b object-file.c: add "stream_loose_object()" to handle large object
-:  ---------- > 5:  3f620466fe unpack-objects: unpack_non_delta_entry() read data in a stream
2:  0d2e0f3a00 ! 6:  8073a3888d object-file API: add a format_object_header() function
    @@ builtin/index-pack.c: static void *unpack_entry_data(off_t offset, unsigned long
      	if (!is_delta_type(type)) {
     -		hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX,
     -				   type_name(type),(uintmax_t)size) + 1;
    -+		hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)size);
    ++		hdrlen = format_object_header(hdr, sizeof(hdr), type, size);
      		the_hash_algo->init_fn(&c);
      		the_hash_algo->update_fn(&c, hdr, hdrlen);
      	} else
    @@ bulk-checkin.c: static int deflate_to_pack(struct bulk_checkin_state *state,
     -	header_len = xsnprintf((char *)obuf, sizeof(obuf), "%s %" PRIuMAX,
     -			       type_name(type), (uintmax_t)size) + 1;
     +	header_len = format_object_header((char *)obuf, sizeof(obuf),
    -+					 type, (uintmax_t)size);
    ++					 type, size);
      	the_hash_algo->init_fn(&ctx);
      	the_hash_algo->update_fn(&ctx, obuf, header_len);
      
    @@ http-push.c: static void start_put(struct transfer_request *request)
      
      	unpacked = read_object_file(&request->obj->oid, &type, &len);
     -	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
    -+	hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)len);
    ++	hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
      
      	/* Set it up */
      	git_deflate_init(&stream, zlib_compression_level);
    @@ object-file.c: static void write_object_file_prepare(const struct git_hash_algo
      
      	/* Sha1.. */
      	algo->init_fn(&c);
    +@@ object-file.c: int stream_loose_object(struct input_stream *in_stream, size_t len,
    + 
    + 	/* Since oid is not determined, save tmp file to odb path. */
    + 	strbuf_addf(&filename, "%s/", get_object_directory());
    +-	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), len) + 1;
    ++	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
    + 
    + 	/* Common steps for write_loose_object and stream_loose_object to
    + 	 * start writing loose oject:
     @@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
      	buf = read_object(the_repository, oid, &type, &len);
      	if (!buf)
4:  1de06a8f5c < -:  ---------- object-file.c: add "write_stream_object_file()" to support read in stream
5:  e7b4e426ef < -:  ---------- unpack-objects: unpack_non_delta_entry() read data in a stream
Signature · 1 line
2.34.1.52.gc288e771b4.agit.6.5.6
author
#49 lore raw

PATCH v9 1/5 unpack-objects: low memory footprint for get_data() in dry_run mode

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given size of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

---
 builtin/unpack-objects.c        | 39 +++++++++++++++++++--------
 t/t5328-unpack-large-objects.sh | 48 +++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+), 11 deletions(-)
 create mode 100755 t/t5328-unpack-large-objects.sh
builtin/unpack-objects.c +28 -11
index 4a9466295b..c6d6c17072 100644
@@ -96,15 +96,31 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
+/*
+ * Decompress zstream from stdin and return specific size of data.
+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
 static void *get_data(unsigned long size)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	unsigned long bufsize;
+	void *buf;
 
 	memset(&stream, 0, sizeof(stream));
+	if (dry_run && size > 8192)
+		bufsize = 8192;
+	else
+		bufsize = size;
+	buf = xmallocz(bufsize);
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -124,8 +140,15 @@ static void *get_data(unsigned long size)
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
+	if (dry_run)
+		FREE_AND_NULL(buf);
 	return buf;
 }
 
@@ -325,10 +348,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 {
 	void *buf = get_data(size);
 
-	if (!dry_run && buf)
+	if (buf)
 		write_object(nr, type, buf, size);
-	else
-		free(buf);
 }
 
 static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +379,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		oidread(&base_oid, fill(the_hash_algo->rawsz));
 		use(the_hash_algo->rawsz);
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		if (has_object_file(&base_oid))
 			; /* Ok we have this one */
 		else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +416,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 			die("offset value out of bound for delta base object");
 
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		lo = 0;
 		hi = nr;
 		while (lo < hi) {
t/t5328-unpack-large-objects.sh new file +48 -0

48 lines not on this page · raw message

Signature · 1 line
2.34.1.52.gc288e771b4.agit.6.5.6
author
#50 lore raw

v9 cover unpack large blobs in stream

In reply to Han Xin

From: Han Xin <[email protected]>

Changes since v8:
* Rename "assert_no_loose ()" into "test_no_loose ()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack ()" and use "test_dir_is_empty" instead.

* Revert changes to "create_tmpfile()" and error handling is now in "start_loose_object_common()".

* Remove "finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.

* Remove the commit "object-file.c: remove the slash for directory_size()", it can be in a separate patch if necessary.

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream

Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function

 builtin/index-pack.c            |   3 +-
 builtin/unpack-objects.c        | 110 ++++++++++++++--
 bulk-checkin.c                  |   4 +-
 cache.h                         |  21 +++
 http-push.c                     |   2 +-
 object-file.c                   | 220 +++++++++++++++++++++++++++-----
 object-store.h                  |   9 ++
 t/t5328-unpack-large-objects.sh |  65 ++++++++++
 8 files changed, 384 insertions(+), 50 deletions(-)
 create mode 100755 t/t5328-unpack-large-objects.sh

Range-diff against v8:
1: bd34da5816 ! 1: 6a6c11ba93 unpack-objects: low memory footprint for get_data() in dry_run mode

    @@ builtin/unpack-objects.c: static void unpack_delta_entry(enum object_type type,
      		hi = nr;
      		while (lo < hi) {
     
    - ## t/t5329-unpack-large-objects.sh (new) ##
    + ## t/t5328-unpack-large-objects.sh (new) ##
     @@
     +#!/bin/sh
     +#
    -+# Copyright (c) 2021 Han Xin
    ++# Copyright (c) 2022 Han Xin
     +#
     +
     +test_description='git unpack-objects with large objects'
    @@ t/t5329-unpack-large-objects.sh (new)
     +	git init --bare dest.git
     +}
     +
    -+assert_no_loose () {
    ++test_no_loose () {
     +	glob=dest.git/objects/?? &&
     +	echo "$glob" >expect &&
     +	eval "echo $glob" >actual &&
     +	test_cmp expect actual
     +}
     +
    -+assert_no_pack () {
    -+	rmdir dest.git/objects/pack
    -+}
    -+
     +test_expect_success "create large objects (1.5 MB) and PACK" '
     +	test-tool genrandom foo 1500000 >big-blob &&
     +	test_commit --append foo big-blob &&
    @@ t/t5329-unpack-large-objects.sh (new)
     +test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
     +	prepare_dest &&
     +	git -C dest.git unpack-objects -n <test-$PACK.pack &&
    -+	assert_no_loose &&
    -+	assert_no_pack
    ++	test_no_loose &&
    ++	test_dir_is_empty dest.git/objects/pack
     +'
     +
     +test_done
2:  f9a4365a7d ! 2:  bab9e0402f object-file.c: refactor write_loose_object() to several steps
    @@ Commit message
         Signed-off-by: Han Xin <[email protected]>
     
      ## object-file.c ##
    -@@ object-file.c: static void write_object_file_prepare(const struct git_hash_algo *algo,
    - 	algo->final_oid_fn(oid, &c);
    - }
    - 
    -+/*
    -+ * Move the just written object with proper mtime into its final resting place.
    -+ */
    -+static int finalize_object_file_with_mtime(const char *tmpfile,
    -+					   const char *filename,
    -+					   time_t mtime,
    -+					   unsigned flags)
    -+{
    -+	struct utimbuf utb;
    -+
    -+	if (mtime) {
    -+		utb.actime = mtime;
    -+		utb.modtime = mtime;
    -+		if (utime(tmpfile, &utb) < 0 && !(flags & HASH_SILENT))
    -+			warning_errno(_("failed utime() on %s"), tmpfile);
    -+	}
    -+	return finalize_object_file(tmpfile, filename);
    -+}
    -+
    - /*
    -  * Move the just written object into its final resting place.
    -  */
    -@@ object-file.c: static inline int directory_size(const char *filename)
    -  * We want to avoid cross-directory filename renames, because those
    -  * can have problems on various filesystems (FAT, NFS, Coda).
    -  */
    --static int create_tmpfile(struct strbuf *tmp, const char *filename)
    -+static int create_tmpfile(struct strbuf *tmp, const char *filename,
    -+			  unsigned flags)
    - {
    - 	int fd, dirlen = directory_size(filename);
    - 
    -@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
    - 	strbuf_add(tmp, filename, dirlen);
    - 	strbuf_addstr(tmp, "tmp_obj_XXXXXX");
    - 	fd = git_mkstemp_mode(tmp->buf, 0444);
    --	if (fd < 0 && dirlen && errno == ENOENT) {
    -+	do {
    -+		if (fd >= 0 || !dirlen || errno != ENOENT)
    -+			break;
    - 		/*
    - 		 * Make sure the directory exists; note that the contents
    - 		 * of the buffer are undefined after mkstemp returns an
     @@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
    - 		strbuf_reset(tmp);
    - 		strbuf_add(tmp, filename, dirlen - 1);
    - 		if (mkdir(tmp->buf, 0777) && errno != EEXIST)
    --			return -1;
    -+			break;
    - 		if (adjust_shared_perm(tmp->buf))
    --			return -1;
    -+			break;
    - 
    - 		/* Try again */
    - 		strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
    - 		fd = git_mkstemp_mode(tmp->buf, 0444);
    -+	} while (0);
    -+
    -+	if (fd < 0 && !(flags & HASH_SILENT)) {
    -+		if (errno == EACCES)
    -+			return error(_("insufficient permission for adding an "
    -+				       "object to repository database %s"),
    -+				     get_object_directory());
    -+		else
    -+			return error_errno(_("unable to create temporary file"));
    - 	}
    -+
      	return fd;
      }
      
    @@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
     +				     git_zstream *stream,
     +				     unsigned char *buf, size_t buflen,
     +				     git_hash_ctx *c,
    -+				     enum object_type type, size_t len,
     +				     char *hdr, int hdrlen)
     +{
     +	int fd;
     +
    -+	fd = create_tmpfile(tmp_file, filename, flags);
    -+	if (fd < 0)
    -+		return -1;
    ++	fd = create_tmpfile(tmp_file, filename);
    ++	if (fd < 0) {
    ++		if (flags & HASH_SILENT)
    ++			return -1;
    ++		else if (errno == EACCES)
    ++			return error(_("insufficient permission for adding "
    ++				       "an object to repository database %s"),
    ++				     get_object_directory());
    ++		else
    ++			return error_errno(
    ++				_("unable to create temporary file"));
    ++	}
     +
     +	/*  Setup zlib stream for compression */
     +	git_deflate_init(stream, zlib_compression_level);
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
     +	 */
     +	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
     +				       &stream, compressed, sizeof(compressed),
    -+				       &c, OBJ_NONE, 0, hdr, hdrlen);
    ++				       &c, hdr, hdrlen);
     +	if (fd < 0)
     +		return -1;
      
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      	if (!oideq(oid, &parano_oid))
      		die(_("confused by unstable object source data for %s"),
      		    oid_to_hex(oid));
    - 
    - 	close_loose_object(fd);
    - 
    --	if (mtime) {
    --		struct utimbuf utb;
    --		utb.actime = mtime;
    --		utb.modtime = mtime;
    --		if (utime(tmp_file.buf, &utb) < 0 &&
    --		    !(flags & HASH_SILENT))
    --			warning_errno(_("failed utime() on %s"), tmp_file.buf);
    --	}
    --
    --	return finalize_object_file(tmp_file.buf, filename.buf);
    -+	return finalize_object_file_with_mtime(tmp_file.buf, filename.buf,
    -+					       mtime, flags);
    - }
    - 
    - static int freshen_loose_object(const struct object_id *oid)
3:  18dd21122d < -:  ---------- object-file.c: remove the slash for directory_size()
4:  964715451b ! 3:  dd13614985 object-file.c: add "stream_loose_object()" to handle large object
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +	 */
     +	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
     +				       &stream, compressed, sizeof(compressed),
    -+				       &c, OBJ_BLOB, len, hdr, hdrlen);
    ++				       &c, hdr, hdrlen);
     +	if (fd < 0) {
     +		err = -1;
     +		goto cleanup;
5:  3f620466fe ! 4:  cd84e27b08 unpack-objects: unpack_non_delta_entry() read data in a stream
    @@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
      		write_object(nr, type, buf, size);
      }
     
    - ## t/t5329-unpack-large-objects.sh ##
    -@@ t/t5329-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
    + ## t/t5328-unpack-large-objects.sh ##
    +@@ t/t5328-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
      
      prepare_dest () {
      	test_when_finished "rm -rf dest.git" &&
    @@ t/t5329-unpack-large-objects.sh: test_description='git unpack-objects with large
     +	fi
      }
      
    - assert_no_loose () {
    -@@ t/t5329-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
    + test_no_loose () {
    +@@ t/t5328-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
      '
      
      test_expect_success 'unpack-objects failed under memory limitation' '
    @@ t/t5329-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1
     -	prepare_dest &&
     +	prepare_dest 2m &&
      	git -C dest.git unpack-objects -n <test-$PACK.pack &&
    - 	assert_no_loose &&
    - 	assert_no_pack
    + 	test_no_loose &&
    + 	test_dir_is_empty dest.git/objects/pack
      '
      
     +test_expect_success 'unpack big object in stream' '
     +	prepare_dest 1m &&
     +	git -C dest.git unpack-objects <test-$PACK.pack &&
    -+	assert_no_pack
    ++	test_dir_is_empty dest.git/objects/pack
     +'
     +
     +test_expect_success 'do not unpack existing large objects' '
     +	prepare_dest 1m &&
     +	git -C dest.git index-pack --stdin <test-$PACK.pack &&
     +	git -C dest.git unpack-objects <test-$PACK.pack &&
    -+	assert_no_loose
    ++	test_no_loose
     +'
     +
      test_done

6: 8073a3888d = 5: 59f0ad95c7 object-file API: add a format_object_header() function

Signature · 1 line
2.34.1.52.gc288e771b4.agit.6.5.6
#51 lore raw

Re: [PATCH v9 0/5] unpack large blobs in stream

Quoted 16 lines · Han Xin

From: Han Xin <[email protected]>

Changes since v8:
* Rename "assert_no_loose ()" into "test_no_loose ()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack ()" and use "test_dir_is_empty" instead.

* Revert changes to "create_tmpfile()" and error handling is now in "start_loose_object_common()".

* Remove "finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.

* Remove the commit "object-file.c: remove the slash for directory_size()", it can be in a separate patch if necessary.

Quoted 8 lines

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream

Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function

I sent
https://lore.kernel.org/git/[email protected]/
today which suggests splitting out the 5/5 cleanup you'd integrated.

I then rebased these patches of yours on top of that, the result is here:
https://github.com/avar/git/tree/han-xin-avar/unpack-loose-object-streaming-9

The range-diff to your version is below. There's a few unrelated fixes/nits in it.

I think with/without basing this on top of my series above your patches here look good with the nits pointed out in the diff below addressed (and some don't need to be). I.e. the dependency on it is rather trivial, and the two could be split up.

What do you think is a good way to proceed? I could just submit the below as a proposed v10 if you'd like & agree...

1: 553a9377eb3 ! 1: 61fcfe7b840 unpack-objects: low memory footprint for get_data() in dry_run mode
@@ Commit message
unpack-objects: low memory footprint for get_data() in dry_run mode

As the name implies, "get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
@@ Commit message

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void use(int bytes)
@@ t/t5328-unpack-large-objects.sh (new)
+
+test_no_loose () {
+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
++ echo $glob >expect &&
++ echo "$glob" >actual &&
+ test_cmp expect actual
+}
+

-:  ----------- > 2:  c6b0437db03 object-file.c: do fsync() and close() before post-write die()
2:  88c91affd61 ! 3:  77bcfe3da6f object-file.c: refactor write_loose_object() to several steps

@@ Commit message
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream
- version function ("stream_loose_object()") in latter commit to resolve
+ version function ("stream_loose_object()") in later commit to resolve
this issue.

- Before introducing a stream vesion function for writing loose object,
- do some refactoring on "write_loose_object()" to reuse code for both
- versions.
+ Before introducing that streaming function, do some refactoring on
+ "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

@@ Commit message

3. Compress data.

- 4. Move common steps for ending zlib stream into a new funciton
+ 4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".

5. Close fd and finalize the object file.
@@ Commit message

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

## object-file.c ##
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
return fd;
}

++/**
++ * Common steps for loose object writers to start writing loose
++ * objects:
++ *
++ * - Create tmpfile for the loose object.
++ * - Setup zlib stream for compression.
++ * - Start to feed header to zlib stream.
++ *
++ * Returns a "fd", which should later be provided to
++ * end_loose_object_common().
++ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+ const char *filename, unsigned flags,
+ git_zstream *stream,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ return fd;
+}
+
-+static void end_loose_object_common(int ret, git_hash_ctx *c,
++/**
++ * Common steps for loose object writers to end writing loose objects:
++ *
++ * - End the compression of zlib stream.
++ * - Get the calculated oid to "parano_oid".
++ * - fsync() and close() the "fd"
++ */
++static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+ git_zstream *stream,
+ struct object_id *parano_oid,
+ const struct object_id *expected_oid,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ if (ret != Z_OK)
+ die(_(die_msg2_fmt), ret, expected_oid);
+ the_hash_algo->final_oid_fn(parano_oid, c);
++
++ /*
++ * We already did a write_buffer() to the "fd", let's fsync()
++ * and close().
++ *
++ * We might still die() on a subsequent sanity check, but
++ * let's not add to that confusion by not flushing any
++ * outstanding writes to disk first.
++ */
++ close_loose_object(fd);
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- while (git_deflate(&stream, 0) == Z_OK)
- ; /* nothing */
- the_hash_algo->update_fn(&c, hdr, hdrlen);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * start writing loose oject:
-+ *
-+ * - Create tmpfile for the loose object.
-+ * - Setup zlib stream for compression.
-+ * - Start to feed header to zlib stream.
-+ */
+ fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+ &stream, compressed, sizeof(compressed),
+ &c, hdr, hdrlen);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
- ret);
- the_hash_algo->final_oid_fn(&parano_oid, &c);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * end writing loose oject:
-+ *
-+ * - End the compression of zlib stream.
-+ * - Get the calculated oid to "parano_oid".
-+ */
-+ end_loose_object_common(ret, &c, &stream, &parano_oid, oid,
+-
+- /*
+- * We already did a write_buffer() to the "fd", let's fsync()
+- * and close().
+- *
+- * We might still die() on a subsequent sanity check, but
+- * let's not add to that confusion by not flushing any
+- * outstanding writes to disk first.
+- */
+- close_loose_object(fd);
++ end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
+ N_("unable to deflate new object %s (%d)"),
+ N_("deflateEnd on object %s failed (%d)"));
-+
+
if (!oideq(oid, &parano_oid))
die(_("confused by unstable object source data for %s"),
- oid_to_hex(oid));
3: 054a00ed21d ! 4: 71c10e734d1 object-file.c: add "stream_loose_object()" to handle large object
@@ Commit message

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this
- function to unpack large blob object in latter commit.
+ function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for
- the object, so we have to save the temporary file in ".git/objects/"
- directory instead.
+ the object.
+
+ Still, we need to save the temporary file we're preparing
+ somewhere. We'll do that in the top-level ".git/objects/"
+ directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've
+ streamed it we'll know the OID, and will move it to its canonical
+ path.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+
+ /* Since oid is not determined, save tmp file to odb path. */
+ strbuf_addf(&filename, "%s/", get_object_directory());
-+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), len) + 1;
++ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+ /* Common steps for write_loose_object and stream_loose_object to
+ * start writing loose oject:
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ * - End the compression of zlib stream.
+ * - Get the calculated oid.
+ */
-+ end_loose_object_common(ret, &c, &stream, oid, NULL,
++ end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
+ N_("unable to stream deflate new object (%d)"),
+ N_("deflateEnd on stream object failed (%d)"));
+
-+ close_loose_object(fd);
-+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
+ goto cleanup;
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+}
+
int write_object_file_flags(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
+ enum object_type type, struct object_id *oid,
unsigned flags)

## object-store.h ##
@@ object-store.h: static inline int write_object_file(const void *buf, unsigned lo

+int stream_loose_object(struct input_stream *in_stream, size_t len,
+ struct object_id *oid);
-+
- int hash_object_file_literally(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
- unsigned flags);
+ int hash_write_object_file_literally(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags);

-:  ----------- > 5:  3c1d788d69d core doc: modernize core.bigFileThreshold documentation
4:  6bcba6bce66 ! 6:  8b83f6d6b83 unpack-objects: unpack_non_delta_entry() read data in a stream

@@ Metadata
Author: Han Xin <[email protected]>

## Commit message ##
- unpack-objects: unpack_non_delta_entry() read data in a stream
+ unpack-objects: use stream_loose_object() to unpack large objects

- We used to call "get_data()" in "unpack_non_delta_entry()" to read the
- entire contents of a blob object, no matter how big it is. This
- implementation may consume all the memory and cause OOM.
+ Make use of the stream_loose_object() function introduced in the
+ preceding commit to unpack large objects. Before this we'd need to
+ malloc() the size of the blob before unpacking it, which could cause
+ OOM with very large blobs.

- By implementing a zstream version of input_stream interface, we can use
- a small fixed buffer for "unpack_non_delta_entry()". However, unpack
- non-delta objects from a stream instead of from an entrie buffer will
- have 10% performance penalty.
+ We could use this new interface to unpack all blobs, but doing so
+ would result in a performance penalty of around 10%, as the below
+ "hyperfine" benchmark will show. We therefore limit this to files
+ larger than "core.bigFileThreshold":

$ hyperfine \
--setup \
@@ Commit message
-c core.bigFileThreshold=16k unpack-objects <small.pack' in 'HEAD~1'

- Therefore, only unpack objects larger than the "core.bigFileThreshold"
- in zstream. Until now, the config variable has been used in the
- following cases, and our new case belongs to the packfile category.
+ An earlier version of this patch introduced a new
+ "core.bigFileStreamingThreshold" instead of re-using the existing
+ "core.bigFileThreshold" variable[1]. As noted in a detailed overview
+ of its users in [2] using it has several different meanings.

- * Archive:
+ Still, we consider it good enough to simply re-use it. While it's
+ possible that someone might want to e.g. consider objects "small" for
+ the purposes of diffing but "big" for the purposes of writing them
+ such use-cases are probably too obscure to worry about. We can always
+ split up "core.bigFileThreshold" in the future if there's a need for
+ that.

- + archive.c: write_entry(): write large blob entries to archive in
- stream.
-
- * Loose objects:
-
- + object-file.c: index_fd(): when hashing large files in worktree,
- read files in a stream, and create one packfile per large blob if
- want to save files to git object store.
-
- + object-file.c: read_loose_object(): when checking loose objects
- using "git-fsck", do not read full content of large loose objects.
-
- * Packfile:
-
- + fast-import.c: parse_and_store_blob(): streaming large blob from
- foreign source to packfile.
-
- + index-pack.c: check_collison(): read and check large blob in stream.
-
- + index-pack.c: unpack_entry_data(): do not return the entire
- contents of the big blob from packfile, but uses a fixed buf to
- perform some integrity checks on the object.
-
- + pack-check.c: verify_packfile(): used by "git-fsck" and will call
- check_object_signature() to check large blob in pack with the
- streaming interface.
-
- + pack-objects.c: get_object_details(): set "no_try_delta" for large
- blobs when counting objects.
-
- + pack-objects.c: write_no_reuse_object(): streaming large blob to
- pack.
-
- + unpack-objects.c: unpack_non_delta_entry(): unpack large blob in
- stream from packfile.
-
- * Others:
-
- + diff.c: diff_populate_filespec(): treat large blob file as binary.
-
- + streaming.c: istream_source(): as a helper of "open_istream()" to
- select proper streaming interface to read large blob from packfile.
+ 1. https://lore.kernel.org/git/[email protected]/
+ 2. https://lore.kernel.org/git/[email protected]/

+ ## Documentation/config/core.txt ##
+@@ Documentation/config/core.txt: usage, at the slight expense of increased disk usage.
+ * Will be generally be streamed when written, which avoids excessive
+ memory usage, at the cost of some fixed overhead. Commands that make
+ use of this include linkgit:git-archive[1],
+-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +-linkgit:git-fsck[1].
++linkgit:git-fast-import[1], linkgit:git-index-pack[1], ++linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
+
+ core.excludesFile::
+ Specifies the pathname to the file that contains patterns to
+
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
}
5: 1bfaf89ee0b < -: ----------- object-file API: add a format_object_header() function

author
#52 lore raw

Re: [PATCH v9 0/5] unpack large blobs in stream

Quoted 40 lines · Ævar Arnfjörð Bjarmason
Quoted 16 lines · Han Xin

From: Han Xin <[email protected]>

Changes since v8:
* Rename "assert_no_loose ()" into "test_no_loose ()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack ()" and use "test_dir_is_empty" instead.

* Revert changes to "create_tmpfile()" and error handling is now in "start_loose_object_common()".

* Remove "finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.

* Remove the commit "object-file.c: remove the slash for directory_size()", it can be in a separate patch if necessary.

Quoted 8 lines

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream

Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function

I sent
https://lore.kernel.org/git/[email protected]/
today which suggests splitting out the 5/5 cleanup you'd integrated.

I then rebased these patches of yours on top of that, the result is here:
https://github.com/avar/git/tree/han-xin-avar/unpack-loose-object-streaming-9

The range-diff to your version is below. There's a few unrelated fixes/nits in it.

Quoted 8 lines

I think with/without basing this on top of my series above your patches here look good with the nits pointed out in the diff below addressed (and some don't need to be). I.e. the dependency on it is rather trivial, and the two could be split up.

What do you think is a good way to proceed? I could just submit the below as a proposed v10 if you'd like & agree...

Yes, thanks for the suggestions, and I'm glad you're happy to do so.

Thanks.
-Han Xin

Quoted 16 lines

1: 553a9377eb3 ! 1: 61fcfe7b840 unpack-objects: low memory footprint for get_data() in dry_run mode
@@ Commit message
unpack-objects: low memory footprint for get_data() in dry_run mode

As the name implies, "get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
@@ Commit message

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

Quoted 13 lines

## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void use(int bytes)
@@ t/t5328-unpack-large-objects.sh (new)
+
+test_no_loose () {
+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
++ echo $glob >expect &&
++ echo "$glob" >actual &&
+ test_cmp expect actual
+}
+

I have a small doubt with this, it works fine with dash, but not others like zsh. Wouldn't
it be better to do compatibility, or would it introduce other issues that I don't know?

Thanks.
-Han Xin

#53 lore raw

Re: [PATCH v9 0/5] unpack large blobs in stream

Quoted 46 lines · Han Xin
Quoted 45 lines · Ævar Arnfjörð Bjarmason
Quoted 16 lines · Han Xin

From: Han Xin <[email protected]>

Changes since v8:
* Rename "assert_no_loose ()" into "test_no_loose ()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack ()" and use "test_dir_is_empty" instead.

* Revert changes to "create_tmpfile()" and error handling is now in "start_loose_object_common()".

* Remove "finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.

* Remove the commit "object-file.c: remove the slash for directory_size()", it can be in a separate patch if necessary.

Quoted 8 lines

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream

Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function

I sent
https://lore.kernel.org/git/[email protected]/
today which suggests splitting out the 5/5 cleanup you'd integrated.

I then rebased these patches of yours on top of that, the result is here:
https://github.com/avar/git/tree/han-xin-avar/unpack-loose-object-streaming-9

The range-diff to your version is below. There's a few unrelated fixes/nits in it.

I think with/without basing this on top of my series above your patches here look good with the nits pointed out in the diff below addressed (and some don't need to be). I.e. the dependency on it is rather trivial, and the two could be split up.

Quoted 5 lines
Quoted 3 lines

What do you think is a good way to proceed? I could just submit the below as a proposed v10 if you'd like & agree...

Yes, thanks for the suggestions, and I'm glad you're happy to do so.

Willdo.

Quoted 30 lines
Quoted 16 lines

1: 553a9377eb3 ! 1: 61fcfe7b840 unpack-objects: low memory footprint for get_data() in dry_run mode
@@ Commit message
unpack-objects: low memory footprint for get_data() in dry_run mode

As the name implies, "get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
@@ Commit message

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

Quoted 13 lines

## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void use(int bytes)
@@ t/t5328-unpack-large-objects.sh (new)
+
+test_no_loose () {
+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
++ echo $glob >expect &&
++ echo "$glob" >actual &&
+ test_cmp expect actual
+}
+

Quoted 4 lines

I have a small doubt with this, it works fine with dash, but not others like zsh. Wouldn't
it be better to do compatibility, or would it introduce other issues that I don't know?

Ah, I hadn't spotted that zsh issue. I don't think the test suite will run on it in general, but in any case I'll fix this.

There's a few other tests that do this just by piping "find" to "wc -l", it's probably better to just follow that pattern. I think the eval works, but I thought it was a bit unusual/stood out.

#54 lore raw

v10 cover unpack-objects: support streaming large objects to disk

In reply to Han Xin

This is a v10 re-roll of Han Xin's series[1] to stream large objects to disk in "git unpack-objects". This v9 had integrated a proposed cleanup patch of mine, which is now a part of its own series, which this series now depends on: [2]. This v10 is sent with Han Xin's approval[3].

Changes since v9:

* Now based on [2]
* Small grammar/typo fixes in commit messages
* Replaced an echo/eval pattern in a test with a $(find ... | wc -l) comparison, which is a pattern we already use in another test for the same (or similar) assertion.
* I added a new 2/6 to do an fsync() before an oideq() assertion. I don't think it matters in practice, but allows 3/6 to be smaller by having that code-now-utility-function share more logic among its two callers.
* Changed inline comments in 3/6 to API docs where appropriate, the helper function now gets a "fd" per 2/6.
* 4/6 could use the format_object_header() function in the base topic, and now does so (instead of that conversion coming later in v9).
* A new 5/6 updates the core.bigFileThreshold documentation to account for 12 years of behavior changes we hadn't documented.
* The updated 6/6 now links to those docs, and I removed a very detailed accounting of all in-tree uses of core.bigFileThreshold from the commit message. I think linking to the summary docs should suffice, and for anyone digging in the future 5/6 links to the more detailed summary in the old patch.

More generally I've been heavily involved in the review for the past iterations, and I think barring any last minute nits in this v10 this topic should be ready to advance. As the above summary shows we're down to typo fixes, doc and test tweaks etc. at this point.

The core functionality being added here isn't changed in any meaningful way, and has had a lot of careful review already.

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/CAO0brD2Pe0aKSiBphZS861gC=nZk+q2GtXDN4pPjAQnPdns3TA@mail.gmail.com/

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects

Ævar Arnfjörð Bjarmason (2):
object-file.c: do fsync() and close() before post-write die() core doc: modernize core.bigFileThreshold documentation

 Documentation/config/core.txt   |  33 +++--
 builtin/unpack-objects.c        | 110 ++++++++++++++--
 object-file.c                   | 221 +++++++++++++++++++++++++++-----
 object-store.h                  |   8 ++
 t/t5328-unpack-large-objects.sh |  62 +++++++++
 5 files changed, 381 insertions(+), 53 deletions(-)
 create mode 100755 t/t5328-unpack-large-objects.sh

Range-diff against v9:
1: 553a9377eb3 ! 1: e46eb75b98f unpack-objects: low memory footprint for get_data() in dry_run mode
@@ Commit message
unpack-objects: low memory footprint for get_data() in dry_run mode

As the name implies, "get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
@@ Commit message
in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

+ The "find [...]objects/?? -type f | wc -l" test idiom being used here
+ is adapted from the same "find" use added to another test in
+ d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).
+

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void use(int bytes)
@@ t/t5328-unpack-large-objects.sh (new)
+}
+
+test_no_loose () {
-+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
-+ test_cmp expect actual
++ test $(find dest.git/objects/?? -type f | wc -l) = 0
+}
+
+test_expect_success "create large objects (1.5 MB) and PACK" '

-:  ----------- > 2:  48bf9090058 object-file.c: do fsync() and close() before post-write die()
2:  88c91affd61 ! 3:  0e33d2a6e35 object-file.c: refactor write_loose_object() to several steps

@@ Commit message
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream
- version function ("stream_loose_object()") in latter commit to resolve
+ version function ("stream_loose_object()") in later commit to resolve
this issue.

- Before introducing a stream vesion function for writing loose object,
- do some refactoring on "write_loose_object()" to reuse code for both
- versions.
+ Before introducing that streaming function, do some refactoring on
+ "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

@@ Commit message

3. Compress data.

- 4. Move common steps for ending zlib stream into a new funciton
+ 4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".

5. Close fd and finalize the object file.
@@ Commit message

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

## object-file.c ##
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
return fd;
}

++/**
++ * Common steps for loose object writers to start writing loose
++ * objects:
++ *
++ * - Create tmpfile for the loose object.
++ * - Setup zlib stream for compression.
++ * - Start to feed header to zlib stream.
++ *
++ * Returns a "fd", which should later be provided to
++ * end_loose_object_common().
++ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+ const char *filename, unsigned flags,
+ git_zstream *stream,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ return fd;
+}
+
-+static void end_loose_object_common(int ret, git_hash_ctx *c,
++/**
++ * Common steps for loose object writers to end writing loose objects:
++ *
++ * - End the compression of zlib stream.
++ * - Get the calculated oid to "parano_oid".
++ * - fsync() and close() the "fd"
++ */
++static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+ git_zstream *stream,
+ struct object_id *parano_oid,
+ const struct object_id *expected_oid,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ if (ret != Z_OK)
+ die(_(die_msg2_fmt), ret, expected_oid);
+ the_hash_algo->final_oid_fn(parano_oid, c);
++
++ /*
++ * We already did a write_buffer() to the "fd", let's fsync()
++ * and close().
++ *
++ * We might still die() on a subsequent sanity check, but
++ * let's not add to that confusion by not flushing any
++ * outstanding writes to disk first.
++ */
++ close_loose_object(fd);
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- while (git_deflate(&stream, 0) == Z_OK)
- ; /* nothing */
- the_hash_algo->update_fn(&c, hdr, hdrlen);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * start writing loose oject:
-+ *
-+ * - Create tmpfile for the loose object.
-+ * - Setup zlib stream for compression.
-+ * - Start to feed header to zlib stream.
-+ */
+ fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+ &stream, compressed, sizeof(compressed),
+ &c, hdr, hdrlen);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
- ret);
- the_hash_algo->final_oid_fn(&parano_oid, &c);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * end writing loose oject:
-+ *
-+ * - End the compression of zlib stream.
-+ * - Get the calculated oid to "parano_oid".
-+ */
-+ end_loose_object_common(ret, &c, &stream, &parano_oid, oid,
+-
+- /*
+- * We already did a write_buffer() to the "fd", let's fsync()
+- * and close().
+- *
+- * We might still die() on a subsequent sanity check, but
+- * let's not add to that confusion by not flushing any
+- * outstanding writes to disk first.
+- */
+- close_loose_object(fd);
++ end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
+ N_("unable to deflate new object %s (%d)"),
+ N_("deflateEnd on object %s failed (%d)"));
-+
+
if (!oideq(oid, &parano_oid))
die(_("confused by unstable object source data for %s"),
- oid_to_hex(oid));
3: 054a00ed21d ! 4: 9644df5c744 object-file.c: add "stream_loose_object()" to handle large object
@@ Commit message

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this
- function to unpack large blob object in latter commit.
+ function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for
- the object, so we have to save the temporary file in ".git/objects/"
- directory instead.
+ the object.
+
+ Still, we need to save the temporary file we're preparing
+ somewhere. We'll do that in the top-level ".git/objects/"
+ directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've
+ streamed it we'll know the OID, and will move it to its canonical
+ path.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+
+ /* Since oid is not determined, save tmp file to odb path. */
+ strbuf_addf(&filename, "%s/", get_object_directory());
-+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), len) + 1;
++ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+ /* Common steps for write_loose_object and stream_loose_object to
+ * start writing loose oject:
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ * - End the compression of zlib stream.
+ * - Get the calculated oid.
+ */
-+ end_loose_object_common(ret, &c, &stream, oid, NULL,
++ end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
+ N_("unable to stream deflate new object (%d)"),
+ N_("deflateEnd on stream object failed (%d)"));
+
-+ close_loose_object(fd);
-+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
+ goto cleanup;
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+}
+
int write_object_file_flags(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
+ enum object_type type, struct object_id *oid,
unsigned flags)

## object-store.h ##
@@ object-store.h: struct object_directory {
struct object_directory *, 1, fspathhash, fspatheq)

@@ object-store.h: static inline int write_object_file(const void *buf, unsigned long len,
- return write_object_file_flags(buf, len, type, oid, 0);
- }
-
+ int write_object_file_literally(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+ struct object_id *oid);
-+
- int hash_object_file_literally(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
- unsigned flags);
+
+ /*
+ * Add an object file to the in-memory object store, without writing it

-:  ----------- > 5:  4550f3a2745 core doc: modernize core.bigFileThreshold documentation
4:  6bcba6bce66 ! 6:  6a70e49a346 unpack-objects: unpack_non_delta_entry() read data in a stream

@@ Metadata
Author: Han Xin <[email protected]>

## Commit message ##
- unpack-objects: unpack_non_delta_entry() read data in a stream
+ unpack-objects: use stream_loose_object() to unpack large objects

- We used to call "get_data()" in "unpack_non_delta_entry()" to read the
- entire contents of a blob object, no matter how big it is. This
- implementation may consume all the memory and cause OOM.
+ Make use of the stream_loose_object() function introduced in the
+ preceding commit to unpack large objects. Before this we'd need to
+ malloc() the size of the blob before unpacking it, which could cause
+ OOM with very large blobs.

- By implementing a zstream version of input_stream interface, we can use
- a small fixed buffer for "unpack_non_delta_entry()". However, unpack
- non-delta objects from a stream instead of from an entrie buffer will
- have 10% performance penalty.
+ We could use this new interface to unpack all blobs, but doing so
+ would result in a performance penalty of around 10%, as the below
+ "hyperfine" benchmark will show. We therefore limit this to files
+ larger than "core.bigFileThreshold":

$ hyperfine \
--setup \
@@ Commit message
-c core.bigFileThreshold=16k unpack-objects <small.pack' in 'HEAD~1'

- Therefore, only unpack objects larger than the "core.bigFileThreshold"
- in zstream. Until now, the config variable has been used in the
- following cases, and our new case belongs to the packfile category.
+ An earlier version of this patch introduced a new
+ "core.bigFileStreamingThreshold" instead of re-using the existing
+ "core.bigFileThreshold" variable[1]. As noted in a detailed overview
+ of its users in [2] using it has several different meanings.

- * Archive:
+ Still, we consider it good enough to simply re-use it. While it's
+ possible that someone might want to e.g. consider objects "small" for
+ the purposes of diffing but "big" for the purposes of writing them
+ such use-cases are probably too obscure to worry about. We can always
+ split up "core.bigFileThreshold" in the future if there's a need for
+ that.

- + archive.c: write_entry(): write large blob entries to archive in
- stream.
-
- * Loose objects:
-
- + object-file.c: index_fd(): when hashing large files in worktree,
- read files in a stream, and create one packfile per large blob if
- want to save files to git object store.
-
- + object-file.c: read_loose_object(): when checking loose objects
- using "git-fsck", do not read full content of large loose objects.
-
- * Packfile:
-
- + fast-import.c: parse_and_store_blob(): streaming large blob from
- foreign source to packfile.
-
- + index-pack.c: check_collison(): read and check large blob in stream.
-
- + index-pack.c: unpack_entry_data(): do not return the entire
- contents of the big blob from packfile, but uses a fixed buf to
- perform some integrity checks on the object.
-
- + pack-check.c: verify_packfile(): used by "git-fsck" and will call
- check_object_signature() to check large blob in pack with the
- streaming interface.
-
- + pack-objects.c: get_object_details(): set "no_try_delta" for large
- blobs when counting objects.
-
- + pack-objects.c: write_no_reuse_object(): streaming large blob to
- pack.
-
- + unpack-objects.c: unpack_non_delta_entry(): unpack large blob in
- stream from packfile.
-
- * Others:
-
- + diff.c: diff_populate_filespec(): treat large blob file as binary.
-
- + streaming.c: istream_source(): as a helper of "open_istream()" to
- select proper streaming interface to read large blob from packfile.
+ 1. https://lore.kernel.org/git/[email protected]/
+ 2. https://lore.kernel.org/git/[email protected]/

+ ## Documentation/config/core.txt ##
+@@ Documentation/config/core.txt: usage, at the slight expense of increased disk usage.
+ * Will be generally be streamed when written, which avoids excessive
+ memory usage, at the cost of some fixed overhead. Commands that make
+ use of this include linkgit:git-archive[1],
+-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +-linkgit:git-fsck[1].
++linkgit:git-fast-import[1], linkgit:git-index-pack[1], ++linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
+
+ core.excludesFile::
+ Specifies the pathname to the file that contains patterns to
+
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
}
5: 1bfaf89ee0b < -: ----------- object-file API: add a format_object_header() function

Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#55 lore raw

PATCH v10 1/6 unpack-objects: low memory footprint for get_data() in dry_run mode

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).

---
 builtin/unpack-objects.c        | 39 ++++++++++++++++++++--------
 t/t5328-unpack-large-objects.sh | 45 +++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+), 11 deletions(-)
 create mode 100755 t/t5328-unpack-large-objects.sh
builtin/unpack-objects.c +28 -11
index dbeb0680a58..896ea8aceb4 100644
@@ -96,15 +96,31 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
+/*
+ * Decompress zstream from stdin and return specific size of data.
+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
 static void *get_data(unsigned long size)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	unsigned long bufsize;
+	void *buf;
 
 	memset(&stream, 0, sizeof(stream));
+	if (dry_run && size > 8192)
+		bufsize = 8192;
+	else
+		bufsize = size;
+	buf = xmallocz(bufsize);
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -124,8 +140,15 @@ static void *get_data(unsigned long size)
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
+	if (dry_run)
+		FREE_AND_NULL(buf);
 	return buf;
 }
 
@@ -325,10 +348,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 {
 	void *buf = get_data(size);
 
-	if (!dry_run && buf)
+	if (buf)
 		write_object(nr, type, buf, size);
-	else
-		free(buf);
 }
 
 static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +379,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		oidread(&base_oid, fill(the_hash_algo->rawsz));
 		use(the_hash_algo->rawsz);
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		if (has_object_file(&base_oid))
 			; /* Ok we have this one */
 		else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +416,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 			die("offset value out of bound for delta base object");
 
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		lo = 0;
 		hi = nr;
 		while (lo < hi) {
t/t5328-unpack-large-objects.sh new file +45 -0

45 lines not on this page · raw message

Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#56 lore raw

PATCH v10 2/6 object-file.c: do fsync() and close() before post-write die()

In reply to Ævar Arnfjörð Bjarmason

Change write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).

I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.

The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.

---
 object-file.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
object-file.c +11 -2
index 5c9525479c2..edebdc91221 100644
@@ -2001,12 +2001,21 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
 		    ret);
 	the_hash_algo->final_oid_fn(&parano_oid, &c);
+
+	/*
+	 * We already did a write_buffer() to the "fd", let's fsync()
+	 * and close().
+	 *
+	 * We might still die() on a subsequent sanity check, but
+	 * let's not add to that confusion by not flushing any
+	 * outstanding writes to disk first.
+	 */
+	close_loose_object(fd);
+
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
-	close_loose_object(fd);
-
 	if (mtime) {
 		struct utimbuf utb;
 		utb.actime = mtime;
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#57 lore raw

PATCH v10 3/6 object-file.c: refactor write_loose_object() to several steps

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.

Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".

2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".

3. Compress data.

4. Move common steps for ending zlib stream into a new function

    "end_loose_object_common()".

5. Close fd and finalize the object file.

---
 object-file.c | 129 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 89 insertions(+), 40 deletions(-)
object-file.c +89 -40
index edebdc91221..f5c579e42e3 100644
@@ -1943,6 +1943,87 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+/**
+ * Common steps for loose object writers to start writing loose
+ * objects:
+ *
+ * - Create tmpfile for the loose object.
+ * - Setup zlib stream for compression.
+ * - Start to feed header to zlib stream.
+ *
+ * Returns a "fd", which should later be provided to
+ * end_loose_object_common().
+ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+				     const char *filename, unsigned flags,
+				     git_zstream *stream,
+				     unsigned char *buf, size_t buflen,
+				     git_hash_ctx *c,
+				     char *hdr, int hdrlen)
+{
+	int fd;
+
+	fd = create_tmpfile(tmp_file, filename);
+	if (fd < 0) {
+		if (flags & HASH_SILENT)
+			return -1;
+		else if (errno == EACCES)
+			return error(_("insufficient permission for adding "
+				       "an object to repository database %s"),
+				     get_object_directory());
+		else
+			return error_errno(
+				_("unable to create temporary file"));
+	}
+
+	/*  Setup zlib stream for compression */
+	git_deflate_init(stream, zlib_compression_level);
+	stream->next_out = buf;
+	stream->avail_out = buflen;
+	the_hash_algo->init_fn(c);
+
+	/*  Start to feed header to zlib stream */
+	stream->next_in = (unsigned char *)hdr;
+	stream->avail_in = hdrlen;
+	while (git_deflate(stream, 0) == Z_OK)
+		; /* nothing */
+	the_hash_algo->update_fn(c, hdr, hdrlen);
+
+	return fd;
+}
+
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "parano_oid".
+ * - fsync() and close() the "fd"
+ */
+static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+				    git_zstream *stream,
+				    struct object_id *parano_oid,
+				    const struct object_id *expected_oid,
+				    const char *die_msg1_fmt,
+				    const char *die_msg2_fmt)
+{
+	if (ret != Z_STREAM_END)
+		die(_(die_msg1_fmt), ret, expected_oid);
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		die(_(die_msg2_fmt), ret, expected_oid);
+	the_hash_algo->final_oid_fn(parano_oid, c);
+
+	/*
+	 * We already did a write_buffer() to the "fd", let's fsync()
+	 * and close().
+	 *
+	 * We might still die() on a subsequent sanity check, but
+	 * let's not add to that confusion by not flushing any
+	 * outstanding writes to disk first.
+	 */
+	close_loose_object(fd);
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1957,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
-	}
-
-	/* Set it up */
-	git_deflate_init(&stream, zlib_compression_level);
-	stream.next_out = compressed;
-	stream.avail_out = sizeof(compressed);
-	the_hash_algo->init_fn(&c);
-
-	/* First header.. */
-	stream.next_in = (unsigned char *)hdr;
-	stream.avail_in = hdrlen;
-	while (git_deflate(&stream, 0) == Z_OK)
-		; /* nothing */
-	the_hash_algo->update_fn(&c, hdr, hdrlen);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0)
+		return -1;
 
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
@@ -1993,24 +2057,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		stream.avail_out = sizeof(compressed);
 	} while (ret == Z_OK);
 
-	if (ret != Z_STREAM_END)
-		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
-		    ret);
-	ret = git_deflate_end_gently(&stream);
-	if (ret != Z_OK)
-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
-		    ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
-
-	/*
-	 * We already did a write_buffer() to the "fd", let's fsync()
-	 * and close().
-	 *
-	 * We might still die() on a subsequent sanity check, but
-	 * let's not add to that confusion by not flushing any
-	 * outstanding writes to disk first.
-	 */
-	close_loose_object(fd);
+	end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
+				N_("unable to deflate new object %s (%d)"),
+				N_("deflateEnd on object %s failed (%d)"));
 
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#58 lore raw

PATCH v10 4/6 object-file.c: add "stream_loose_object()" to handle large object

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".

---
 object-file.c  | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++
 object-store.h |  8 ++++
 2 files changed, 107 insertions(+)
object-file.c +99 -0
index f5c579e42e3..2ef0bf0e5c3 100644
@@ -2095,6 +2095,105 @@ static int freshen_packed_object(const struct object_id *oid)
 	return 1;
 }
 
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid)
+{
+	int fd, ret, err = 0, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen;
+
+	/* Since oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+	/* Common steps for write_loose_object and stream_loose_object to
+	 * start writing loose oject:
+	 *
+	 *  - Create tmpfile for the loose object.
+	 *  - Setup zlib stream for compression.
+	 *  - Start to feed header to zlib stream.
+	 */
+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0) {
+		err = -1;
+		goto cleanup;
+	}
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+		if (!stream.avail_in && !in_stream->is_finished) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (in_stream->is_finished)
+				flush = Z_FINISH;
+		}
+		ret = git_deflate(&stream, flush);
+		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
+		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
+			die(_("unable to write loose object file"));
+		stream.next_out = compressed;
+		stream.avail_out = sizeof(compressed);
+		/*
+		 * Unlike write_loose_object(), we do not have the entire
+		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
+		 * then we'll replenish them in the next input_stream->read()
+		 * call when we loop.
+		 */
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (stream.total_in != len + hdrlen)
+		die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
+		    (uintmax_t)len + hdrlen);
+
+	/* Common steps for write_loose_object and stream_loose_object to
+	 * end writing loose oject:
+	 *
+	 *  - End the compression of zlib stream.
+	 *  - Get the calculated oid.
+	 */
+	end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
+				N_("unable to stream deflate new object (%d)"),
+				N_("deflateEnd on stream object failed (%d)"));
+
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		goto cleanup;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen);
+
+		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
+			err = error_errno(_("unable to create directory %s"), dir.buf);
+			strbuf_release(&dir);
+			goto cleanup;
+		}
+		strbuf_release(&dir);
+	}
+
+	err = finalize_object_file(tmp_file.buf, filename.buf);
+cleanup:
+	strbuf_release(&tmp_file);
+	strbuf_release(&filename);
+	return err;
+}
+
 int write_object_file_flags(const void *buf, unsigned long len,
 			    enum object_type type, struct object_id *oid,
 			    unsigned flags)
object-store.h +8 -0
index bd2322ed8ce..1099455bc2e 100644
@@ -46,6 +46,12 @@ struct object_directory {
 	char *path;
 };
 
+struct input_stream {
+	const void *(*read)(struct input_stream *, unsigned long *len);
+	void *data;
+	int is_finished;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -261,6 +267,8 @@ static inline int write_object_file(const void *buf, unsigned long len,
 int write_object_file_literally(const void *buf, unsigned long len,
 				const char *type, struct object_id *oid,
 				unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid);
 
 /*
  * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#59 lore raw

PATCH v10 5/6 core doc: modernize core.bigFileThreshold documentation

In reply to Ævar Arnfjörð Bjarmason

The core.bigFileThreshold documentation has been largely unchanged since 5eef828bc03 (fast-import: Stream very large blobs directly to pack, 2010-02-01).

But since then this setting has been expanded to affect a lot more than that description indicated. Most notably in how "git diff" treats them, see 6bf3b813486 (diff --stat: mark any file larger than core.bigfilethreshold binary, 2014-08-16).

In addition to that, numerous commands and APIs make use of a streaming mode for files above this threshold.

So let's attempt to summarize 12 years of changes in behavior, which can be seen with:

    git log --oneline -Gbig_file_thre 5eef828bc03.. -- '*.c'

To do that turn this into a bullet-point list. The summary Han Xin produced in [1] helped a lot, but is a bit too detailed for documentation aimed at users. Let's instead summarize how user-observable behavior differs, and generally describe how we tend to stream these files in various commands.

1. https://lore.kernel.org/git/[email protected]/

---
 Documentation/config/core.txt | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)
Documentation/config/core.txt +24 -9
index c04f62a54a1..b6a12218665 100644
@@ -412,17 +412,32 @@ You probably do not need to adjust this value.
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
 core.bigFileThreshold::
-	Files larger than this size are stored deflated, without
-	attempting delta compression.  Storing large files without
-	delta compression avoids excessive memory usage, at the
-	slight expense of increased disk usage. Additionally files
-	larger than this size are always treated as binary.
+	The size of files considered "big", which as discussed below
+	changes the behavior of numerous git commands, as well as how
+	such files are stored within the repository. The default is
+	512 MiB. Common unit suffixes of 'k', 'm', or 'g' are
+	supported.
 +
-Default is 512 MiB on all platforms.  This should be reasonable
-for most projects as source code and other text files can still
-be delta compressed, but larger binary media files won't be.
+Files above the configured limit will be:
 +
-Common unit suffixes of 'k', 'm', or 'g' are supported.
+* Stored deflated, without attempting delta compression.
++
+The default limit is primarily set with this use-case in mind. With it
+most projects will have their source code and other text files delta
+compressed, but not larger binary media files.
++
+Storing large files without delta compression avoids excessive memory
+usage, at the slight expense of increased disk usage.
++
+* Will be treated as if though they were labeled "binary" (see
+  linkgit:gitattributes[5]). This means that e.g. linkgit:git-log[1]
+  and linkgit:git-diff[1] will not diffs for files above this limit.
++
+* Will be generally be streamed when written, which avoids excessive
+memory usage, at the cost of some fixed overhead. Commands that make
+use of this include linkgit:git-archive[1],
+linkgit:git-fast-import[1], linkgit:git-index-pack[1] and
+linkgit:git-fsck[1].
 
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#60 lore raw

PATCH v10 6/6 unpack-objects: use stream_loose_object() to unpack large objects

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

Make use of the stream_loose_object() function introduced in the preceding commit to unpack large objects. Before this we'd need to malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.

We could use this new interface to unpack all blobs, but doing so would result in a performance penalty of around 10%, as the below "hyperfine" benchmark will show. We therefore limit this to files larger than "core.bigFileThreshold":

    $ hyperfine \
      --setup \
      'if ! test -d scalar.git; then git clone --bare
       https://github.com/microsoft/scalar.git;
       cp scalar.git/objects/pack/*.pack small.pack; fi' \
      --prepare 'rm -rf dest.git && git init --bare dest.git' \
      ...
    Summary
      './git -C dest.git -c core.bigFileThreshold=512m
      unpack-objects <small.pack' in 'origin/master'
        1.01 ± 0.04 times faster than './git -C dest.git
                -c core.bigFileThreshold=512m unpack-objects
                <small.pack' in 'HEAD~1'
        1.01 ± 0.04 times faster than './git -C dest.git
                -c core.bigFileThreshold=512m unpack-objects
                <small.pack' in 'HEAD~0'
        1.03 ± 0.10 times faster than './git -C dest.git
                -c core.bigFileThreshold=16k unpack-objects
                <small.pack' in 'origin/master'
        1.02 ± 0.07 times faster than './git -C dest.git
                -c core.bigFileThreshold=16k unpack-objects
                <small.pack' in 'HEAD~0'
        1.10 ± 0.04 times faster than './git -C dest.git
                -c core.bigFileThreshold=16k unpack-objects
                <small.pack' in 'HEAD~1'

An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing "core.bigFileThreshold" variable[1]. As noted in a detailed overview of its users in [2] using it has several different meanings.

Still, we consider it good enough to simply re-use it. While it's possible that someone might want to e.g. consider objects "small" for the purposes of diffing but "big" for the purposes of writing them such use-cases are probably too obscure to worry about. We can always split up "core.bigFileThreshold" in the future if there's a need for that.

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/

---
 Documentation/config/core.txt   |  4 +-
 builtin/unpack-objects.c        | 71 ++++++++++++++++++++++++++++++++-
 t/t5328-unpack-large-objects.sh | 23 +++++++++--
 3 files changed, 92 insertions(+), 6 deletions(-)
Documentation/config/core.txt +2 -2
index b6a12218665..5aca987632c 100644
@@ -436,8 +436,8 @@ usage, at the slight expense of increased disk usage.
 * Will be generally be streamed when written, which avoids excessive
 memory usage, at the cost of some fixed overhead. Commands that make
 use of this include linkgit:git-archive[1],
-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and
-linkgit:git-fsck[1].
+linkgit:git-fast-import[1], linkgit:git-index-pack[1],
+linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
 
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
builtin/unpack-objects.c +70 -1

81 lines not on this page · raw message

t/t5328-unpack-large-objects.sh +20 -3
index 1432dfc8386..5c1042b4d91 100755
@@ -9,7 +9,11 @@ test_description='git unpack-objects with large objects'
 
 prepare_dest () {
 	test_when_finished "rm -rf dest.git" &&
-	git init --bare dest.git
+	git init --bare dest.git &&
+	if test -n "$1"
+	then
+		git -C dest.git config core.bigFileThreshold $1
+	fi
 }
 
 test_no_loose () {
@@ -30,16 +34,29 @@ test_expect_success 'set memory limitation to 1MB' '
 '
 
 test_expect_success 'unpack-objects failed under memory limitation' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
 	grep "fatal: attempting to allocate" err
 '
 
 test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	git -C dest.git unpack-objects -n <test-$PACK.pack &&
 	test_no_loose &&
 	test_dir_is_empty dest.git/objects/pack
 '
 
+test_expect_success 'unpack big object in stream' '
+	prepare_dest 1m &&
+	git -C dest.git unpack-objects <test-$PACK.pack &&
+	test_dir_is_empty dest.git/objects/pack
+'
+
+test_expect_success 'do not unpack existing large objects' '
+	prepare_dest 1m &&
+	git -C dest.git index-pack --stdin <test-$PACK.pack &&
+	git -C dest.git unpack-objects <test-$PACK.pack &&
+	test_no_loose
+'
+
 test_done
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
#61 lore raw

v11 cover unpack-objects: support streaming blobs to disk

In reply to Ævar Arnfjörð Bjarmason

This series by Han Xin was waiting on some in-flight patches that landed in 430883a70c7 (Merge branch 'ab/object-file-api-updates', 2022-03-16).

This series teaches "git unpack-objects" to stream objects larger than core.bigFileThreshold to disk. As 8/8 shows streaming e.g. a 100MB blob now uses ~5MB of memory instead of ~105MB. This streaming method is slower if you've got memory to handle the blobs in-core, but if you don't it allows you to unpack objects at all, as you might otherwise OOM.

Changes since v10:

* Renamed the new test file, its number conflicted with a since-landed commit-graph test.

* Some minor code changes to make diffs to the pre-image smaller (e.g. the top of the range-diff below)

* The whole "find dest.git" to see if we have loose objects is now either a test for "do we have objects at all?" (--dry-run mode), or uses a simpler implementation. We could use "test_stdout_line_count" for that.

* We also test that as we use "unpack-objects" to stream directly to a pack that the result is byte-for-byte the same as the source.

* A new 4/8 that I added allows for more code sharing in object-file.c, our two end-state functions now share more logic.

* Minor typo/grammar/comment etc. fixes throughout.

* Updated 8/8 with benchmarks, somewhere along the line we lost the code to run the benchmark mentioned in the commit message...

1. https://lore.kernel.org/git/[email protected]/

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects

Ævar Arnfjörð Bjarmason (4):
object-file.c: do fsync() and close() before post-write die() object-file.c: factor out deflate part of write_loose_object() core doc: modernize core.bigFileThreshold documentation
unpack-objects: refactor away unpack_non_delta_entry()

 Documentation/config/core.txt   |  33 +++--
 builtin/unpack-objects.c        | 109 +++++++++++---
 object-file.c                   | 250 +++++++++++++++++++++++++++-----
 object-store.h                  |   8 +
 t/t5351-unpack-large-objects.sh |  61 ++++++++
 5 files changed, 397 insertions(+), 64 deletions(-)
 create mode 100755 t/t5351-unpack-large-objects.sh

Range-diff against v10:
1: e46eb75b98f ! 1: 2103d5bfd96 unpack-objects: low memory footprint for get_data() in dry_run mode

    @@ builtin/unpack-objects.c: static void use(int bytes)
      {
      	git_zstream stream;
     -	void *buf = xmallocz(size);
    -+	unsigned long bufsize;
    -+	void *buf;
    ++	unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
    ++	void *buf = xmallocz(bufsize);
      
      	memset(&stream, 0, sizeof(stream));
    -+	if (dry_run && size > 8192)
    -+		bufsize = 8192;
    -+	else
    -+		bufsize = size;
    -+	buf = xmallocz(bufsize);
      
      	stream.next_out = buf;
     -	stream.avail_out = size;
    @@ builtin/unpack-objects.c: static void unpack_delta_entry(enum object_type type,
      		hi = nr;
      		while (lo < hi) {
     
    - ## t/t5328-unpack-large-objects.sh (new) ##
    + ## t/t5351-unpack-large-objects.sh (new) ##
     @@
     +#!/bin/sh
     +#
    @@ t/t5328-unpack-large-objects.sh (new)
     +	git init --bare dest.git
     +}
     +
    -+test_no_loose () {
    -+	test $(find dest.git/objects/?? -type f | wc -l) = 0
    -+}
    -+
     +test_expect_success "create large objects (1.5 MB) and PACK" '
     +	test-tool genrandom foo 1500000 >big-blob &&
     +	test_commit --append foo big-blob &&
     +	test-tool genrandom bar 1500000 >big-blob &&
     +	test_commit --append bar big-blob &&
    -+	PACK=$(echo HEAD | git pack-objects --revs test)
    ++	PACK=$(echo HEAD | git pack-objects --revs pack)
     +'
     +
     +test_expect_success 'set memory limitation to 1MB' '
    @@ t/t5328-unpack-large-objects.sh (new)
     +
     +test_expect_success 'unpack-objects failed under memory limitation' '
     +	prepare_dest &&
    -+	test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
    ++	test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err &&
     +	grep "fatal: attempting to allocate" err
     +'
     +
     +test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
     +	prepare_dest &&
    -+	git -C dest.git unpack-objects -n <test-$PACK.pack &&
    -+	test_no_loose &&
    ++	git -C dest.git unpack-objects -n <pack-$PACK.pack &&
    ++	test_stdout_line_count = 0 find dest.git/objects -type f &&
     +	test_dir_is_empty dest.git/objects/pack
     +'
     +
2:  48bf9090058 = 2:  6acd8759772 object-file.c: do fsync() and close() before post-write die()
3:  0e33d2a6e35 = 3:  f7b02c307fc object-file.c: refactor write_loose_object() to several steps
-:  ----------- > 4:  20d97cc2605 object-file.c: factor out deflate part of write_loose_object()
4:  9644df5c744 ! 5:  db40f4160c4 object-file.c: add "stream_loose_object()" to handle large object
    @@ Commit message
         Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
         Helped-by: Jiang Xin <[email protected]>
         Signed-off-by: Han Xin <[email protected]>
    +    Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
     
      ## object-file.c ##
     @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +	strbuf_addf(&filename, "%s/", get_object_directory());
     +	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
     +
    -+	/* Common steps for write_loose_object and stream_loose_object to
    -+	 * start writing loose oject:
    ++	/*
    ++	 * Common steps for write_loose_object and stream_loose_object to
    ++	 * start writing loose objects:
     +	 *
     +	 *  - Create tmpfile for the loose object.
     +	 *  - Setup zlib stream for compression.
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +	/* Then the data itself.. */
     +	do {
     +		unsigned char *in0 = stream.next_in;
    ++
     +		if (!stream.avail_in && !in_stream->is_finished) {
     +			const void *in = in_stream->read(in_stream, &stream.avail_in);
     +			stream.next_in = (void *)in;
     +			in0 = (unsigned char *)in;
     +			/* All data has been read. */
     +			if (in_stream->is_finished)
    -+				flush = Z_FINISH;
    ++				flush = 1;
     +		}
    -+		ret = git_deflate(&stream, flush);
    -+		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
    -+		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
    -+			die(_("unable to write loose object file"));
    -+		stream.next_out = compressed;
    -+		stream.avail_out = sizeof(compressed);
    ++		ret = write_loose_object_common(&c, &stream, flush, in0, fd,
    ++						compressed, sizeof(compressed));
     +		/*
     +		 * Unlike write_loose_object(), we do not have the entire
     +		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
5:  4550f3a2745 = 6:  d8ae2eadb98 core doc: modernize core.bigFileThreshold documentation
-:  ----------- > 7:  2b403e7cd9c unpack-objects: refactor away unpack_non_delta_entry()
6:  6a70e49a346 ! 8:  5eded902496 unpack-objects: use stream_loose_object() to unpack large objects

@@ Commit message
malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.

- We could use this new interface to unpack all blobs, but doing so
- would result in a performance penalty of around 10%, as the below
- "hyperfine" benchmark will show. We therefore limit this to files
- larger than "core.bigFileThreshold":
-
- $ hyperfine \
- --setup \
- 'if ! test -d scalar.git; then git clone --bare
- https://github.com/microsoft/scalar.git;
- cp scalar.git/objects/pack/*.pack small.pack; fi' \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- ...
-
- Summary
- './git -C dest.git -c core.bigFileThreshold=512m
- unpack-objects <small.pack' in 'origin/master'
- 1.01 ± 0.04 times faster than './git -C dest.git
- -c core.bigFileThreshold=512m unpack-objects
- <small.pack' in 'HEAD~1'
- 1.01 ± 0.04 times faster than './git -C dest.git
- -c core.bigFileThreshold=512m unpack-objects
- <small.pack' in 'HEAD~0'
- 1.03 ± 0.10 times faster than './git -C dest.git
- -c core.bigFileThreshold=16k unpack-objects
- <small.pack' in 'origin/master'
- 1.02 ± 0.07 times faster than './git -C dest.git
- -c core.bigFileThreshold=16k unpack-objects
- <small.pack' in 'HEAD~0'
- 1.10 ± 0.04 times faster than './git -C dest.git
- -c core.bigFileThreshold=16k unpack-objects
- <small.pack' in 'HEAD~1'
+ We could use the new streaming interface to unpack all blobs, but
+ doing so would be much slower, as demonstrated e.g. with this
+ benchmark using git-hyperfine[0]:
+
+ rm -rf /tmp/scalar.git &&
+ git clone --bare https://github.com/Microsoft/scalar.git /tmp/scalar.git &&
+ mv /tmp/scalar.git/objects/pack/*.pack /tmp/scalar.git/my.pack &&
+ git hyperfine \
+ -r 2 --warmup 1 \
+ -L rev origin/master,HEAD -L v "10,512,1k,1m" \
+ -s 'make' \
+ -p 'git init --bare dest.git' \
+ -c 'rm -rf dest.git' \
+ './git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/scalar.git/my.pack'
+
+ Here we'll perform worse with lower core.bigFileThreshold settings
+ with this change in terms of speed, but we're getting lower memory use
+ in return:
+
+ Summary
+ './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' ran
+ 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'origin/master'
+ 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'origin/master'
+ 1.01 ± 0.02 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+ 1.02 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master'
+ 1.09 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+ 1.10 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+ 1.11 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+
+ A better benchmark to demonstrate the benefits of that this one, which
+ creates an artificial repo with a 1, 25, 50, 75 and 100MB blob:
+
+ rm -rf /tmp/repo &&
+ git init /tmp/repo &&
+ (
+ cd /tmp/repo &&
+ for i in 1 25 50 75 100
+ do
+ dd if=/dev/urandom of=blob.$i count=$(($i*1024)) bs=1024
+ done &&
+ git add blob.* &&
+ git commit -mblobs &&
+ git gc &&
+ PACK=$(echo .git/objects/pack/pack-*.pack) &&
+ cp "$PACK" my.pack
+ ) &&
+ git hyperfine \
+ --show-output \
+ -L rev origin/master,HEAD -L v "512,50m,100m" \
+ -s 'make' \
+ -p 'git init --bare dest.git' \
+ -c 'rm -rf dest.git' \
+ '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum'
+
+ Using this test we'll always use >100MB of memory on
+ origin/master (around ~105MB), but max out at e.g. ~55MB if we set
+ core.bigFileThreshold=50m.
+
+ The relevant "Maximum resident set size" lines were manually added
+ below the relevant benchmark:
+
+ '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master' ran
+ Maximum resident set size (kbytes): 107080
+ 1.02 ± 0.78 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
+ Maximum resident set size (kbytes): 106968
+ 1.09 ± 0.79 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
+ Maximum resident set size (kbytes): 107032
+ 1.42 ± 1.07 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
+ Maximum resident set size (kbytes): 107072
+ 1.83 ± 1.02 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
+ Maximum resident set size (kbytes): 55704
+ 2.16 ± 1.19 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
+ Maximum resident set size (kbytes): 4564
+
+ This shows that if you have enough memory this new streaming method is
+ slower the lower you set the streaming threshold, but the benefit is
+ more bounded memory use.

An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing
@@ Commit message
split up "core.bigFileThreshold" in the future if there's a need for that.

+ 0. https://github.com/avar/git-hyperfine/
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/

@@ Commit message

+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>

## Documentation/config/core.txt ##
@@ Documentation/config/core.txt: usage, at the slight expense of increased disk usage.
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ return data->buf;
+}
+
-+static void write_stream_blob(unsigned nr, size_t size)
++static void stream_blob(unsigned long size, unsigned nr)
+{
+ git_zstream zstream = { 0 };
+ struct input_zstream_data data = { 0 };
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ .read = feed_input_zstream,
+ .data = &data,
+ };
++ struct obj_info *info = &obj_list[nr];
+
+ data.zstream = &zstream;
+ git_inflate_init(&zstream);
+
-+ if (stream_loose_object(&in_stream, size, &obj_list[nr].oid))
++ if (stream_loose_object(&in_stream, size, &info->oid))
+ die(_("failed to write object in stream"));
+
+ if (data.status != Z_STREAM_END)
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ git_inflate_end(&zstream);
+
+ if (strict) {
-+ struct blob *blob =
-+ lookup_blob(the_repository, &obj_list[nr].oid);
-+ if (blob)
-+ blob->object.flags |= FLAG_WRITTEN;
-+ else
++ struct blob *blob = lookup_blob(the_repository, &info->oid);
++
++ if (!blob)
+ die(_("invalid blob object from stream"));
++ blob->object.flags |= FLAG_WRITTEN;
+ }
-+ obj_list[nr].obj = NULL;
++ info->obj = NULL;
+}
+
- static void unpack_non_delta_entry(enum object_type type, unsigned long size,
- unsigned nr)
+ static int resolve_against_held(unsigned nr, const struct object_id *base,
+ void *delta_data, unsigned long delta_size)
{
-- void *buf = get_data(size);
-+ void *buf;
-+
-+ /* Write large blob in stream without allocating full buffer. */
-+ if (!dry_run && type == OBJ_BLOB && size > big_file_threshold) {
-+ write_stream_blob(nr, size);
-+ return;
-+ }
+@@ builtin/unpack-objects.c: static void unpack_one(unsigned nr)

-+ buf = get_data(size);
- if (buf)
- write_object(nr, type, buf, size);
- }
+ switch (type) {
+ case OBJ_BLOB:
++ if (!dry_run && size > big_file_threshold) {
++ stream_blob(size, nr);
++ return;
++ }
++ /* fallthrough */
+ case OBJ_COMMIT:
+ case OBJ_TREE:
+ case OBJ_TAG:

- ## t/t5328-unpack-large-objects.sh ##
-@@ t/t5328-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
+ ## t/t5351-unpack-large-objects.sh ##
+@@ t/t5351-unpack-large-objects.sh: test_description='git unpack-objects with large objects'

prepare_dest () {
test_when_finished "rm -rf dest.git" &&
- git init --bare dest.git
+ git init --bare dest.git &&
-+ if test -n "$1"
-+ then
-+ git -C dest.git config core.bigFileThreshold $1
-+ fi
++ git -C dest.git config core.bigFileThreshold "$1"
}

- test_no_loose () {
-@@ t/t5328-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
+ test_expect_success "create large objects (1.5 MB) and PACK" '
+@@ t/t5351-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
'

test_expect_success 'unpack-objects failed under memory limitation' '
- prepare_dest &&
+ prepare_dest 2m &&
- test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
+ test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err &&
grep "fatal: attempting to allocate" err
'

test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
- prepare_dest &&
+ prepare_dest 2m &&
- git -C dest.git unpack-objects -n <test-$PACK.pack &&
- test_no_loose &&
+ git -C dest.git unpack-objects -n <pack-$PACK.pack &&
+ test_stdout_line_count = 0 find dest.git/objects -type f &&
test_dir_is_empty dest.git/objects/pack
'

+test_expect_success 'unpack big object in stream' '
+ prepare_dest 1m &&
-+ git -C dest.git unpack-objects <test-$PACK.pack &&
++ git -C dest.git unpack-objects <pack-$PACK.pack &&
+ test_dir_is_empty dest.git/objects/pack
+'
+
+test_expect_success 'do not unpack existing large objects' '
+ prepare_dest 1m &&
-+ git -C dest.git index-pack --stdin <test-$PACK.pack &&
-+ git -C dest.git unpack-objects <test-$PACK.pack &&
-+ test_no_loose
++ git -C dest.git index-pack --stdin <pack-$PACK.pack &&
++ git -C dest.git unpack-objects <pack-$PACK.pack &&
++
++ # The destination came up with the exact same pack...
++ DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) &&
++ test_cmp pack-$PACK.pack $DEST_PACK &&
++
++ # ...and wrote no loose objects
++ test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*"
+'
+
test_done

Signature · 1 line
2.35.1.1438.g8874c8eeb35
#62 lore raw

PATCH v11 1/8 unpack-objects: low memory footprint for get_data() in dry_run mode

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).

---
 builtin/unpack-objects.c        | 34 ++++++++++++++++++---------
 t/t5351-unpack-large-objects.sh | 41 +++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100755 t/t5351-unpack-large-objects.sh
builtin/unpack-objects.c +23 -11
index dbeb0680a58..e3d30025979 100644
@@ -96,15 +96,26 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
+/*
+ * Decompress zstream from stdin and return specific size of data.
+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
 static void *get_data(unsigned long size)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
+	void *buf = xmallocz(bufsize);
 
 	memset(&stream, 0, sizeof(stream));
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -124,8 +135,15 @@ static void *get_data(unsigned long size)
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
+	if (dry_run)
+		FREE_AND_NULL(buf);
 	return buf;
 }
 
@@ -325,10 +343,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 {
 	void *buf = get_data(size);
 
-	if (!dry_run && buf)
+	if (buf)
 		write_object(nr, type, buf, size);
-	else
-		free(buf);
 }
 
 static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +374,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		oidread(&base_oid, fill(the_hash_algo->rawsz));
 		use(the_hash_algo->rawsz);
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		if (has_object_file(&base_oid))
 			; /* Ok we have this one */
 		else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +411,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 			die("offset value out of bound for delta base object");
 
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		lo = 0;
 		hi = nr;
 		while (lo < hi) {
t/t5351-unpack-large-objects.sh new file +41 -0

41 lines not on this page · raw message

Signature · 1 line
2.35.1.1438.g8874c8eeb35
#63 lore raw

PATCH v11 2/8 object-file.c: do fsync() and close() before post-write die()

In reply to Ævar Arnfjörð Bjarmason

Change write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).

I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.

The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.

---
 object-file.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
object-file.c +11 -2
index bdc5cbdd386..4c140eda6bf 100644
@@ -2001,12 +2001,21 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
 		    ret);
 	the_hash_algo->final_oid_fn(&parano_oid, &c);
+
+	/*
+	 * We already did a write_buffer() to the "fd", let's fsync()
+	 * and close().
+	 *
+	 * We might still die() on a subsequent sanity check, but
+	 * let's not add to that confusion by not flushing any
+	 * outstanding writes to disk first.
+	 */
+	close_loose_object(fd);
+
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
-	close_loose_object(fd);
-
 	if (mtime) {
 		struct utimbuf utb;
 		utb.actime = mtime;
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#64 lore raw

PATCH v11 3/8 object-file.c: refactor write_loose_object() to several steps

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.

Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".

2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".

3. Compress data.

4. Move common steps for ending zlib stream into a new function

    "end_loose_object_common()".

5. Close fd and finalize the object file.

---
 object-file.c | 129 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 89 insertions(+), 40 deletions(-)
object-file.c +89 -40
index 4c140eda6bf..4fcaf7a36ce 100644
@@ -1943,6 +1943,87 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+/**
+ * Common steps for loose object writers to start writing loose
+ * objects:
+ *
+ * - Create tmpfile for the loose object.
+ * - Setup zlib stream for compression.
+ * - Start to feed header to zlib stream.
+ *
+ * Returns a "fd", which should later be provided to
+ * end_loose_object_common().
+ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+				     const char *filename, unsigned flags,
+				     git_zstream *stream,
+				     unsigned char *buf, size_t buflen,
+				     git_hash_ctx *c,
+				     char *hdr, int hdrlen)
+{
+	int fd;
+
+	fd = create_tmpfile(tmp_file, filename);
+	if (fd < 0) {
+		if (flags & HASH_SILENT)
+			return -1;
+		else if (errno == EACCES)
+			return error(_("insufficient permission for adding "
+				       "an object to repository database %s"),
+				     get_object_directory());
+		else
+			return error_errno(
+				_("unable to create temporary file"));
+	}
+
+	/*  Setup zlib stream for compression */
+	git_deflate_init(stream, zlib_compression_level);
+	stream->next_out = buf;
+	stream->avail_out = buflen;
+	the_hash_algo->init_fn(c);
+
+	/*  Start to feed header to zlib stream */
+	stream->next_in = (unsigned char *)hdr;
+	stream->avail_in = hdrlen;
+	while (git_deflate(stream, 0) == Z_OK)
+		; /* nothing */
+	the_hash_algo->update_fn(c, hdr, hdrlen);
+
+	return fd;
+}
+
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "parano_oid".
+ * - fsync() and close() the "fd"
+ */
+static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+				    git_zstream *stream,
+				    struct object_id *parano_oid,
+				    const struct object_id *expected_oid,
+				    const char *die_msg1_fmt,
+				    const char *die_msg2_fmt)
+{
+	if (ret != Z_STREAM_END)
+		die(_(die_msg1_fmt), ret, expected_oid);
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		die(_(die_msg2_fmt), ret, expected_oid);
+	the_hash_algo->final_oid_fn(parano_oid, c);
+
+	/*
+	 * We already did a write_buffer() to the "fd", let's fsync()
+	 * and close().
+	 *
+	 * We might still die() on a subsequent sanity check, but
+	 * let's not add to that confusion by not flushing any
+	 * outstanding writes to disk first.
+	 */
+	close_loose_object(fd);
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1957,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
-	}
-
-	/* Set it up */
-	git_deflate_init(&stream, zlib_compression_level);
-	stream.next_out = compressed;
-	stream.avail_out = sizeof(compressed);
-	the_hash_algo->init_fn(&c);
-
-	/* First header.. */
-	stream.next_in = (unsigned char *)hdr;
-	stream.avail_in = hdrlen;
-	while (git_deflate(&stream, 0) == Z_OK)
-		; /* nothing */
-	the_hash_algo->update_fn(&c, hdr, hdrlen);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0)
+		return -1;
 
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
@@ -1993,24 +2057,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		stream.avail_out = sizeof(compressed);
 	} while (ret == Z_OK);
 
-	if (ret != Z_STREAM_END)
-		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
-		    ret);
-	ret = git_deflate_end_gently(&stream);
-	if (ret != Z_OK)
-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
-		    ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
-
-	/*
-	 * We already did a write_buffer() to the "fd", let's fsync()
-	 * and close().
-	 *
-	 * We might still die() on a subsequent sanity check, but
-	 * let's not add to that confusion by not flushing any
-	 * outstanding writes to disk first.
-	 */
-	close_loose_object(fd);
+	end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
+				N_("unable to deflate new object %s (%d)"),
+				N_("deflateEnd on object %s failed (%d)"));
 
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#65 lore raw

Re: [PATCH v11 3/8] object-file.c: refactor write_loose_object() to several steps

Am 19.03.22 um 01:23 schrieb Ævar Arnfjörð Bjarmason:

Quoted 100 lines

From: Han Xin <[email protected]>

When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.

Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".

2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".

3. Compress data.

4. Move common steps for ending zlib stream into a new function

    "end_loose_object_common()".

5. Close fd and finalize the object file.

---
 object-file.c | 129 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 89 insertions(+), 40 deletions(-)
object-file.c +58 -0
index 4c140eda6bf..4fcaf7a36ce 100644
@@ -1943,6 +1943,87 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+/**
+ * Common steps for loose object writers to start writing loose
+ * objects:
+ *
+ * - Create tmpfile for the loose object.
+ * - Setup zlib stream for compression.
+ * - Start to feed header to zlib stream.
+ *
+ * Returns a "fd", which should later be provided to
+ * end_loose_object_common().
+ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+				     const char *filename, unsigned flags,
+				     git_zstream *stream,
+				     unsigned char *buf, size_t buflen,
+				     git_hash_ctx *c,
+				     char *hdr, int hdrlen)
+{
+	int fd;
+
+	fd = create_tmpfile(tmp_file, filename);
+	if (fd < 0) {
+		if (flags & HASH_SILENT)
+			return -1;
+		else if (errno == EACCES)
+			return error(_("insufficient permission for adding "
+				       "an object to repository database %s"),
+				     get_object_directory());
+		else
+			return error_errno(
+				_("unable to create temporary file"));
+	}
+
+	/*  Setup zlib stream for compression */
+	git_deflate_init(stream, zlib_compression_level);
+	stream->next_out = buf;
+	stream->avail_out = buflen;
+	the_hash_algo->init_fn(c);
+
+	/*  Start to feed header to zlib stream */
+	stream->next_in = (unsigned char *)hdr;
+	stream->avail_in = hdrlen;
+	while (git_deflate(stream, 0) == Z_OK)
+		; /* nothing */
+	the_hash_algo->update_fn(c, hdr, hdrlen);
+
+	return fd;
+}
+
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "parano_oid".
+ * - fsync() and close() the "fd"
+ */
+static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+				    git_zstream *stream,

object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 2013

Quoted 10 lines
object-file.c +10 -0
+				    struct object_id *parano_oid,
+				    const struct object_id *expected_oid,
+				    const char *die_msg1_fmt,
+				    const char *die_msg2_fmt)
+{
+	if (ret != Z_STREAM_END)
+		die(_(die_msg1_fmt), ret, expected_oid);
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		die(_(die_msg2_fmt), ret, expected_oid);

stream_loose_object(), added in patch 5, passes NULL as expected_oid, but these die() messages need a valid value. end_loose_object_common() has more parameters than lines of code in its body. Inline it to allow fully custom messages?

Quoted 78 lines
object-file.c +21 -40
+	the_hash_algo->final_oid_fn(parano_oid, c);
+
+	/*
+	 * We already did a write_buffer() to the "fd", let's fsync()
+	 * and close().
+	 *
+	 * We might still die() on a subsequent sanity check, but
+	 * let's not add to that confusion by not flushing any
+	 * outstanding writes to disk first.
+	 */
+	close_loose_object(fd);
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1957,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
-	}
-
-	/* Set it up */
-	git_deflate_init(&stream, zlib_compression_level);
-	stream.next_out = compressed;
-	stream.avail_out = sizeof(compressed);
-	the_hash_algo->init_fn(&c);
-
-	/* First header.. */
-	stream.next_in = (unsigned char *)hdr;
-	stream.avail_in = hdrlen;
-	while (git_deflate(&stream, 0) == Z_OK)
-		; /* nothing */
-	the_hash_algo->update_fn(&c, hdr, hdrlen);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0)
+		return -1;
 
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
@@ -1993,24 +2057,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		stream.avail_out = sizeof(compressed);
 	} while (ret == Z_OK);
 
-	if (ret != Z_STREAM_END)
-		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
-		    ret);
-	ret = git_deflate_end_gently(&stream);
-	if (ret != Z_OK)
-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
-		    ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
-
-	/*
-	 * We already did a write_buffer() to the "fd", let's fsync()
-	 * and close().
-	 *
-	 * We might still die() on a subsequent sanity check, but
-	 * let's not add to that confusion by not flushing any
-	 * outstanding writes to disk first.
-	 */
-	close_loose_object(fd);
+	end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
+				N_("unable to deflate new object %s (%d)"),
+				N_("deflateEnd on object %s failed (%d)"));
 
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
#66 lore raw

PATCH v11 4/8 object-file.c: factor out deflate part of write_loose_object()

In reply to Ævar Arnfjörð Bjarmason

Split out the part of write_loose_object() that deals with calling git_deflate() into a utility function, a subsequent commit will introduce another function that'll make use of it.

---
 object-file.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)
object-file.c +25 -6
index 4fcaf7a36ce..b66dc24e4b8 100644
@@ -1992,6 +1992,28 @@ static int start_loose_object_common(struct strbuf *tmp_file,
 	return fd;
 }
 
+/**
+ * Common steps for the inner git_deflate() loop for writing loose
+ * objects. Returns what git_deflate() returns.
+ */
+static int write_loose_object_common(git_hash_ctx *c,
+				     git_zstream *stream, const int flush,
+				     unsigned char *in0, const int fd,
+				     unsigned char *compressed,
+				     const size_t compressed_len)
+{
+	int ret;
+
+	ret = git_deflate(stream, flush ? Z_FINISH : 0);
+	the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+	if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+		die(_("unable to write loose object file"));
+	stream->next_out = compressed;
+	stream->avail_out = compressed_len;
+
+	return ret;
+}
+
 /**
  * Common steps for loose object writers to end writing loose objects:
  *
@@ -2049,12 +2071,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	stream.avail_in = len;
 	do {
 		unsigned char *in0 = stream.next_in;
-		ret = git_deflate(&stream, Z_FINISH);
-		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-			die(_("unable to write loose object file"));
-		stream.next_out = compressed;
-		stream.avail_out = sizeof(compressed);
+
+		ret = write_loose_object_common(&c, &stream, 1, in0, fd,
+						compressed, sizeof(compressed));
 	} while (ret == Z_OK);
 
 	end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#67 lore raw

PATCH v11 5/8 object-file.c: add "stream_loose_object()" to handle large object

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".

---
 object-file.c  | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++
 object-store.h |  8 +++++
 2 files changed, 105 insertions(+)
object-file.c +97 -0
index b66dc24e4b8..548fef71b98 100644
@@ -2114,6 +2114,103 @@ static int freshen_packed_object(const struct object_id *oid)
 	return 1;
 }
 
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid)
+{
+	int fd, ret, err = 0, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen;
+
+	/* Since oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+	/*
+	 * Common steps for write_loose_object and stream_loose_object to
+	 * start writing loose objects:
+	 *
+	 *  - Create tmpfile for the loose object.
+	 *  - Setup zlib stream for compression.
+	 *  - Start to feed header to zlib stream.
+	 */
+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0) {
+		err = -1;
+		goto cleanup;
+	}
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+
+		if (!stream.avail_in && !in_stream->is_finished) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (in_stream->is_finished)
+				flush = 1;
+		}
+		ret = write_loose_object_common(&c, &stream, flush, in0, fd,
+						compressed, sizeof(compressed));
+		/*
+		 * Unlike write_loose_object(), we do not have the entire
+		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
+		 * then we'll replenish them in the next input_stream->read()
+		 * call when we loop.
+		 */
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (stream.total_in != len + hdrlen)
+		die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
+		    (uintmax_t)len + hdrlen);
+
+	/* Common steps for write_loose_object and stream_loose_object to
+	 * end writing loose oject:
+	 *
+	 *  - End the compression of zlib stream.
+	 *  - Get the calculated oid.
+	 */
+	end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
+				N_("unable to stream deflate new object (%d)"),
+				N_("deflateEnd on stream object failed (%d)"));
+
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		goto cleanup;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen);
+
+		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
+			err = error_errno(_("unable to create directory %s"), dir.buf);
+			strbuf_release(&dir);
+			goto cleanup;
+		}
+		strbuf_release(&dir);
+	}
+
+	err = finalize_object_file(tmp_file.buf, filename.buf);
+cleanup:
+	strbuf_release(&tmp_file);
+	strbuf_release(&filename);
+	return err;
+}
+
 int write_object_file_flags(const void *buf, unsigned long len,
 			    enum object_type type, struct object_id *oid,
 			    unsigned flags)
object-store.h +8 -0
index bd2322ed8ce..1099455bc2e 100644
@@ -46,6 +46,12 @@ struct object_directory {
 	char *path;
 };
 
+struct input_stream {
+	const void *(*read)(struct input_stream *, unsigned long *len);
+	void *data;
+	int is_finished;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -261,6 +267,8 @@ static inline int write_object_file(const void *buf, unsigned long len,
 int write_object_file_literally(const void *buf, unsigned long len,
 				const char *type, struct object_id *oid,
 				unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid);
 
 /*
  * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#68 lore raw

PATCH v11 6/8 core doc: modernize core.bigFileThreshold documentation

In reply to Ævar Arnfjörð Bjarmason

The core.bigFileThreshold documentation has been largely unchanged since 5eef828bc03 (fast-import: Stream very large blobs directly to pack, 2010-02-01).

But since then this setting has been expanded to affect a lot more than that description indicated. Most notably in how "git diff" treats them, see 6bf3b813486 (diff --stat: mark any file larger than core.bigfilethreshold binary, 2014-08-16).

In addition to that, numerous commands and APIs make use of a streaming mode for files above this threshold.

So let's attempt to summarize 12 years of changes in behavior, which can be seen with:

    git log --oneline -Gbig_file_thre 5eef828bc03.. -- '*.c'

To do that turn this into a bullet-point list. The summary Han Xin produced in [1] helped a lot, but is a bit too detailed for documentation aimed at users. Let's instead summarize how user-observable behavior differs, and generally describe how we tend to stream these files in various commands.

1. https://lore.kernel.org/git/[email protected]/

---
 Documentation/config/core.txt | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)
Documentation/config/core.txt +24 -9
index c04f62a54a1..b6a12218665 100644
@@ -412,17 +412,32 @@ You probably do not need to adjust this value.
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
 core.bigFileThreshold::
-	Files larger than this size are stored deflated, without
-	attempting delta compression.  Storing large files without
-	delta compression avoids excessive memory usage, at the
-	slight expense of increased disk usage. Additionally files
-	larger than this size are always treated as binary.
+	The size of files considered "big", which as discussed below
+	changes the behavior of numerous git commands, as well as how
+	such files are stored within the repository. The default is
+	512 MiB. Common unit suffixes of 'k', 'm', or 'g' are
+	supported.
 +
-Default is 512 MiB on all platforms.  This should be reasonable
-for most projects as source code and other text files can still
-be delta compressed, but larger binary media files won't be.
+Files above the configured limit will be:
 +
-Common unit suffixes of 'k', 'm', or 'g' are supported.
+* Stored deflated, without attempting delta compression.
++
+The default limit is primarily set with this use-case in mind. With it
+most projects will have their source code and other text files delta
+compressed, but not larger binary media files.
++
+Storing large files without delta compression avoids excessive memory
+usage, at the slight expense of increased disk usage.
++
+* Will be treated as if though they were labeled "binary" (see
+  linkgit:gitattributes[5]). This means that e.g. linkgit:git-log[1]
+  and linkgit:git-diff[1] will not diffs for files above this limit.
++
+* Will be generally be streamed when written, which avoids excessive
+memory usage, at the cost of some fixed overhead. Commands that make
+use of this include linkgit:git-archive[1],
+linkgit:git-fast-import[1], linkgit:git-index-pack[1] and
+linkgit:git-fsck[1].
 
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#69 lore raw

PATCH v11 7/8 unpack-objects: refactor away unpack_non_delta_entry()

In reply to Ævar Arnfjörð Bjarmason

The unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.

Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.

This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about. As it'll be handling "OBJ_BLOB" in a special manner let's re-arrange that "case" in preparation for that change.

---
 builtin/unpack-objects.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
builtin/unpack-objects.c +7 -11
index e3d30025979..d374599d544 100644
@@ -338,15 +338,6 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
-static void unpack_non_delta_entry(enum object_type type, unsigned long size,
-				   unsigned nr)
-{
-	void *buf = get_data(size);
-
-	if (buf)
-		write_object(nr, type, buf, size);
-}
-
 static int resolve_against_held(unsigned nr, const struct object_id *base,
 				void *delta_data, unsigned long delta_size)
 {
@@ -479,12 +470,17 @@ static void unpack_one(unsigned nr)
 	}
 
 	switch (type) {
+	case OBJ_BLOB:
 	case OBJ_COMMIT:
 	case OBJ_TREE:
-	case OBJ_BLOB:
 	case OBJ_TAG:
-		unpack_non_delta_entry(type, size, nr);
+	{
+		void *buf = get_data(size);
+
+		if (buf)
+			write_object(nr, type, buf, size);
 		return;
+	}
 	case OBJ_REF_DELTA:
 	case OBJ_OFS_DELTA:
 		unpack_delta_entry(type, size, nr);
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#70 lore raw

PATCH v11 8/8 unpack-objects: use stream_loose_object() to unpack large objects

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

Make use of the stream_loose_object() function introduced in the preceding commit to unpack large objects. Before this we'd need to malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.

We could use the new streaming interface to unpack all blobs, but doing so would be much slower, as demonstrated e.g. with this benchmark using git-hyperfine[0]:

	rm -rf /tmp/scalar.git &&
	git clone --bare https://github.com/Microsoft/scalar.git /tmp/scalar.git &&
	mv /tmp/scalar.git/objects/pack/*.pack /tmp/scalar.git/my.pack &&
	git hyperfine \
		-r 2 --warmup 1 \
		-L rev origin/master,HEAD -L v "10,512,1k,1m" \
		-s 'make' \
		-p 'git init --bare dest.git' \
		-c 'rm -rf dest.git' \
		'./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/scalar.git/my.pack'

Here we'll perform worse with lower core.bigFileThreshold settings with this change in terms of speed, but we're getting lower memory use in return:

Summary
'./git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' ran
1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.02 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.02 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.09 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.10 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.11 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'

A better benchmark to demonstrate the benefits of that this one, which creates an artificial repo with a 1, 25, 50, 75 and 100MB blob:

	rm -rf /tmp/repo &&
	git init /tmp/repo &&
	(
		cd /tmp/repo &&
		for i in 1 25 50 75 100
		do
			dd if=/dev/urandom of=blob.$i count=$(($i*1024)) bs=1024
		done &&
		git add blob.* &&
		git commit -mblobs &&
		git gc &&
		PACK=$(echo .git/objects/pack/pack-*.pack) &&
		cp "$PACK" my.pack
	) &&
	git hyperfine \
		--show-output \
		-L rev origin/master,HEAD -L v "512,50m,100m" \
		-s 'make' \
		-p 'git init --bare dest.git' \
		-c 'rm -rf dest.git' \
		'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum'

Using this test we'll always use >100MB of memory on origin/master (around ~105MB), but max out at e.g. ~55MB if we set core.bigFileThreshold=50m.

The relevant "Maximum resident set size" lines were manually added below the relevant benchmark:

'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master' ran
Maximum resident set size (kbytes): 107080
1.02 ± 0.78 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 106968
1.09 ± 0.79 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 107032
1.42 ± 1.07 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 107072
1.83 ± 1.02 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 55704
2.16 ± 1.19 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 4564

This shows that if you have enough memory this new streaming method is slower the lower you set the streaming threshold, but the benefit is more bounded memory use.

An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing "core.bigFileThreshold" variable[1]. As noted in a detailed overview of its users in [2] using it has several different meanings.

Still, we consider it good enough to simply re-use it. While it's possible that someone might want to e.g. consider objects "small" for the purposes of diffing but "big" for the purposes of writing them such use-cases are probably too obscure to worry about. We can always split up "core.bigFileThreshold" in the future if there's a need for that.

0. https://github.com/avar/git-hyperfine/
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/

---
 Documentation/config/core.txt   |  4 +-
 builtin/unpack-objects.c        | 67 +++++++++++++++++++++++++++++++++
 t/t5351-unpack-large-objects.sh | 26 +++++++++++--
 3 files changed, 92 insertions(+), 5 deletions(-)
Documentation/config/core.txt +2 -2
index b6a12218665..5aca987632c 100644
@@ -436,8 +436,8 @@ usage, at the slight expense of increased disk usage.
 * Will be generally be streamed when written, which avoids excessive
 memory usage, at the cost of some fixed overhead. Commands that make
 use of this include linkgit:git-archive[1],
-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and
-linkgit:git-fsck[1].
+linkgit:git-fast-import[1], linkgit:git-index-pack[1],
+linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
 
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
builtin/unpack-objects.c +67 -0

79 lines not on this page · raw message

t/t5351-unpack-large-objects.sh +23 -3
index 8d84313221c..461ca060b2b 100755
@@ -9,7 +9,8 @@ test_description='git unpack-objects with large objects'
 
 prepare_dest () {
 	test_when_finished "rm -rf dest.git" &&
-	git init --bare dest.git
+	git init --bare dest.git &&
+	git -C dest.git config core.bigFileThreshold "$1"
 }
 
 test_expect_success "create large objects (1.5 MB) and PACK" '
@@ -26,16 +27,35 @@ test_expect_success 'set memory limitation to 1MB' '
 '
 
 test_expect_success 'unpack-objects failed under memory limitation' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err &&
 	grep "fatal: attempting to allocate" err
 '
 
 test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	git -C dest.git unpack-objects -n <pack-$PACK.pack &&
 	test_stdout_line_count = 0 find dest.git/objects -type f &&
 	test_dir_is_empty dest.git/objects/pack
 '
 
+test_expect_success 'unpack big object in stream' '
+	prepare_dest 1m &&
+	git -C dest.git unpack-objects <pack-$PACK.pack &&
+	test_dir_is_empty dest.git/objects/pack
+'
+
+test_expect_success 'do not unpack existing large objects' '
+	prepare_dest 1m &&
+	git -C dest.git index-pack --stdin <pack-$PACK.pack &&
+	git -C dest.git unpack-objects <pack-$PACK.pack &&
+
+	# The destination came up with the exact same pack...
+	DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) &&
+	test_cmp pack-$PACK.pack $DEST_PACK &&
+
+	# ...and wrote no loose objects
+	test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*"
+'
+
 test_done
Signature · 1 line
2.35.1.1438.g8874c8eeb35
#71 lore raw

v12 cover unpack-objects: support streaming blobs to disk

In reply to Ævar Arnfjörð Bjarmason

This series by Han Xin was waiting on some in-flight patches that landed in 430883a70c7 (Merge branch 'ab/object-file-api-updates', 2022-03-16).

This series teaches "git unpack-objects" to stream objects larger than core.bigFileThreshold to disk. As 8/8 shows streaming e.g. a 100MB blob now uses ~5MB of memory instead of ~105MB. This streaming method is slower if you've got memory to handle the blobs in-core, but if you don't it allows you to unpack objects at all, as you might otherwise OOM.

Changes since v10[1]:

* René rightly spotted that the end_loose_object_common() function was feeding NULL to a format. That's now fixed, and parts of that function were pulled out into the two callers to make the trade-off of factoring that logic out worth it.

* This topic conflicts with ns/batch-fsync in "seen" (see below). I moved an inline comment on close_loose_object() around to make the conflict easier (and it's better placed with the function anyway, as we'll get two callers of it).

Conflict this is the --remerge-diff with "seen" after resolving the conflict. Botht textual and semantic (there's a new caller in this topic) conflicts are caught by the compiler:

	diff --git a/object-file.c b/object-file.c
	remerge CONFLICT (content): Merge conflict in object-file.c
	index 9c640f1f39d..6068f8ec6c4 100644
	--- a/object-file.c
	+++ b/object-file.c
	@@ -1887,7 +1887,6 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
	 	hash_object_file_literally(algo, buf, len, type_name(type), oid);
	 }

	-<<<<<<< 34ee6a28a54 (unpack-objects: use stream_loose_object() to unpack large objects)
	 /*
	  * We already did a write_buffer() to the "fd", let's fsync()
	  * and close().
	@@ -1896,11 +1895,7 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
	  * subsequent sanity check, but let's not add to that confusion by not
	  * flushing any outstanding writes to disk first.
	  */
	-static void close_loose_object(int fd)
	-=======
	-/* Finalize a file on disk, and close it. */
	 static void close_loose_object(int fd, const char *filename)
	->>>>>>> b1423c89b5a (Merge branch 'ab/reftable-aix-xlc-12' into seen)
	 {
	 	if (the_repository->objects->odb->will_destroy)
	 		goto out;
	@@ -2093,17 +2088,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
	 	ret = end_loose_object_common(&c, &stream, &parano_oid);
	 	if (ret != Z_OK)
	 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
	-	close_loose_object(fd);
	+	close_loose_object(fd, tmp_file.buf);

	 	if (!oideq(oid, &parano_oid))
	 		die(_("confused by unstable object source data for %s"),
	 		    oid_to_hex(oid));

	-<<<<<<< 34ee6a28a54 (unpack-objects: use stream_loose_object() to unpack large objects)
	-=======
	-	close_loose_object(fd, tmp_file.buf);
	-
	->>>>>>> b1423c89b5a (Merge branch 'ab/reftable-aix-xlc-12' into seen)
	 	if (mtime) {
	 		struct utimbuf utb;
	 		utb.actime = mtime;
	@@ -2206,7 +2196,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,
	 	ret = end_loose_object_common(&c, &stream, oid);
	 	if (ret != Z_OK)
	 		die(_("deflateEnd on stream object failed (%d)"), ret);
	-	close_loose_object(fd);
	+	close_loose_object(fd, tmp_file.buf);

	 	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
	 		unlink_or_warn(tmp_file.buf);

1. https://lore.kernel.org/git/[email protected]/

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects

Ævar Arnfjörð Bjarmason (4):
object-file.c: do fsync() and close() before post-write die() object-file.c: factor out deflate part of write_loose_object() core doc: modernize core.bigFileThreshold documentation
unpack-objects: refactor away unpack_non_delta_entry()

 Documentation/config/core.txt   |  33 +++--
 builtin/unpack-objects.c        | 109 +++++++++++---
 object-file.c                   | 246 +++++++++++++++++++++++++++-----
 object-store.h                  |   8 ++
 t/t5351-unpack-large-objects.sh |  61 ++++++++
 5 files changed, 396 insertions(+), 61 deletions(-)
 create mode 100755 t/t5351-unpack-large-objects.sh

Range-diff against v11:

1:  2103d5bfd96 = 1:  e95f6a1cfb6 unpack-objects: low memory footprint for get_data() in dry_run mode
2:  6acd8759772 ! 2:  54060eb8c6b object-file.c: do fsync() and close() before post-write die()

@@ Commit message

## object-file.c ##
+@@ object-file.c: void hash_object_file(const struct git_hash_algo *algo, const void *buf,
+ hash_object_file_literally(algo, buf, len, type_name(type), oid);
+ }
+
+-/* Finalize a file on disk, and close it. */
++/*
++ * We already did a write_buffer() to the "fd", let's fsync()
++ * and close().
++ *
++ * Finalize a file on disk, and close it. We might still die() on a
++ * subsequent sanity check, but let's not add to that confusion by not
++ * flushing any outstanding writes to disk first.
++ */
+ static void close_loose_object(int fd)
+ {
+ if (the_repository->objects->odb->will_destroy)
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
ret);
the_hash_algo->final_oid_fn(&parano_oid, &c);
-+
-+ /*
-+ * We already did a write_buffer() to the "fd", let's fsync()
-+ * and close().
-+ *
-+ * We might still die() on a subsequent sanity check, but
-+ * let's not add to that confusion by not flushing any
-+ * outstanding writes to disk first.
-+ */
+ close_loose_object(fd);
+
if (!oideq(oid, &parano_oid))
3: f7b02c307fc ! 3: 3dcaa5d6589 object-file.c: refactor write_loose_object() to several steps

    @@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
     + * Common steps for loose object writers to end writing loose objects:
     + *
     + * - End the compression of zlib stream.
    -+ * - Get the calculated oid to "parano_oid".
    ++ * - Get the calculated oid to "oid".
     + * - fsync() and close() the "fd"
     + */
    -+static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
    -+				    git_zstream *stream,
    -+				    struct object_id *parano_oid,
    -+				    const struct object_id *expected_oid,
    -+				    const char *die_msg1_fmt,
    -+				    const char *die_msg2_fmt)
    ++static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,
    ++				   struct object_id *oid)
     +{
    -+	if (ret != Z_STREAM_END)
    -+		die(_(die_msg1_fmt), ret, expected_oid);
    ++	int ret;
    ++
     +	ret = git_deflate_end_gently(stream);
     +	if (ret != Z_OK)
    -+		die(_(die_msg2_fmt), ret, expected_oid);
    -+	the_hash_algo->final_oid_fn(parano_oid, c);
    ++		return ret;
    ++	the_hash_algo->final_oid_fn(oid, c);
     +
    -+	/*
    -+	 * We already did a write_buffer() to the "fd", let's fsync()
    -+	 * and close().
    -+	 *
    -+	 * We might still die() on a subsequent sanity check, but
    -+	 * let's not add to that confusion by not flushing any
    -+	 * outstanding writes to disk first.
    -+	 */
    -+	close_loose_object(fd);
    ++	return Z_OK;
     +}
     +
      static int write_loose_object(const struct object_id *oid, char *hdr,
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
      	/* Then the data itself.. */
      	stream.next_in = (void *)buf;
     @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
    - 		stream.avail_out = sizeof(compressed);
    - 	} while (ret == Z_OK);
    - 
    --	if (ret != Z_STREAM_END)
    --		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
    --		    ret);
    + 	if (ret != Z_STREAM_END)
    + 		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
    + 		    ret);
     -	ret = git_deflate_end_gently(&stream);
    --	if (ret != Z_OK)
    ++	ret = end_loose_object_common(&c, &stream, &parano_oid);
    + 	if (ret != Z_OK)
     -		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
     -		    ret);
     -	the_hash_algo->final_oid_fn(&parano_oid, &c);
    --
    --	/*
    --	 * We already did a write_buffer() to the "fd", let's fsync()
    --	 * and close().
    --	 *
    --	 * We might still die() on a subsequent sanity check, but
    --	 * let's not add to that confusion by not flushing any
    --	 * outstanding writes to disk first.
    --	 */
    --	close_loose_object(fd);
    -+	end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
    -+				N_("unable to deflate new object %s (%d)"),
    -+				N_("deflateEnd on object %s failed (%d)"));
    ++		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
    + 	close_loose_object(fd);
      
      	if (!oideq(oid, &parano_oid))
    - 		die(_("confused by unstable object source data for %s"),

4: 20d97cc2605 ! 4: 03f4e91ac89 object-file.c: factor out deflate part of write_loose_object()
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
+ compressed, sizeof(compressed));
} while (ret == Z_OK);

- end_loose_object_common(fd, ret, &c, &stream, &parano_oid, oid,
+ if (ret != Z_STREAM_END)
5: db40f4160c4 ! 5: 3d64cf1cf33 object-file.c: add "stream_loose_object()" to handle large object

    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +	 *  - End the compression of zlib stream.
     +	 *  - Get the calculated oid.
     +	 */
    -+	end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
    -+				N_("unable to stream deflate new object (%d)"),
    -+				N_("deflateEnd on stream object failed (%d)"));
    ++	if (ret != Z_STREAM_END)
    ++		die(_("unable to stream deflate new object (%d)"), ret);
    ++	ret = end_loose_object_common(&c, &stream, oid);
    ++	if (ret != Z_OK)
    ++		die(_("deflateEnd on stream object failed (%d)"), ret);
    ++	close_loose_object(fd);
     +
     +	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
     +		unlink_or_warn(tmp_file.buf);
6:  d8ae2eadb98 = 6:  33ffcbbc1f0 core doc: modernize core.bigFileThreshold documentation
7:  2b403e7cd9c = 7:  11f7aa026b4 unpack-objects: refactor away unpack_non_delta_entry()
8:  5eded902496 = 8:  34ee6a28a54 unpack-objects: use stream_loose_object() to unpack large objects
Signature · 1 line
2.35.1.1548.g36973b18e52
#72 lore raw

PATCH v12 1/8 unpack-objects: low memory footprint for get_data() in dry_run mode

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).

---
 builtin/unpack-objects.c        | 34 ++++++++++++++++++---------
 t/t5351-unpack-large-objects.sh | 41 +++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100755 t/t5351-unpack-large-objects.sh
builtin/unpack-objects.c +23 -11
index dbeb0680a58..e3d30025979 100644
@@ -96,15 +96,26 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
+/*
+ * Decompress zstream from stdin and return specific size of data.
+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
 static void *get_data(unsigned long size)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
+	void *buf = xmallocz(bufsize);
 
 	memset(&stream, 0, sizeof(stream));
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -124,8 +135,15 @@ static void *get_data(unsigned long size)
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
+	if (dry_run)
+		FREE_AND_NULL(buf);
 	return buf;
 }
 
@@ -325,10 +343,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 {
 	void *buf = get_data(size);
 
-	if (!dry_run && buf)
+	if (buf)
 		write_object(nr, type, buf, size);
-	else
-		free(buf);
 }
 
 static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +374,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		oidread(&base_oid, fill(the_hash_algo->rawsz));
 		use(the_hash_algo->rawsz);
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		if (has_object_file(&base_oid))
 			; /* Ok we have this one */
 		else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +411,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 			die("offset value out of bound for delta base object");
 
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		lo = 0;
 		hi = nr;
 		while (lo < hi) {
t/t5351-unpack-large-objects.sh new file +41 -0

41 lines not on this page · raw message

Signature · 1 line
2.35.1.1548.g36973b18e52
#73 lore raw

PATCH v12 2/8 object-file.c: do fsync() and close() before post-write die()

In reply to Ævar Arnfjörð Bjarmason

Change write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).

I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.

The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.

---
 object-file.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)
object-file.c +10 -3
index 62ebe236c90..5da458eccbf 100644
@@ -1886,7 +1886,14 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
 	hash_object_file_literally(algo, buf, len, type_name(type), oid);
 }
 
-/* Finalize a file on disk, and close it. */
+/*
+ * We already did a write_buffer() to the "fd", let's fsync()
+ * and close().
+ *
+ * Finalize a file on disk, and close it. We might still die() on a
+ * subsequent sanity check, but let's not add to that confusion by not
+ * flushing any outstanding writes to disk first.
+ */
 static void close_loose_object(int fd)
 {
 	if (the_repository->objects->odb->will_destroy)
@@ -2006,12 +2013,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
 		    ret);
 	the_hash_algo->final_oid_fn(&parano_oid, &c);
+	close_loose_object(fd);
+
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
-	close_loose_object(fd);
-
 	if (mtime) {
 		struct utimbuf utb;
 		utb.actime = mtime;
Signature · 1 line
2.35.1.1548.g36973b18e52
#74 lore raw

PATCH v12 3/8 object-file.c: refactor write_loose_object() to several steps

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.

Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".

2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".

3. Compress data.

4. Move common steps for ending zlib stream into a new function

    "end_loose_object_common()".

5. Close fd and finalize the object file.

---
 object-file.c | 102 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 76 insertions(+), 26 deletions(-)
object-file.c +76 -26
index 5da458eccbf..7f160929e00 100644
@@ -1955,6 +1955,75 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+/**
+ * Common steps for loose object writers to start writing loose
+ * objects:
+ *
+ * - Create tmpfile for the loose object.
+ * - Setup zlib stream for compression.
+ * - Start to feed header to zlib stream.
+ *
+ * Returns a "fd", which should later be provided to
+ * end_loose_object_common().
+ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+				     const char *filename, unsigned flags,
+				     git_zstream *stream,
+				     unsigned char *buf, size_t buflen,
+				     git_hash_ctx *c,
+				     char *hdr, int hdrlen)
+{
+	int fd;
+
+	fd = create_tmpfile(tmp_file, filename);
+	if (fd < 0) {
+		if (flags & HASH_SILENT)
+			return -1;
+		else if (errno == EACCES)
+			return error(_("insufficient permission for adding "
+				       "an object to repository database %s"),
+				     get_object_directory());
+		else
+			return error_errno(
+				_("unable to create temporary file"));
+	}
+
+	/*  Setup zlib stream for compression */
+	git_deflate_init(stream, zlib_compression_level);
+	stream->next_out = buf;
+	stream->avail_out = buflen;
+	the_hash_algo->init_fn(c);
+
+	/*  Start to feed header to zlib stream */
+	stream->next_in = (unsigned char *)hdr;
+	stream->avail_in = hdrlen;
+	while (git_deflate(stream, 0) == Z_OK)
+		; /* nothing */
+	the_hash_algo->update_fn(c, hdr, hdrlen);
+
+	return fd;
+}
+
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "oid".
+ * - fsync() and close() the "fd"
+ */
+static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,
+				   struct object_id *oid)
+{
+	int ret;
+
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		return ret;
+	the_hash_algo->final_oid_fn(oid, c);
+
+	return Z_OK;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1969,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
-	}
-
-	/* Set it up */
-	git_deflate_init(&stream, zlib_compression_level);
-	stream.next_out = compressed;
-	stream.avail_out = sizeof(compressed);
-	the_hash_algo->init_fn(&c);
-
-	/* First header.. */
-	stream.next_in = (unsigned char *)hdr;
-	stream.avail_in = hdrlen;
-	while (git_deflate(&stream, 0) == Z_OK)
-		; /* nothing */
-	the_hash_algo->update_fn(&c, hdr, hdrlen);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0)
+		return -1;
 
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
@@ -2008,11 +2060,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	if (ret != Z_STREAM_END)
 		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
 		    ret);
-	ret = git_deflate_end_gently(&stream);
+	ret = end_loose_object_common(&c, &stream, &parano_oid);
 	if (ret != Z_OK)
-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
-		    ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
+		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
 	close_loose_object(fd);
 
 	if (!oideq(oid, &parano_oid))
Signature · 1 line
2.35.1.1548.g36973b18e52
author
#75 lore raw

Re: [PATCH v12 3/8] object-file.c: refactor write_loose_object() to several steps

Quoted 7 lines · Ævar Arnfjörð Bjarmason

+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "oid".
+ * - fsync() and close() the "fd"

Since we removed close_loose_object() from end_loose_object_common() , I think this comment should also be removed.

Thanks.
-Han Xin

Quoted 14 lines

+ */

object-file.c +13 -0
+static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,
+				   struct object_id *oid)
+{
+	int ret;
+
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		return ret;
+	the_hash_algo->final_oid_fn(oid, c);
+
+	return Z_OK;
+}
+
#76 lore raw

Re: [PATCH v12 3/8] object-file.c: refactor write_loose_object() to several steps

Quoted 11 lines · Han Xin
Quoted 7 lines · Ævar Arnfjörð Bjarmason

+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "oid".
+ * - fsync() and close() the "fd"

Since we removed close_loose_object() from end_loose_object_common() , I think this comment should also be removed.

You're right. I adjusted it for the "parano_oid" in this v12, but managed to miss that somehow.

Will submit a re-roll with those changes, but will wait a bit more to see if there's any other comments on this v12 first. Thanks!

#77 lore raw

PATCH v12 4/8 object-file.c: factor out deflate part of write_loose_object()

In reply to Ævar Arnfjörð Bjarmason

Split out the part of write_loose_object() that deals with calling git_deflate() into a utility function, a subsequent commit will introduce another function that'll make use of it.

---
 object-file.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)
object-file.c +25 -6
index 7f160929e00..6e2f2264f8c 100644
@@ -2004,6 +2004,28 @@ static int start_loose_object_common(struct strbuf *tmp_file,
 	return fd;
 }
 
+/**
+ * Common steps for the inner git_deflate() loop for writing loose
+ * objects. Returns what git_deflate() returns.
+ */
+static int write_loose_object_common(git_hash_ctx *c,
+				     git_zstream *stream, const int flush,
+				     unsigned char *in0, const int fd,
+				     unsigned char *compressed,
+				     const size_t compressed_len)
+{
+	int ret;
+
+	ret = git_deflate(stream, flush ? Z_FINISH : 0);
+	the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+	if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+		die(_("unable to write loose object file"));
+	stream->next_out = compressed;
+	stream->avail_out = compressed_len;
+
+	return ret;
+}
+
 /**
  * Common steps for loose object writers to end writing loose objects:
  *
@@ -2049,12 +2071,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	stream.avail_in = len;
 	do {
 		unsigned char *in0 = stream.next_in;
-		ret = git_deflate(&stream, Z_FINISH);
-		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-			die(_("unable to write loose object file"));
-		stream.next_out = compressed;
-		stream.avail_out = sizeof(compressed);
+
+		ret = write_loose_object_common(&c, &stream, 1, in0, fd,
+						compressed, sizeof(compressed));
 	} while (ret == Z_OK);
 
 	if (ret != Z_STREAM_END)
Signature · 1 line
2.35.1.1548.g36973b18e52
#78 lore raw

PATCH v12 5/8 object-file.c: add "stream_loose_object()" to handle large object

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".

---
 object-file.c  | 100 +++++++++++++++++++++++++++++++++++++++++++++++++
 object-store.h |   8 ++++
 2 files changed, 108 insertions(+)
object-file.c +100 -0
index 6e2f2264f8c..2be2bae9afa 100644
@@ -2118,6 +2118,106 @@ static int freshen_packed_object(const struct object_id *oid)
 	return 1;
 }
 
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid)
+{
+	int fd, ret, err = 0, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen;
+
+	/* Since oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+	/*
+	 * Common steps for write_loose_object and stream_loose_object to
+	 * start writing loose objects:
+	 *
+	 *  - Create tmpfile for the loose object.
+	 *  - Setup zlib stream for compression.
+	 *  - Start to feed header to zlib stream.
+	 */
+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0) {
+		err = -1;
+		goto cleanup;
+	}
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+
+		if (!stream.avail_in && !in_stream->is_finished) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (in_stream->is_finished)
+				flush = 1;
+		}
+		ret = write_loose_object_common(&c, &stream, flush, in0, fd,
+						compressed, sizeof(compressed));
+		/*
+		 * Unlike write_loose_object(), we do not have the entire
+		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
+		 * then we'll replenish them in the next input_stream->read()
+		 * call when we loop.
+		 */
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (stream.total_in != len + hdrlen)
+		die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
+		    (uintmax_t)len + hdrlen);
+
+	/* Common steps for write_loose_object and stream_loose_object to
+	 * end writing loose oject:
+	 *
+	 *  - End the compression of zlib stream.
+	 *  - Get the calculated oid.
+	 */
+	if (ret != Z_STREAM_END)
+		die(_("unable to stream deflate new object (%d)"), ret);
+	ret = end_loose_object_common(&c, &stream, oid);
+	if (ret != Z_OK)
+		die(_("deflateEnd on stream object failed (%d)"), ret);
+	close_loose_object(fd);
+
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		goto cleanup;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen);
+
+		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
+			err = error_errno(_("unable to create directory %s"), dir.buf);
+			strbuf_release(&dir);
+			goto cleanup;
+		}
+		strbuf_release(&dir);
+	}
+
+	err = finalize_object_file(tmp_file.buf, filename.buf);
+cleanup:
+	strbuf_release(&tmp_file);
+	strbuf_release(&filename);
+	return err;
+}
+
 int write_object_file_flags(const void *buf, unsigned long len,
 			    enum object_type type, struct object_id *oid,
 			    unsigned flags)
object-store.h +8 -0
index bd2322ed8ce..1099455bc2e 100644
@@ -46,6 +46,12 @@ struct object_directory {
 	char *path;
 };
 
+struct input_stream {
+	const void *(*read)(struct input_stream *, unsigned long *len);
+	void *data;
+	int is_finished;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -261,6 +267,8 @@ static inline int write_object_file(const void *buf, unsigned long len,
 int write_object_file_literally(const void *buf, unsigned long len,
 				const char *type, struct object_id *oid,
 				unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid);
 
 /*
  * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.35.1.1548.g36973b18e52
#79 lore raw

Re: [PATCH v12 5/8] object-file.c: add "stream_loose_object()" to handle large object

Quoted 11 lines · Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Just a thought for optimization which you might want to try on top of this series:
try using mmap on both the source and target files of your stream. Use a big 'window' for the mmap (multiple MB) to reduce the TLB flush costs. TLB flush costs should be minimal anyway if Git is single-threaded.

If you can set the source and target buffers of zlib to the source and dest mappings respectively, you'd eliminate two copies of data into Git's stack buffers. You might need to over-allocate the dst file if you don't know the size up front, but doing an over-allocate and truncate should be pretty cheap if you're working with a big file.

Thanks,
Neeraj

#80 lore raw

PATCH v12 6/8 core doc: modernize core.bigFileThreshold documentation

In reply to Ævar Arnfjörð Bjarmason

The core.bigFileThreshold documentation has been largely unchanged since 5eef828bc03 (fast-import: Stream very large blobs directly to pack, 2010-02-01).

But since then this setting has been expanded to affect a lot more than that description indicated. Most notably in how "git diff" treats them, see 6bf3b813486 (diff --stat: mark any file larger than core.bigfilethreshold binary, 2014-08-16).

In addition to that, numerous commands and APIs make use of a streaming mode for files above this threshold.

So let's attempt to summarize 12 years of changes in behavior, which can be seen with:

    git log --oneline -Gbig_file_thre 5eef828bc03.. -- '*.c'

To do that turn this into a bullet-point list. The summary Han Xin produced in [1] helped a lot, but is a bit too detailed for documentation aimed at users. Let's instead summarize how user-observable behavior differs, and generally describe how we tend to stream these files in various commands.

1. https://lore.kernel.org/git/[email protected]/

---
 Documentation/config/core.txt | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)
Documentation/config/core.txt +24 -9
index 9da3e5d88f6..5fccbd56995 100644
@@ -412,17 +412,32 @@ You probably do not need to adjust this value.
 Common unit suffixes of 'k', 'm', or 'g' are supported.
 
 core.bigFileThreshold::
-	Files larger than this size are stored deflated, without
-	attempting delta compression.  Storing large files without
-	delta compression avoids excessive memory usage, at the
-	slight expense of increased disk usage. Additionally files
-	larger than this size are always treated as binary.
+	The size of files considered "big", which as discussed below
+	changes the behavior of numerous git commands, as well as how
+	such files are stored within the repository. The default is
+	512 MiB. Common unit suffixes of 'k', 'm', or 'g' are
+	supported.
 +
-Default is 512 MiB on all platforms.  This should be reasonable
-for most projects as source code and other text files can still
-be delta compressed, but larger binary media files won't be.
+Files above the configured limit will be:
 +
-Common unit suffixes of 'k', 'm', or 'g' are supported.
+* Stored deflated, without attempting delta compression.
++
+The default limit is primarily set with this use-case in mind. With it
+most projects will have their source code and other text files delta
+compressed, but not larger binary media files.
++
+Storing large files without delta compression avoids excessive memory
+usage, at the slight expense of increased disk usage.
++
+* Will be treated as if though they were labeled "binary" (see
+  linkgit:gitattributes[5]). This means that e.g. linkgit:git-log[1]
+  and linkgit:git-diff[1] will not diffs for files above this limit.
++
+* Will be generally be streamed when written, which avoids excessive
+memory usage, at the cost of some fixed overhead. Commands that make
+use of this include linkgit:git-archive[1],
+linkgit:git-fast-import[1], linkgit:git-index-pack[1] and
+linkgit:git-fsck[1].
 
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
Signature · 1 line
2.35.1.1548.g36973b18e52
#81 lore raw

PATCH v12 7/8 unpack-objects: refactor away unpack_non_delta_entry()

In reply to Ævar Arnfjörð Bjarmason

The unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.

Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.

This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about. As it'll be handling "OBJ_BLOB" in a special manner let's re-arrange that "case" in preparation for that change.

---
 builtin/unpack-objects.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
builtin/unpack-objects.c +7 -11
index e3d30025979..d374599d544 100644
@@ -338,15 +338,6 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
-static void unpack_non_delta_entry(enum object_type type, unsigned long size,
-				   unsigned nr)
-{
-	void *buf = get_data(size);
-
-	if (buf)
-		write_object(nr, type, buf, size);
-}
-
 static int resolve_against_held(unsigned nr, const struct object_id *base,
 				void *delta_data, unsigned long delta_size)
 {
@@ -479,12 +470,17 @@ static void unpack_one(unsigned nr)
 	}
 
 	switch (type) {
+	case OBJ_BLOB:
 	case OBJ_COMMIT:
 	case OBJ_TREE:
-	case OBJ_BLOB:
 	case OBJ_TAG:
-		unpack_non_delta_entry(type, size, nr);
+	{
+		void *buf = get_data(size);
+
+		if (buf)
+			write_object(nr, type, buf, size);
 		return;
+	}
 	case OBJ_REF_DELTA:
 	case OBJ_OFS_DELTA:
 		unpack_delta_entry(type, size, nr);
Signature · 1 line
2.35.1.1548.g36973b18e52
#82 lore raw

Re: [PATCH v12 7/8] unpack-objects: refactor away unpack_non_delta_entry()

Am 29.03.22 um 15:56 schrieb Ævar Arnfjörð Bjarmason:

Quoted 4 lines

The unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.

Quoted 8 lines

Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.

This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about.

How exactly does inlining the function make the next patch easier to understand or discuss? Plugging in the stream_blob() call to handle the big blobs looks the same with or without the unpack_non_delta_entry() call to me.

Quoted 8 lines

As it'll be handling "OBJ_BLOB" in a
special manner let's re-arrange that "case" in preparation for that change.

---
 builtin/unpack-objects.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

Reducing the number of lines can be an advantage. *shrug*

Quoted 41 lines
builtin/unpack-objects.c +7 -11
index e3d30025979..d374599d544 100644
@@ -338,15 +338,6 @@ static void added_object(unsigned nr, enum object_type type,
 	}
 }
 
-static void unpack_non_delta_entry(enum object_type type, unsigned long size,
-				   unsigned nr)
-{
-	void *buf = get_data(size);
-
-	if (buf)
-		write_object(nr, type, buf, size);
-}
-
 static int resolve_against_held(unsigned nr, const struct object_id *base,
 				void *delta_data, unsigned long delta_size)
 {
@@ -479,12 +470,17 @@ static void unpack_one(unsigned nr)
 	}
 
 	switch (type) {
+	case OBJ_BLOB:
 	case OBJ_COMMIT:
 	case OBJ_TREE:
-	case OBJ_BLOB:
 	case OBJ_TAG:
-		unpack_non_delta_entry(type, size, nr);
+	{
+		void *buf = get_data(size);
+
+		if (buf)
+			write_object(nr, type, buf, size);
 		return;
+	}
 	case OBJ_REF_DELTA:
 	case OBJ_OFS_DELTA:
 		unpack_delta_entry(type, size, nr);
#83 lore raw

Re: [PATCH v12 7/8] unpack-objects: refactor away unpack_non_delta_entry()

Quoted 10 lines · René Scharfe

Am 29.03.22 um 15:56 schrieb Ævar Arnfjörð Bjarmason:

Quoted 9 lines

The unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.

Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.

Quoted 8 lines
Quoted 3 lines

This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about.

How exactly does inlining the function make the next patch easier to understand or discuss? Plugging in the stream_blob() call to handle the big blobs looks the same with or without the unpack_non_delta_entry() call to me.

The earlier version of it without this prep cleanup can be seen at
https://lore.kernel.org/git/[email protected]/

So yes, this could be skipped, but I tought with this step it was easier to understand.

We previously had to change "void *buf = get_data(size);" in the function to just "void *buf", and do the assignment after the condition that's being checked here.

I think it's also more obvious in terms of control flow if we're checking OBJ_BLOB here to not call a function which has a special-case just for OBJ_BLOB, we can just do that here instead.

Quoted 10 lines
Quoted 8 lines

As it'll be handling "OBJ_BLOB" in a
special manner let's re-arrange that "case" in preparation for that change.

---
 builtin/unpack-objects.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

Reducing the number of lines can be an advantage. *shrug*

There was also the (admittedly rather small) knock-on-effect on 8/8. Before this it was 8 lines added / 1 removed when it came to the code impacted by this change, now it's a 5 added/0 removed in the below "switch".

So I think it's worth keeping.

#84 lore raw

Re: [PATCH v12 7/8] unpack-objects: refactor away unpack_non_delta_entry()

Am 31.03.22 um 14:42 schrieb Ævar Arnfjörð Bjarmason:

Quoted 17 lines · René Scharfe
Quoted 15 lines

Am 29.03.22 um 15:56 schrieb Ævar Arnfjörð Bjarmason:

Quoted 12 lines

The unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.

Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.

This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about.

Quoted 7 lines
Quoted 4 lines

How exactly does inlining the function make the next patch easier to understand or discuss? Plugging in the stream_blob() call to handle the big blobs looks the same with or without the unpack_non_delta_entry() call to me.

The earlier version of it without this prep cleanup can be seen at
https://lore.kernel.org/git/[email protected]/

This plugged the special case into unpack_non_delta_entry(). The alternative I had in mind was to plug it into the switch statement as the current patch does, just without inlining unpack_non_delta_entry().

Quoted 28 lines

So yes, this could be skipped, but I tought with this step it was easier to understand.

We previously had to change "void *buf = get_data(size);" in the function to just "void *buf", and do the assignment after the condition that's being checked here.

I think it's also more obvious in terms of control flow if we're checking OBJ_BLOB here to not call a function which has a special-case just for OBJ_BLOB, we can just do that here instead.

Quoted 10 lines
Quoted 8 lines

As it'll be handling "OBJ_BLOB" in a
special manner let's re-arrange that "case" in preparation for that change.

---
 builtin/unpack-objects.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

Reducing the number of lines can be an advantage. *shrug*

There was also the (admittedly rather small) knock-on-effect on 8/8. Before this it was 8 lines added / 1 removed when it came to the code impacted by this change, now it's a 5 added/0 removed in the below "switch".

So I think it's worth keeping.

#85 lore raw

PATCH v12 8/8 unpack-objects: use stream_loose_object() to unpack large objects

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

Make use of the stream_loose_object() function introduced in the preceding commit to unpack large objects. Before this we'd need to malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.

We could use the new streaming interface to unpack all blobs, but doing so would be much slower, as demonstrated e.g. with this benchmark using git-hyperfine[0]:

	rm -rf /tmp/scalar.git &&
	git clone --bare https://github.com/Microsoft/scalar.git /tmp/scalar.git &&
	mv /tmp/scalar.git/objects/pack/*.pack /tmp/scalar.git/my.pack &&
	git hyperfine \
		-r 2 --warmup 1 \
		-L rev origin/master,HEAD -L v "10,512,1k,1m" \
		-s 'make' \
		-p 'git init --bare dest.git' \
		-c 'rm -rf dest.git' \
		'./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/scalar.git/my.pack'

Here we'll perform worse with lower core.bigFileThreshold settings with this change in terms of speed, but we're getting lower memory use in return:

Summary
'./git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' ran
1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.02 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.02 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.09 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.10 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.11 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'

A better benchmark to demonstrate the benefits of that this one, which creates an artificial repo with a 1, 25, 50, 75 and 100MB blob:

	rm -rf /tmp/repo &&
	git init /tmp/repo &&
	(
		cd /tmp/repo &&
		for i in 1 25 50 75 100
		do
			dd if=/dev/urandom of=blob.$i count=$(($i*1024)) bs=1024
		done &&
		git add blob.* &&
		git commit -mblobs &&
		git gc &&
		PACK=$(echo .git/objects/pack/pack-*.pack) &&
		cp "$PACK" my.pack
	) &&
	git hyperfine \
		--show-output \
		-L rev origin/master,HEAD -L v "512,50m,100m" \
		-s 'make' \
		-p 'git init --bare dest.git' \
		-c 'rm -rf dest.git' \
		'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum'

Using this test we'll always use >100MB of memory on origin/master (around ~105MB), but max out at e.g. ~55MB if we set core.bigFileThreshold=50m.

The relevant "Maximum resident set size" lines were manually added below the relevant benchmark:

'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master' ran
Maximum resident set size (kbytes): 107080
1.02 ± 0.78 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 106968
1.09 ± 0.79 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 107032
1.42 ± 1.07 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 107072
1.83 ± 1.02 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 55704
2.16 ± 1.19 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 4564

This shows that if you have enough memory this new streaming method is slower the lower you set the streaming threshold, but the benefit is more bounded memory use.

An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing "core.bigFileThreshold" variable[1]. As noted in a detailed overview of its users in [2] using it has several different meanings.

Still, we consider it good enough to simply re-use it. While it's possible that someone might want to e.g. consider objects "small" for the purposes of diffing but "big" for the purposes of writing them such use-cases are probably too obscure to worry about. We can always split up "core.bigFileThreshold" in the future if there's a need for that.

0. https://github.com/avar/git-hyperfine/
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/

---
 Documentation/config/core.txt   |  4 +-
 builtin/unpack-objects.c        | 67 +++++++++++++++++++++++++++++++++
 t/t5351-unpack-large-objects.sh | 26 +++++++++++--
 3 files changed, 92 insertions(+), 5 deletions(-)
Documentation/config/core.txt +2 -2
index 5fccbd56995..716259b6762 100644
@@ -436,8 +436,8 @@ usage, at the slight expense of increased disk usage.
 * Will be generally be streamed when written, which avoids excessive
 memory usage, at the cost of some fixed overhead. Commands that make
 use of this include linkgit:git-archive[1],
-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and
-linkgit:git-fsck[1].
+linkgit:git-fast-import[1], linkgit:git-index-pack[1],
+linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
 
 core.excludesFile::
 	Specifies the pathname to the file that contains patterns to
builtin/unpack-objects.c +67 -0

79 lines not on this page · raw message

t/t5351-unpack-large-objects.sh +23 -3
index 8d84313221c..461ca060b2b 100755
@@ -9,7 +9,8 @@ test_description='git unpack-objects with large objects'
 
 prepare_dest () {
 	test_when_finished "rm -rf dest.git" &&
-	git init --bare dest.git
+	git init --bare dest.git &&
+	git -C dest.git config core.bigFileThreshold "$1"
 }
 
 test_expect_success "create large objects (1.5 MB) and PACK" '
@@ -26,16 +27,35 @@ test_expect_success 'set memory limitation to 1MB' '
 '
 
 test_expect_success 'unpack-objects failed under memory limitation' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err &&
 	grep "fatal: attempting to allocate" err
 '
 
 test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
-	prepare_dest &&
+	prepare_dest 2m &&
 	git -C dest.git unpack-objects -n <pack-$PACK.pack &&
 	test_stdout_line_count = 0 find dest.git/objects -type f &&
 	test_dir_is_empty dest.git/objects/pack
 '
 
+test_expect_success 'unpack big object in stream' '
+	prepare_dest 1m &&
+	git -C dest.git unpack-objects <pack-$PACK.pack &&
+	test_dir_is_empty dest.git/objects/pack
+'
+
+test_expect_success 'do not unpack existing large objects' '
+	prepare_dest 1m &&
+	git -C dest.git index-pack --stdin <pack-$PACK.pack &&
+	git -C dest.git unpack-objects <pack-$PACK.pack &&
+
+	# The destination came up with the exact same pack...
+	DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) &&
+	test_cmp pack-$PACK.pack $DEST_PACK &&
+
+	# ...and wrote no loose objects
+	test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*"
+'
+
 test_done
Signature · 1 line
2.35.1.1548.g36973b18e52
#86 lore raw

v13 cover unpack-objects: support streaming blobs to disk

In reply to Ævar Arnfjörð Bjarmason

This series makes "unpack-objects" capable of streaming large objects to disk.

As 7/7 shows streaming e.g. a 100MB blob now uses ~5MB of memory instead of ~105MB. This streaming method is slower if you've got memory to handle the blobs in-core, but if you don't it allows you to unpack objects at all, as you might otherwise OOM.

This series by Han Xin was originally waiting on some in-flight patches that landed in 430883a70c7 (Merge branch 'ab/object-file-api-updates', 2022-03-16), and until yesterday with 83937e95928 (Merge branch 'ns/batch-fsync', 2022-06-03) had a textual and semantic conflict with "master".

Changes since v12:

* Since v12 Han Xin submitted 1/1 here as
https://lore.kernel.org/git/[email protected]/;
I think this is better off reviewed as a whole, and hopefully will be picked up as such.

* Dropped the previous 7/8, which was a refactoring to make 8/8 slightly smaller. Per dicsussion with René it's better to leave it out.

* The rest (especially 2/8) is due to rebasing on ns/batch-fsync.

Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects

Ævar Arnfjörð Bjarmason (3):
object-file.c: do fsync() and close() before post-write die() object-file.c: factor out deflate part of write_loose_object() core doc: modernize core.bigFileThreshold documentation

 Documentation/config/core.txt   |  33 +++--
 builtin/unpack-objects.c        | 103 ++++++++++++--
 object-file.c                   | 237 +++++++++++++++++++++++++++-----
 object-store.h                  |   8 ++
 t/t5351-unpack-large-objects.sh |  61 ++++++++
 5 files changed, 387 insertions(+), 55 deletions(-)
 create mode 100755 t/t5351-unpack-large-objects.sh

Range-diff against v12:

1:  e95f6a1cfb6 = 1:  12873fc9915 unpack-objects: low memory footprint for get_data() in dry_run mode
2:  54060eb8c6b ! 2:  b3568f0c5c0 object-file.c: do fsync() and close() before post-write die()
    @@ Commit message
         Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
     
      ## object-file.c ##
    -@@ object-file.c: void hash_object_file(const struct git_hash_algo *algo, const void *buf,
    - 	hash_object_file_literally(algo, buf, len, type_name(type), oid);
    - }
    - 
    --/* Finalize a file on disk, and close it. */
    -+/*
    -+ * We already did a write_buffer() to the "fd", let's fsync()
    -+ * and close().
    -+ *
    -+ * Finalize a file on disk, and close it. We might still die() on a
    -+ * subsequent sanity check, but let's not add to that confusion by not
    -+ * flushing any outstanding writes to disk first.
    -+ */
    - static void close_loose_object(int fd)
    - {
    - 	if (the_repository->objects->odb->will_destroy)
     @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
      		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
      		    ret);
      	the_hash_algo->final_oid_fn(&parano_oid, &c);
    -+	close_loose_object(fd);
    ++	close_loose_object(fd, tmp_file.buf);
     +
      	if (!oideq(oid, &parano_oid))
      		die(_("confused by unstable object source data for %s"),
      		    oid_to_hex(oid));
      
    --	close_loose_object(fd);
    +-	close_loose_object(fd, tmp_file.buf);
     -
      	if (mtime) {
      		struct utimbuf utb;
3:  3dcaa5d6589 ! 3:  9dc0f56878a object-file.c: refactor write_loose_object() to several steps
    @@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
     -		    ret);
     -	the_hash_algo->final_oid_fn(&parano_oid, &c);
     +		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
    - 	close_loose_object(fd);
    + 	close_loose_object(fd, tmp_file.buf);
      
      	if (!oideq(oid, &parano_oid))
4:  03f4e91ac89 = 4:  a0434835fe7 object-file.c: factor out deflate part of write_loose_object()
5:  3d64cf1cf33 ! 5:  0b07b29836b object-file.c: add "stream_loose_object()" to handle large object
    @@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
     +	ret = end_loose_object_common(&c, &stream, oid);
     +	if (ret != Z_OK)
     +		die(_("deflateEnd on stream object failed (%d)"), ret);
    -+	close_loose_object(fd);
    ++	close_loose_object(fd, tmp_file.buf);
     +
     +	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
     +		unlink_or_warn(tmp_file.buf);
6:  33ffcbbc1f0 = 6:  5ed79c58b18 core doc: modernize core.bigFileThreshold documentation
7:  11f7aa026b4 < -:  ----------- unpack-objects: refactor away unpack_non_delta_entry()
8:  34ee6a28a54 ! 7:  5bc8fa9bc8d unpack-objects: use stream_loose_object() to unpack large objects

@@ Documentation/config/core.txt: usage, at the slight expense of increased disk us
Specifies the pathname to the file that contains patterns to

## builtin/unpack-objects.c ##
-@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
- }
+@@ builtin/unpack-objects.c: static void unpack_non_delta_entry(enum object_type type, unsigned long size,
+ write_object(nr, type, buf, size);
}

+struct input_zstream_data {
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
void *delta_data, unsigned long delta_size)
{
@@ builtin/unpack-objects.c: static void unpack_one(unsigned nr)
+ }

switch (type) {
- case OBJ_BLOB:
++ case OBJ_BLOB:
+ if (!dry_run && size > big_file_threshold) {
+ stream_blob(size, nr);
+ return;
@@ builtin/unpack-objects.c: static void unpack_one(unsigned nr)
+ /* fallthrough */
case OBJ_COMMIT:
case OBJ_TREE:
+- case OBJ_BLOB:
case OBJ_TAG:
+ unpack_non_delta_entry(type, size, nr);
+ return;

## t/t5351-unpack-large-objects.sh ##
@@ t/t5351-unpack-large-objects.sh: test_description='git unpack-objects with large objects'

Signature · 1 line
2.36.1.1124.g52838f02905
#87 lore raw

PATCH v13 1/7 unpack-objects: low memory footprint for get_data() in dry_run mode

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).

---
 builtin/unpack-objects.c        | 34 ++++++++++++++++++---------
 t/t5351-unpack-large-objects.sh | 41 +++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100755 t/t5351-unpack-large-objects.sh
builtin/unpack-objects.c +23 -11
index 56d05e2725d..64abba8dbac 100644
@@ -97,15 +97,26 @@ static void use(int bytes)
 	display_throughput(progress, consumed_bytes);
 }
 
+/*
+ * Decompress zstream from stdin and return specific size of data.
+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
 static void *get_data(unsigned long size)
 {
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
+	void *buf = xmallocz(bufsize);
 
 	memset(&stream, 0, sizeof(stream));
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -125,8 +136,15 @@ static void *get_data(unsigned long size)
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
+	if (dry_run)
+		FREE_AND_NULL(buf);
 	return buf;
 }
 
@@ -326,10 +344,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 {
 	void *buf = get_data(size);
 
-	if (!dry_run && buf)
+	if (buf)
 		write_object(nr, type, buf, size);
-	else
-		free(buf);
 }
 
 static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -359,10 +375,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		oidread(&base_oid, fill(the_hash_algo->rawsz));
 		use(the_hash_algo->rawsz);
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		if (has_object_file(&base_oid))
 			; /* Ok we have this one */
 		else if (resolve_against_held(nr, &base_oid,
@@ -398,10 +412,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 			die("offset value out of bound for delta base object");
 
 		delta_data = get_data(delta_size);
-		if (dry_run || !delta_data) {
-			free(delta_data);
+		if (!delta_data)
 			return;
-		}
 		lo = 0;
 		hi = nr;
 		while (lo < hi) {
t/t5351-unpack-large-objects.sh new file +41 -0

41 lines not on this page · raw message

Signature · 1 line
2.36.1.1124.g52838f02905
#88 lore raw

Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode

Quoted 11 lines · Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore,

"reuse" -> "use"

Quoted 2 lines

in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

It makes it sound as if we used to return garbage data, but I do not think that is what happened in reality. Perhaps rewrite the last sentence like

Make the function return NULL in the dry-run mode, as no callers use the returned buffer.

or something?

The overall logic sounds quite sensible.

Quoted 7 lines

The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).

+/*
+ * Decompress zstream from stdin and return specific size of data.

"specific size"? The caller specifies the size of data (because it knows a-priori how many bytes the zstream should inflate to), so

Decompress zstream from the standard input into a newly allocated buffer of specified size and return the buffer.

or something, perhaps. In any case, it needs to say that the caller is responsible for giving the "right" size.

Quoted 10 lines

+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
static void *get_data(unsigned long size)
{

builtin/unpack-objects.c static void use(int bytes) · line 114

Quoted 4 lines
builtin/unpack-objects.c +2 -1
 	git_zstream stream;
-	void *buf = xmallocz(size);
+	unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
+	void *buf = xmallocz(bufsize);

OK.

builtin/unpack-objects.c static void *get_data(unsigned long size)

Quoted 9 lines
builtin/unpack-objects.c +1 -1
 	memset(&stream, 0, sizeof(stream));
 
 	stream.next_out = buf;
-	stream.avail_out = size;
+	stream.avail_out = bufsize;
 	stream.next_in = fill(1);
 	stream.avail_in = len;
 	git_inflate_init(&stream);
@@ -125,8 +136,15 @@ static void *get_data(unsigned long size)

What's hidden in the pre-context is this bit:

		int ret = git_inflate(&stream, 0);
		use(len - stream.avail_in);
		if (stream.total_out == size && ret == Z_STREAM_END)
			break;
		if (ret != Z_OK) {
			error("inflate returned %d", ret);
			FREE_AND_NULL(buf);
			if (!recover)
				exit(1);
			has_errors = 1;
			break;
		}

and it is correct to use "size", not "bufsize", for this check. Unless we receive exactly the caller-specified "size" bytes from the inflated zstream with Z_STREAM_END, we want to detect an error and bail out.

I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so

(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.

(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.

With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?

Quoted 14 lines
builtin/unpack-objects.c +7 -0
 		}
 		stream.next_in = fill(1);
 		stream.avail_in = len;
+		if (dry_run) {
+			/* reuse the buffer in dry_run mode */
+			stream.next_out = buf;
+			stream.avail_out = bufsize;
+		}
 	}
 	git_inflate_end(&stream);
+	if (dry_run)
+		FREE_AND_NULL(buf);
 	return buf;
 }
author
#89 lore raw

Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode

Quoted 107 lines · Junio C Hamano
Quoted 11 lines · Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.

Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore,

"reuse" -> "use"

Quoted 2 lines

in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.

It makes it sound as if we used to return garbage data, but I do not think that is what happened in reality. Perhaps rewrite the last sentence like

Make the function return NULL in the dry-run mode, as no callers use the returned buffer.

or something?

The overall logic sounds quite sensible.

Quoted 7 lines

The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).

+/*
+ * Decompress zstream from stdin and return specific size of data.

"specific size"? The caller specifies the size of data (because it knows a-priori how many bytes the zstream should inflate to), so

Decompress zstream from the standard input into a newly allocated buffer of specified size and return the buffer.

or something, perhaps. In any case, it needs to say that the caller is responsible for giving the "right" size.

Quoted 11 lines

+ * The caller is responsible to free the returned buffer.
+ *
+ * But for dry_run mode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, in dry_run mode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
static void *get_data(unsigned long size)

 {
      git_zstream stream;

builtin/unpack-objects.c static void use(int bytes) · line 114

Quoted 3 lines
builtin/unpack-objects.c +2 -1
-     void *buf = xmallocz(size);
+     unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
+     void *buf = xmallocz(bufsize);

OK.

builtin/unpack-objects.c static void *get_data(unsigned long size)

Quoted 9 lines
      memset(&stream, 0, sizeof(stream));

      stream.next_out = buf;
builtin/unpack-objects.c +1 -1
-     stream.avail_out = size;
+     stream.avail_out = bufsize;
      stream.next_in = fill(1);
      stream.avail_in = len;
      git_inflate_init(&stream);
@@ -125,8 +136,15 @@ static void *get_data(unsigned long size)

What's hidden in the pre-context is this bit:

                int ret = git_inflate(&stream, 0);
                use(len - stream.avail_in);
                if (stream.total_out == size && ret == Z_STREAM_END)
                        break;
                if (ret != Z_OK) {
                        error("inflate returned %d", ret);
                        FREE_AND_NULL(buf);
                        if (!recover)
                                exit(1);
                        has_errors = 1;
                        break;
                }

and it is correct to use "size", not "bufsize", for this check. Unless we receive exactly the caller-specified "size" bytes from the inflated zstream with Z_STREAM_END, we want to detect an error and bail out.

I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so

(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.

(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.

Quoted 7 lines

With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?

Unlike the original implementation, if we get a corrupted zstream, we won't break at Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.

Thanks.
-Han Xin

Quoted 14 lines
Quoted 14 lines
builtin/unpack-objects.c +7 -0
              }
              stream.next_in = fill(1);
              stream.avail_in = len;
+             if (dry_run) {
+                     /* reuse the buffer in dry_run mode */
+                     stream.next_out = buf;
+                     stream.avail_out = bufsize;
+             }
      }
      git_inflate_end(&stream);
+     if (dry_run)
+             FREE_AND_NULL(buf);
      return buf;
 }
#90 lore raw

Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode

Quoted 20 lines · Han Xin
Quoted 12 lines

I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so

(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.

(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.

Quoted 7 lines

With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?

Quoted 4 lines

Unlike the original implementation, if we get a corrupted zstream, we won't break at Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.

That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?

author
#91 lore raw

Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode

Quoted 28 lines · Junio C Hamano
Quoted 20 lines · Han Xin
Quoted 12 lines

I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so

(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.

(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.

Quoted 7 lines

With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?

Quoted 4 lines

Unlike the original implementation, if we get a corrupted zstream, we won't break at Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.

Quoted 7 lines

That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?

Yes, you are right.

Instead of always using a fixed "bufsize" even if there is not enough expected output remaining, we can get a more accurate one by comparing "total_out" to "size", so we can catch problems early by getting Z_BUFFER_ERROR.

builtin/unpack-objects.c +2 -1
index 64abba8dba..5d59144883 100644
@@ -139,7 +139,8 @@ static void *get_data(unsigned long size)
                if (dry_run) {
                        /* reuse the buffer in dry_run mode */
                        stream.next_out = buf;
-                       stream.avail_out = bufsize;
+                       stream.avail_out = bufsize > size - stream.total_out ?
+                               size - stream.total_out : bufsize;
                }
        }
        git_inflate_end(&stream);

Thanks
-Han Xin

#92 lore raw

Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode

Quoted 59 lines · Han Xin
Quoted 28 lines · Junio C Hamano
Quoted 20 lines · Han Xin
Quoted 12 lines

I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so

(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.

(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.

Quoted 7 lines

With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?

Quoted 4 lines

Unlike the original implementation, if we get a corrupted zstream, we won't break at Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.

Quoted 7 lines

That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?

Yes, you are right.

Instead of always using a fixed "bufsize" even if there is not enough expected output remaining, we can get a more accurate one by comparing "total_out" to "size", so we can catch problems early by getting Z_BUFFER_ERROR.

builtin/unpack-objects.c +2 -1
index 64abba8dba..5d59144883 100644
@@ -139,7 +139,8 @@ static void *get_data(unsigned long size)
                if (dry_run) {
                        /* reuse the buffer in dry_run mode */
                        stream.next_out = buf;
-                       stream.avail_out = bufsize;
+                       stream.avail_out = bufsize > size - stream.total_out ?
+                               size - stream.total_out : bufsize;
                }
        }
        git_inflate_end(&stream);
Quoted 2 lines

Thanks
-Han Xin

Han, do you want to pick this up again for a v14? It looks like you're very on top of it already, and I re-sent your patches because I saw that your
https://lore.kernel.org/git/[email protected]/
wasn't picked up in the interim & you hadn't been active on-list otherwise.

But it looks like there's some interest now, and that you have more time to test & follow-up on this topic than I do at the moment, so if you wanted to do the work of properly rebasing ot in tho recent fsync changes that would be great. Thanks.

author
#93 lore raw

Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode

Quoted 73 lines · Ævar Arnfjörð Bjarmason
Quoted 59 lines · Han Xin
Quoted 28 lines · Junio C Hamano
Quoted 20 lines · Han Xin
Quoted 12 lines

I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so

(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.

(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.

Quoted 7 lines

With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?

Quoted 4 lines

Unlike the original implementation, if we get a corrupted zstream, we won't break at Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.

Quoted 7 lines

That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?

Yes, you are right.

Instead of always using a fixed "bufsize" even if there is not enough expected output remaining, we can get a more accurate one by comparing "total_out" to "size", so we can catch problems early by getting Z_BUFFER_ERROR.

builtin/unpack-objects.c +2 -1
index 64abba8dba..5d59144883 100644
@@ -139,7 +139,8 @@ static void *get_data(unsigned long size)
                if (dry_run) {
                        /* reuse the buffer in dry_run mode */
                        stream.next_out = buf;
-                       stream.avail_out = bufsize;
+                       stream.avail_out = bufsize > size - stream.total_out ?
+                               size - stream.total_out : bufsize;
                }
        }
        git_inflate_end(&stream);
Quoted 2 lines

Thanks
-Han Xin

Han, do you want to pick this up again for a v14? It looks like you're very on top of it already, and I re-sent your patches because I saw that your
https://lore.kernel.org/git/[email protected]/
wasn't picked up in the interim & you hadn't been active on-list otherwise.

Quoted 4 lines

But it looks like there's some interest now, and that you have more time to test & follow-up on this topic than I do at the moment, so if you wanted to do the work of properly rebasing ot in tho recent fsync changes that would be great. Thanks.

OK, I am glad to do that.

Thank you very much.

-Han Xin

#94 lore raw

PATCH v13 2/7 object-file.c: do fsync() and close() before post-write die()

In reply to Ævar Arnfjörð Bjarmason

Change write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).

I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.

The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.

---
 object-file.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
object-file.c +2 -2
index 79eb8339b60..e4a83012ba4 100644
@@ -2012,12 +2012,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
 		    ret);
 	the_hash_algo->final_oid_fn(&parano_oid, &c);
+	close_loose_object(fd, tmp_file.buf);
+
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
-	close_loose_object(fd, tmp_file.buf);
-
 	if (mtime) {
 		struct utimbuf utb;
 		utb.actime = mtime;
Signature · 1 line
2.36.1.1124.g52838f02905
#95 lore raw

Re: [PATCH v13 2/7] object-file.c: do fsync() and close() before post-write die()

Quoted 9 lines · Ævar Arnfjörð Bjarmason

Change write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).

I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.

write_loose_object()

- prepares a temporary file
- deflates into the temporary file
- closes and syncs it
- moves the temporary file to the final locaiton

And any die() inserted in between any of these steps will cause the corrupt temporary file not to become the final loose object file.

So, "git fsck" does not need this change at all.

Quoted 5 lines

The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.

If a "split" relies on the "close and sync" step being in any particular place, that smells really fishy. Is the series loosening the object integrity check? Are we adding some exploitable hole into our codebase without people knowing, or something? I am not sure if I am following the above logic.

Quoted 24 lines
---
 object-file.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
object-file.c +2 -2
index 79eb8339b60..e4a83012ba4 100644
@@ -2012,12 +2012,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
 		    ret);
 	the_hash_algo->final_oid_fn(&parano_oid, &c);
+	close_loose_object(fd, tmp_file.buf);
+
 	if (!oideq(oid, &parano_oid))
 		die(_("confused by unstable object source data for %s"),
 		    oid_to_hex(oid));
 
-	close_loose_object(fd, tmp_file.buf);
-
 	if (mtime) {
 		struct utimbuf utb;
 		utb.actime = mtime;
#96 lore raw

PATCH v13 3/7 object-file.c: refactor write_loose_object() to several steps

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.

Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.

Rewrite "write_loose_object()" as follows:

1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".

2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".

3. Compress data.

4. Move common steps for ending zlib stream into a new function

    "end_loose_object_common()".

5. Close fd and finalize the object file.

---
 object-file.c | 102 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 76 insertions(+), 26 deletions(-)
object-file.c +76 -26
index e4a83012ba4..ce8b52a8dc3 100644
@@ -1951,6 +1951,75 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
 	return fd;
 }
 
+/**
+ * Common steps for loose object writers to start writing loose
+ * objects:
+ *
+ * - Create tmpfile for the loose object.
+ * - Setup zlib stream for compression.
+ * - Start to feed header to zlib stream.
+ *
+ * Returns a "fd", which should later be provided to
+ * end_loose_object_common().
+ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+				     const char *filename, unsigned flags,
+				     git_zstream *stream,
+				     unsigned char *buf, size_t buflen,
+				     git_hash_ctx *c,
+				     char *hdr, int hdrlen)
+{
+	int fd;
+
+	fd = create_tmpfile(tmp_file, filename);
+	if (fd < 0) {
+		if (flags & HASH_SILENT)
+			return -1;
+		else if (errno == EACCES)
+			return error(_("insufficient permission for adding "
+				       "an object to repository database %s"),
+				     get_object_directory());
+		else
+			return error_errno(
+				_("unable to create temporary file"));
+	}
+
+	/*  Setup zlib stream for compression */
+	git_deflate_init(stream, zlib_compression_level);
+	stream->next_out = buf;
+	stream->avail_out = buflen;
+	the_hash_algo->init_fn(c);
+
+	/*  Start to feed header to zlib stream */
+	stream->next_in = (unsigned char *)hdr;
+	stream->avail_in = hdrlen;
+	while (git_deflate(stream, 0) == Z_OK)
+		; /* nothing */
+	the_hash_algo->update_fn(c, hdr, hdrlen);
+
+	return fd;
+}
+
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "oid".
+ * - fsync() and close() the "fd"
+ */
+static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,
+				   struct object_id *oid)
+{
+	int ret;
+
+	ret = git_deflate_end_gently(stream);
+	if (ret != Z_OK)
+		return ret;
+	the_hash_algo->final_oid_fn(oid, c);
+
+	return Z_OK;
+}
+
 static int write_loose_object(const struct object_id *oid, char *hdr,
 			      int hdrlen, const void *buf, unsigned long len,
 			      time_t mtime, unsigned flags)
@@ -1968,28 +2037,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 
 	loose_object_path(the_repository, &filename, oid);
 
-	fd = create_tmpfile(&tmp_file, filename.buf);
-	if (fd < 0) {
-		if (flags & HASH_SILENT)
-			return -1;
-		else if (errno == EACCES)
-			return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
-		else
-			return error_errno(_("unable to create temporary file"));
-	}
-
-	/* Set it up */
-	git_deflate_init(&stream, zlib_compression_level);
-	stream.next_out = compressed;
-	stream.avail_out = sizeof(compressed);
-	the_hash_algo->init_fn(&c);
-
-	/* First header.. */
-	stream.next_in = (unsigned char *)hdr;
-	stream.avail_in = hdrlen;
-	while (git_deflate(&stream, 0) == Z_OK)
-		; /* nothing */
-	the_hash_algo->update_fn(&c, hdr, hdrlen);
+	fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0)
+		return -1;
 
 	/* Then the data itself.. */
 	stream.next_in = (void *)buf;
@@ -2007,11 +2059,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	if (ret != Z_STREAM_END)
 		die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
 		    ret);
-	ret = git_deflate_end_gently(&stream);
+	ret = end_loose_object_common(&c, &stream, &parano_oid);
 	if (ret != Z_OK)
-		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
-		    ret);
-	the_hash_algo->final_oid_fn(&parano_oid, &c);
+		die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
 	close_loose_object(fd, tmp_file.buf);
 
 	if (!oideq(oid, &parano_oid))
Signature · 1 line
2.36.1.1124.g52838f02905
#97 lore raw

PATCH v13 4/7 object-file.c: factor out deflate part of write_loose_object()

In reply to Ævar Arnfjörð Bjarmason

Split out the part of write_loose_object() that deals with calling git_deflate() into a utility function, a subsequent commit will introduce another function that'll make use of it.

---
 object-file.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)
object-file.c +25 -6
index ce8b52a8dc3..7946fa5e088 100644
@@ -2000,6 +2000,28 @@ static int start_loose_object_common(struct strbuf *tmp_file,
 	return fd;
 }
 
+/**
+ * Common steps for the inner git_deflate() loop for writing loose
+ * objects. Returns what git_deflate() returns.
+ */
+static int write_loose_object_common(git_hash_ctx *c,
+				     git_zstream *stream, const int flush,
+				     unsigned char *in0, const int fd,
+				     unsigned char *compressed,
+				     const size_t compressed_len)
+{
+	int ret;
+
+	ret = git_deflate(stream, flush ? Z_FINISH : 0);
+	the_hash_algo->update_fn(c, in0, stream->next_in - in0);
+	if (write_buffer(fd, compressed, stream->next_out - compressed) < 0)
+		die(_("unable to write loose object file"));
+	stream->next_out = compressed;
+	stream->avail_out = compressed_len;
+
+	return ret;
+}
+
 /**
  * Common steps for loose object writers to end writing loose objects:
  *
@@ -2048,12 +2070,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
 	stream.avail_in = len;
 	do {
 		unsigned char *in0 = stream.next_in;
-		ret = git_deflate(&stream, Z_FINISH);
-		the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-			die(_("unable to write loose object file"));
-		stream.next_out = compressed;
-		stream.avail_out = sizeof(compressed);
+
+		ret = write_loose_object_common(&c, &stream, 1, in0, fd,
+						compressed, sizeof(compressed));
 	} while (ret == Z_OK);
 
 	if (ret != Z_STREAM_END)
Signature · 1 line
2.36.1.1124.g52838f02905
#98 lore raw

PATCH v13 5/7 object-file.c: add "stream_loose_object()" to handle large object

In reply to Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".

---
 object-file.c  | 100 +++++++++++++++++++++++++++++++++++++++++++++++++
 object-store.h |   8 ++++
 2 files changed, 108 insertions(+)
object-file.c +100 -0
index 7946fa5e088..9fd449693c4 100644
@@ -2119,6 +2119,106 @@ static int freshen_packed_object(const struct object_id *oid)
 	return 1;
 }
 
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid)
+{
+	int fd, ret, err = 0, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen;
+
+	/* Since oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+	/*
+	 * Common steps for write_loose_object and stream_loose_object to
+	 * start writing loose objects:
+	 *
+	 *  - Create tmpfile for the loose object.
+	 *  - Setup zlib stream for compression.
+	 *  - Start to feed header to zlib stream.
+	 */
+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0) {
+		err = -1;
+		goto cleanup;
+	}
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+
+		if (!stream.avail_in && !in_stream->is_finished) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (in_stream->is_finished)
+				flush = 1;
+		}
+		ret = write_loose_object_common(&c, &stream, flush, in0, fd,
+						compressed, sizeof(compressed));
+		/*
+		 * Unlike write_loose_object(), we do not have the entire
+		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
+		 * then we'll replenish them in the next input_stream->read()
+		 * call when we loop.
+		 */
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (stream.total_in != len + hdrlen)
+		die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
+		    (uintmax_t)len + hdrlen);
+
+	/* Common steps for write_loose_object and stream_loose_object to
+	 * end writing loose oject:
+	 *
+	 *  - End the compression of zlib stream.
+	 *  - Get the calculated oid.
+	 */
+	if (ret != Z_STREAM_END)
+		die(_("unable to stream deflate new object (%d)"), ret);
+	ret = end_loose_object_common(&c, &stream, oid);
+	if (ret != Z_OK)
+		die(_("deflateEnd on stream object failed (%d)"), ret);
+	close_loose_object(fd, tmp_file.buf);
+
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		goto cleanup;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen);
+
+		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
+			err = error_errno(_("unable to create directory %s"), dir.buf);
+			strbuf_release(&dir);
+			goto cleanup;
+		}
+		strbuf_release(&dir);
+	}
+
+	err = finalize_object_file(tmp_file.buf, filename.buf);
+cleanup:
+	strbuf_release(&tmp_file);
+	strbuf_release(&filename);
+	return err;
+}
+
 int write_object_file_flags(const void *buf, unsigned long len,
 			    enum object_type type, struct object_id *oid,
 			    unsigned flags)
object-store.h +8 -0
index 539ea439046..5222ee54600 100644
@@ -46,6 +46,12 @@ struct object_directory {
 	char *path;
 };
 
+struct input_stream {
+	const void *(*read)(struct input_stream *, unsigned long *len);
+	void *data;
+	int is_finished;
+};
+
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
 	struct object_directory *, 1, fspathhash, fspatheq)
 
@@ -269,6 +275,8 @@ static inline int write_object_file(const void *buf, unsigned long len,
 int write_object_file_literally(const void *buf, unsigned long len,
 				const char *type, struct object_id *oid,
 				unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid);
 
 /*
  * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.36.1.1124.g52838f02905
#99 lore raw

Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object

Quoted 10 lines · Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Yay.

Quoted 2 lines

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance.

That is somewhat curious. Is it fundamentally impossible, or is it just that this patch was written in such a way that conflates the two and it is cumbersome to split the "we repeat the sequence of reading and deflating just a bit until we process all" and the "we compute the hash over the data first and then we write out for real"?

Quoted 10 lines

In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

This may have negative implications on some filesystems where cross directory links do not work atomically, but it is a small price to pay.

I am very tempted to ask why we do not do this to _all_ loose object files. Instead of running the machinery twice over the data (once to compute the object name, then to compute the contents and write out), if we can produce loose object files of any size with a single pass, wouldn't that be an overall win?

Is the fixed overhead, i.e. cost of setting up the streaming interface, reasonably large to make it not worth doing for smaller objects?

Quoted 2 lines

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".

That much we can read from the patch text. Saying just "we do X" without explaining "why we do so" in the proposed log message leaves readers more confused than otherwise. Why is it worth pointing out in the proposed log message? Is the reason why we need to do so involve something tricky?

Quoted 50 lines
object-file.c +50 -0
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid)
+{
+	int fd, ret, err = 0, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen;
+
+	/* Since oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+	/*
+	 * Common steps for write_loose_object and stream_loose_object to
+	 * start writing loose objects:
+	 *
+	 *  - Create tmpfile for the loose object.
+	 *  - Setup zlib stream for compression.
+	 *  - Start to feed header to zlib stream.
+	 */
+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0) {
+		err = -1;
+		goto cleanup;
+	}
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+
+		if (!stream.avail_in && !in_stream->is_finished) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (in_stream->is_finished)
+				flush = 1;
+		}
+		ret = write_loose_object_common(&c, &stream, flush, in0, fd,
+						compressed, sizeof(compressed));
+		/*
+		 * Unlike write_loose_object(), we do not have the entire
+		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2181

Quoted 10 lines
object-file.c +8 -0
+		 * then we'll replenish them in the next input_stream->read()
+		 * call when we loop.
+		 */
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
 
+	if (stream.total_in != len + hdrlen)
+		die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
+		    (uintmax_t)len + hdrlen);
 
+	/* Common steps for write_loose_object and stream_loose_object to

Style.

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2188

Quoted 7 lines
object-file.c +7 -0
+	 * end writing loose oject:
+	 *
+	 *  - End the compression of zlib stream.
+	 *  - Get the calculated oid.
+	 */
+	if (ret != Z_STREAM_END)
+		die(_("unable to stream deflate new object (%d)"), ret);

Good to check this, after the loop exits above. I was expecting to see it immediately after the loop, but here is also OK.

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2196

Quoted 8 lines
object-file.c +8 -0
+	ret = end_loose_object_common(&c, &stream, oid);
+	if (ret != Z_OK)
+		die(_("deflateEnd on stream object failed (%d)"), ret);
+	close_loose_object(fd, tmp_file.buf);
+
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		goto cleanup;

So, we were told to write an object, we wrote to a temporary file, and we wanted to mark the object to be recent and found that there indeed is already the object. We remove the temporary and do not leave the new copy of the object, and the value of err at this point is 0 (success) which is what is returned from cleanup: label.

Good.

Quoted 25 lines
object-file.c +25 -0
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+
+	/* We finally know the object path, and create the missing dir. */
+	dirlen = directory_size(filename.buf);
+	if (dirlen) {
+		struct strbuf dir = STRBUF_INIT;
+		strbuf_add(&dir, filename.buf, dirlen);
+
+		if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
+			err = error_errno(_("unable to create directory %s"), dir.buf);
+			strbuf_release(&dir);
+			goto cleanup;
+		}
+		strbuf_release(&dir);
+	}
+
+	err = finalize_object_file(tmp_file.buf, filename.buf);
+cleanup:
+	strbuf_release(&tmp_file);
+	strbuf_release(&filename);
+	return err;
+}
+
#100 lore raw

Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object

Quoted 9 lines · Junio C Hamano
Quoted 2 lines

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance.

That is somewhat curious. Is it fundamentally impossible, or is it just that this patch was written in such a way that conflates the two and it is cumbersome to split the "we repeat the sequence of reading and deflating just a bit until we process all" and the "we compute the hash over the data first and then we write out for real"?

OK, the answer lies somewhere in between.

The initial user of this streaming interface reads from an incoming packfile and feeds the inflated bytestream to the interface, which means we cannot seek. That meaks it "fundamentally impossible" for that codepath (i.e. unpack-objects to read from packstream and write to on-disk loose objects).

But if the input source is seekable (e.g. a file in the working tree), there is no fundamental reason why the new interface has "no chance to run prepare to calculate the oid in advance". It's just that the such a different caller is not added by the series and we chose not to allow the "prepare and then write" two-step process, because we currently do not need it when this series lands.

Quoted 5 lines

I am very tempted to ask why we do not do this to _all_ loose object files. Instead of running the machinery twice over the data (once to compute the object name, then to compute the contents and write out), if we can produce loose object files of any size with a single pass, wouldn't that be an overall win?

There is a patch later in the series whose proposed log message has benchmarks to show that it is slower in general. It still is curious where the slowness comes from and if it is something we can tune, though.

Thanks.

author
#101 lore raw

Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object

Quoted 9 lines · Junio C Hamano
Quoted 5 lines · Junio C Hamano

I am very tempted to ask why we do not do this to _all_ loose object files. Instead of running the machinery twice over the data (once to compute the object name, then to compute the contents and write out), if we can produce loose object files of any size with a single pass, wouldn't that be an overall win?

Quoted 5 lines

There is a patch later in the series whose proposed log message has benchmarks to show that it is slower in general. It still is curious where the slowness comes from and if it is something we can tune, though.

Compared with getting the whole object buffer, stream_loose_object() uses limited avail_in buffer and never fill new content until the whole avail_in has been
deflated. It will generate small avail_in fragments due to the limited avail_out,
and I think it is precisely because these avail_in fragments generate additional git_deflate() loops.

In "unpack-objects", we use a buffer size of 8192. Increasing the buffer can alleviate this problem, but maybe it's not worth it?

Quoted 1 line

Thanks.

author
#102 lore raw

Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object

In reply to Junio C Hamano
Quoted 11 lines · Junio C Hamano
Quoted 4 lines

"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".

That much we can read from the patch text. Saying just "we do X" without explaining "why we do so" in the proposed log message leaves readers more confused than otherwise. Why is it worth pointing out in the proposed log message? Is the reason why we need to do so involve something tricky?

Yes, it really should be made clear why this is done here.

Thanks.
-Han Xin

Quoted 17 lines

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2196

Quoted 8 lines
object-file.c +8 -0
+     ret = end_loose_object_common(&c, &stream, oid);
+     if (ret != Z_OK)
+             die(_("deflateEnd on stream object failed (%d)"), ret);
+     close_loose_object(fd, tmp_file.buf);
+
+     if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+             unlink_or_warn(tmp_file.buf);
+             goto cleanup;

So, we were told to write an object, we wrote to a temporary file, and we wanted to mark the object to be recent and found that there indeed is already the object. We remove the temporary and do not leave the new copy of the object, and the value of err at this point is 0 (success) which is what is returned from cleanup: label.

Good.

#103 lore raw

Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object

In reply to Ævar Arnfjörð Bjarmason
Quoted 18 lines · Ævar Arnfjörð Bjarmason

From: Han Xin <[email protected]>

If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.

Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.

Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.

Quoted 6 lines

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

I think this new logic doesn't play well with batched-fsync. Even through we don't know the final OID, we should still call prepare_loose_object_bulk_checkin to potentially create the bulk checkin objdir.

Quoted 70 lines
object-file.c +62 -0
index 7946fa5e088..9fd449693c4 100644
@@ -2119,6 +2119,106 @@ static int freshen_packed_object(const struct object_id *oid)
  	return 1;
  }
 
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid)
+{
+	int fd, ret, err = 0, flush = 0;
+	unsigned char compressed[4096];
+	git_zstream stream;
+	git_hash_ctx c;
+	struct strbuf tmp_file = STRBUF_INIT;
+	struct strbuf filename = STRBUF_INIT;
+	int dirlen;
+	char hdr[MAX_HEADER_LEN];
+	int hdrlen;
+
+	/* Since oid is not determined, save tmp file to odb path. */
+	strbuf_addf(&filename, "%s/", get_object_directory());
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+	/*
+	 * Common steps for write_loose_object and stream_loose_object to
+	 * start writing loose objects:
+	 *
+	 *  - Create tmpfile for the loose object.
+	 *  - Setup zlib stream for compression.
+	 *  - Start to feed header to zlib stream.
+	 */
+	fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+				       &stream, compressed, sizeof(compressed),
+				       &c, hdr, hdrlen);
+	if (fd < 0) {
+		err = -1;
+		goto cleanup;
+	}
+
+	/* Then the data itself.. */
+	do {
+		unsigned char *in0 = stream.next_in;
+
+		if (!stream.avail_in && !in_stream->is_finished) {
+			const void *in = in_stream->read(in_stream, &stream.avail_in);
+			stream.next_in = (void *)in;
+			in0 = (unsigned char *)in;
+			/* All data has been read. */
+			if (in_stream->is_finished)
+				flush = 1;
+		}
+		ret = write_loose_object_common(&c, &stream, flush, in0, fd,
+						compressed, sizeof(compressed));
+		/*
+		 * Unlike write_loose_object(), we do not have the entire
+		 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
+		 * then we'll replenish them in the next input_stream->read()
+		 * call when we loop.
+		 */
+	} while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+	if (stream.total_in != len + hdrlen)
+		die(_("write stream object %ld != %"PRIuMAX), stream.total_in,
+		    (uintmax_t)len + hdrlen);
+
+	/* Common steps for write_loose_object and stream_loose_object to
+	 * end writing loose oject:
+	 *

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2193

Quoted 10 lines
object-file.c +10 -0
+	 *  - End the compression of zlib stream.
+	 *  - Get the calculated oid.
+	 */
+	if (ret != Z_STREAM_END)
+		die(_("unable to stream deflate new object (%d)"), ret);
+	ret = end_loose_object_common(&c, &stream, oid);
+	if (ret != Z_OK)
+		die(_("deflateEnd on stream object failed (%d)"), ret);
+	close_loose_object(fd, tmp_file.buf);
+

If batch fsync is enabled, the close_loose_object call will refrain from syncing the tmp file.

object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2200

Quoted 7 lines
object-file.c +7 -0
+	if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+		unlink_or_warn(tmp_file.buf);
+		goto cleanup;
+	}
+
+	loose_object_path(the_repository, &filename, oid);
+

We expect this loose_object_path call to return a path in the bulk fsync object directory. It might not do so if we don't call prepare_loose_object_bulk_checkin.

In the new test case introduced in (7/7), we seem to be getting lucky in that there are some small objects (commits) earlier in the packfile, so we go through write_loose_object first.

Thanks for including me on the review!

-Neeraj

#104 lore raw

Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object

Quoted 11 lines · Neeraj Singh
Quoted 6 lines

Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.

I think this new logic doesn't play well with batched-fsync. Even through we don't know the final OID, we should still call prepare_loose_object_bulk_checkin to potentially create the bulk checkin objdir.

Good point. Careful sanity checks like this are very much appreciated.

Quoted 1 line

Thanks for including me on the review!

Yes, indeed.

Thanks, both.

author
#105 lore raw

RFC PATCH 1/1 object-file.c: batched disk flushes for stream_loose_object()

In reply to Neeraj Singh

Neeraj Singh[1] pointed out that if batch fsync is enabled, we should still call prepare_loose_object_bulk_checkin() to potentially create the bulk checkin objdir.

1. https://lore.kernel.org/git/[email protected]/

---
 object-file.c                   |  3 +++
 t/t5351-unpack-large-objects.sh | 15 ++++++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)
object-file.c +3 -0
index 2dd828b45b..3a1be74775 100644
@@ -2131,6 +2131,9 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,
 	char hdr[MAX_HEADER_LEN];
 	int hdrlen;
 
+	if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
+		prepare_loose_object_bulk_checkin();
+
 	/* Since oid is not determined, save tmp file to odb path. */
 	strbuf_addf(&filename, "%s/", get_object_directory());
 	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
t/t5351-unpack-large-objects.sh +14 -1
index 461ca060b2..a66a51f7df 100755
@@ -18,7 +18,10 @@ test_expect_success "create large objects (1.5 MB) and PACK" '
 	test_commit --append foo big-blob &&
 	test-tool genrandom bar 1500000 >big-blob &&
 	test_commit --append bar big-blob &&
-	PACK=$(echo HEAD | git pack-objects --revs pack)
+	PACK=$(echo HEAD | git pack-objects --revs pack) &&
+	git verify-pack -v pack-$PACK.pack |
+	    grep -E "commit|tree|blob" |
+		sed -n -e "s/^\([0-9a-f]*\).*/\1/p" >obj-list
 '
 
 test_expect_success 'set memory limitation to 1MB' '
@@ -45,6 +48,16 @@ test_expect_success 'unpack big object in stream' '
 	test_dir_is_empty dest.git/objects/pack
 '
 
+BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch'
+
+test_expect_success 'unpack big object in stream (core.fsyncmethod=batch)' '
+	prepare_dest 1m &&
+	git $BATCH_CONFIGURATION -C dest.git unpack-objects <pack-$PACK.pack &&
+	test_dir_is_empty dest.git/objects/pack &&
+	git -C dest.git cat-file --batch-check="%(objectname)" <obj-list >current &&
+	cmp obj-list current
+'
+
 test_expect_success 'do not unpack existing large objects' '
 	prepare_dest 1m &&
 	git -C dest.git index-pack --stdin <pack-$PACK.pack &&
Signature · 1 line
2.36.1
#106 lore raw

Re: [RFC PATCH] object-file.c: batched disk flushes for stream_loose_object()

Quoted 49 lines · Han Xin

Neeraj Singh[1] pointed out that if batch fsync is enabled, we should still call prepare_loose_object_bulk_checkin() to potentially create the bulk checkin objdir.

1. https://lore.kernel.org/git/[email protected]/

---
  object-file.c                   |  3 +++
  t/t5351-unpack-large-objects.sh | 15 ++++++++++++++-
  2 files changed, 17 insertions(+), 1 deletion(-)
object-file.c +3 -0
index 2dd828b45b..3a1be74775 100644
@@ -2131,6 +2131,9 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,
  	char hdr[MAX_HEADER_LEN];
  	int hdrlen;
 
+	if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
+		prepare_loose_object_bulk_checkin();
+
  	/* Since oid is not determined, save tmp file to odb path. */
  	strbuf_addf(&filename, "%s/", get_object_directory());
  	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
t/t5351-unpack-large-objects.sh +7 -1
index 461ca060b2..a66a51f7df 100755
@@ -18,7 +18,10 @@ test_expect_success "create large objects (1.5 MB) and PACK" '
  	test_commit --append foo big-blob &&
  	test-tool genrandom bar 1500000 >big-blob &&
  	test_commit --append bar big-blob &&
-	PACK=$(echo HEAD | git pack-objects --revs pack)
+	PACK=$(echo HEAD | git pack-objects --revs pack) &&
+	git verify-pack -v pack-$PACK.pack |
+	    grep -E "commit|tree|blob" |
+		sed -n -e "s/^\([0-9a-f]*\).*/\1/p" >obj-list
  '
 
  test_expect_success 'set memory limitation to 1MB' '
@@ -45,6 +48,16 @@ test_expect_success 'unpack big object in stream' '
  	test_dir_is_empty dest.git/objects/pack
  '
 
+BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch'
+
+test_expect_success 'unpack big object in stream (core.fsyncmethod=batch)' '

t/t5351-unpack-large-objects.sh test_expect_success 'unpack big object in stream' ' · line 63

Quoted 10 lines
t/t5351-unpack-large-objects.sh +7 -0
+	prepare_dest 1m &&
+	git $BATCH_CONFIGURATION -C dest.git unpack-objects <pack-$PACK.pack &&
+	test_dir_is_empty dest.git/objects/pack &&
+	git -C dest.git cat-file --batch-check="%(objectname)" <obj-list >current &&
+	cmp obj-list current
+'
+
  test_expect_success 'do not unpack existing large objects' '
  	prepare_dest 1m &&
  	git -C dest.git index-pack --stdin <pack-$PACK.pack &&

This fix looks good to me.

Thanks.

-Neeraj